v0.3.51: mcp support, copilot improvements, polling for live execution data, bug fixes

* update infra and remove railway

* fix(webooks-ui): made spacing more clear, added copy button for webhook URL & fixed race condition for mcp tools/server fetching in the mcp block

* Revert "update infra and remove railway"

This reverts commit 5a8876209d.

* remove extraneous comments

* ack PR comments

.devcontainer/docker-compose.yml

@@ -77,7 +77,7 @@ services:
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=simstudio
     ports:
-      - "5432:5432"
+      - "${POSTGRES_PORT:-5432}:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s

.github/workflows/build.yml

@@ -2,8 +2,7 @@ name: Build and Publish Docker Image
 
 on:
   push:
-    branches: [main]
-    tags: ['v*']
+    branches: [main, staging]
 
 jobs:
   build-and-push:
@@ -56,7 +55,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to the Container registry
-        if: github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -70,10 +69,7 @@ jobs:
           images: ${{ matrix.image }}
           tags: |
             type=raw,value=latest-${{ matrix.arch }},enable=${{ github.ref == 'refs/heads/main' }}
-            type=ref,event=pr,suffix=-${{ matrix.arch }}
-            type=semver,pattern={{version}},suffix=-${{ matrix.arch }}
-            type=semver,pattern={{major}}.{{minor}},suffix=-${{ matrix.arch }}
-            type=semver,pattern={{major}}.{{minor}}.{{patch}},suffix=-${{ matrix.arch }}
+            type=raw,value=staging-${{ github.sha }}-${{ matrix.arch }},enable=${{ github.ref == 'refs/heads/staging' }}
             type=sha,format=long,suffix=-${{ matrix.arch }}
 
       - name: Build and push Docker image
@@ -82,7 +78,7 @@ jobs:
           context: .
           file: ${{ matrix.dockerfile }}
           platforms: ${{ matrix.platform }}
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,scope=build-v3
@@ -93,7 +89,7 @@ jobs:
   create-manifests:
     runs-on: ubuntu-latest
     needs: build-and-push
-    if: github.event_name != 'pull_request'
+    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
     strategy:
       matrix:
         include:
@@ -119,10 +115,6 @@ jobs:
           images: ${{ matrix.image }}
           tags: |
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}.{{minor}}.{{patch}}
             type=sha,format=long
 
       - name: Create and push manifest

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
           node-version: latest
 
       - name: Install dependencies
-        run: bun install
+        run: bun install --frozen-lockfile
 
       - name: Run tests with coverage
         env:

.github/workflows/trigger-deploy.yml

@@ -35,10 +35,10 @@ jobs:
       - name: Deploy to Staging
         if: github.ref == 'refs/heads/staging'
         working-directory: ./apps/sim
-        run: npx --yes trigger.dev@4.0.0 deploy -e staging
+        run: npx --yes trigger.dev@4.0.1 deploy -e staging
 
       - name: Deploy to Production
         if: github.ref == 'refs/heads/main'
         working-directory: ./apps/sim
-        run: npx --yes trigger.dev@4.0.0 deploy
+        run: npx --yes trigger.dev@4.0.1 deploy
 

README.md

@@ -159,8 +159,7 @@ bun run dev:sockets
 Copilot is a Sim-managed service. To use Copilot on a self-hosted instance:
 
 - Go to https://sim.ai → Settings → Copilot and generate a Copilot API key
-- Set `COPILOT_API_KEY` in your self-hosted environment to that value
-- Host Sim on a publicly available DNS and set NEXT_PUBLIC_APP_URL and BETTER_AUTH_URL to that value ([ngrok](https://ngrok.com/))
+- Set `COPILOT_API_KEY` environment variable in your self-hosted apps/sim/.env file to that value
 
 ## Tech Stack
 
@@ -175,6 +174,7 @@ Copilot is a Sim-managed service. To use Copilot on a self-hosted instance:
 - **Monorepo**: [Turborepo](https://turborepo.org/)
 - **Realtime**: [Socket.io](https://socket.io/)
 - **Background Jobs**: [Trigger.dev](https://trigger.dev/)
+- **Remote Code Execution**: [E2B](https://www.e2b.dev/)
 
 ## Contributing
 

apps/docs/content/docs/blocks/router.mdx

@@ -117,7 +117,7 @@ Your API key for the selected LLM provider. This is securely stored and used for
 
 After a router makes a decision, you can access its outputs:
 
-- **`<router.content>`**: Summary of the routing decision made
+- **`<router.prompt>`**: Summary of the routing prompt used
 - **`<router.selected_path>`**: Details of the chosen destination block
 - **`<router.tokens>`**: Token usage statistics from the LLM
 - **`<router.model>`**: The model used for decision-making
@@ -182,7 +182,7 @@ Confidence Threshold: 0.7 // Minimum confidence for routing
   <Tab>
     <ul className="list-disc space-y-2 pl-6">
       <li>
-        <strong>router.content</strong>: Summary of routing decision
+        <strong>router.prompt</strong>: Summary of routing prompt used
       </li>
       <li>
         <strong>router.selected_path</strong>: Details of chosen destination

apps/docs/content/docs/copilot/index.mdx

@@ -7,8 +7,6 @@ import { Callout } from 'fumadocs-ui/components/callout'
 import { Card, Cards } from 'fumadocs-ui/components/card'
 import { MessageCircle, Package, Zap, Infinity as InfinityIcon, Brain, BrainCircuit } from 'lucide-react'
 
-## What is Copilot
-
 Copilot is your in-editor assistant that helps you build, understand, and improve workflows. It can:
 
 - **Explain**: Answer questions about Sim and your current workflow
@@ -18,35 +16,34 @@ Copilot is your in-editor assistant that helps you build, understand, and improv
 <Callout type="info">
   Copilot is a Sim-managed service. For self-hosted deployments, generate a Copilot API key in the hosted app (sim.ai → Settings → Copilot)
   1. Go to [sim.ai](https://sim.ai) → Settings → Copilot and generate a Copilot API key
-2. Set `COPILOT_API_KEY` in your self-hosted environment to that value
-3. Host Sim on a publicly available DNS and set `NEXT_PUBLIC_APP_URL` and `BETTER_AUTH_URL` to that value (e.g., using ngrok)
+  2. Set `COPILOT_API_KEY` in your self-hosted environment to that value
 </Callout>
 
 ## Modes
 
 <Cards>
-  <Card title="Ask">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <MessageCircle className="h-4 w-4 text-muted-foreground" />
+        Ask
       </span>
-      <div>
-        <p className="m-0 text-sm">
-          Q&A mode for explanations, guidance, and suggestions without making changes to your workflow.
-        </p>
-      </div>
+    }
+  >
+    <div className="m-0 text-sm">
+      Q&A mode for explanations, guidance, and suggestions without making changes to your workflow.
     </div>
   </Card>
-  <Card title="Agent">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <Package className="h-4 w-4 text-muted-foreground" />
+        Agent
       </span>
-      <div>
-        <p className="m-0 text-sm">
-          Build-and-edit mode. Copilot proposes specific edits (add blocks, wire variables, tweak settings) and applies them when you approve.
-        </p>
-      </div>
+    }
+  >
+    <div className="m-0 text-sm">
+      Build-and-edit mode. Copilot proposes specific edits (add blocks, wire variables, tweak settings) and applies them when you approve.
     </div>
   </Card>
 </Cards>
@@ -54,44 +51,71 @@ Copilot is your in-editor assistant that helps you build, understand, and improv
 ## Depth Levels
 
 <Cards>
-  <Card title="Fast">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <Zap className="h-4 w-4 text-muted-foreground" />
+        Fast
       </span>
-      <div>
-        <p className="m-0 text-sm">Quickest and cheapest. Best for small edits, simple workflows, and minor tweaks.</p>
-      </div>
-    </div>
+    }
+  >
+    <div className="m-0 text-sm">Quickest and cheapest. Best for small edits, simple workflows, and minor tweaks.</div>
   </Card>
-  <Card title="Auto">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <InfinityIcon className="h-4 w-4 text-muted-foreground" />
+        Auto
       </span>
-      <div>
-        <p className="m-0 text-sm">Balanced speed and reasoning. Recommended default for most tasks.</p>
-      </div>
-    </div>
+    }
+  >
+    <div className="m-0 text-sm">Balanced speed and reasoning. Recommended default for most tasks.</div>
   </Card>
-  <Card title="Pro">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <Brain className="h-4 w-4 text-muted-foreground" />
+        Advanced
       </span>
-      <div>
-        <p className="m-0 text-sm">More reasoning for larger workflows and complex edits while staying performant.</p>
-      </div>
-    </div>
+    }
+  >
+    <div className="m-0 text-sm">More reasoning for larger workflows and complex edits while staying performant.</div>
   </Card>
-  <Card title="Max">
-    <div className="flex items-start gap-3">
-      <span className="mt-0.5 inline-flex h-8 w-8 items-center justify-center rounded-md border border-border/50 bg-muted/60">
+  <Card
+    title={
+      <span className="inline-flex items-center gap-2">
         <BrainCircuit className="h-4 w-4 text-muted-foreground" />
+        Behemoth
       </span>
-      <div>
-        <p className="m-0 text-sm">Maximum reasoning for deep planning, debugging, and complex architectural changes.</p>
-      </div>
-    </div>
+    }
+  >
+    <div className="m-0 text-sm">Maximum reasoning for deep planning, debugging, and complex architectural changes.</div>
   </Card>
-</Cards> 
+</Cards>
+
+## Billing and Cost Calculation
+
+### How Costs Are Calculated
+
+Copilot usage is billed per token from the underlying LLM:
+
+- **Input tokens**: billed at the provider's base rate (**at-cost**)
+- **Output tokens**: billed at **1.5×** the provider's base output rate
+
+```javascript
+copilotCost = (inputTokens × inputPrice + outputTokens × (outputPrice × 1.5)) / 1,000,000
+```
+
+| Component | Rate Applied         |
+|----------|----------------------|
+| Input    | inputPrice           |
+| Output   | outputPrice × 1.5    |
+
+<Callout type="warning">
+  Pricing shown reflects rates as of September 4, 2025. Check provider documentation for current pricing.
+</Callout>
+
+<Callout type="info">
+  Model prices are per million tokens. The calculation divides by 1,000,000 to get the actual cost. See <a href="/execution/advanced#cost-calculation">Logging and Cost Calculation</a> for background and examples.
+</Callout>
+

apps/docs/content/docs/execution/advanced.mdx

@@ -212,3 +212,47 @@ Monitor your usage and billing in Settings → Subscription:
 - **Usage Limits**: Plan limits with visual progress indicators
 - **Billing Details**: Projected charges and minimum commitments
 - **Plan Management**: Upgrade options and billing history
+
+### Programmatic Rate Limits & Usage (API)
+
+You can query your current API rate limits and usage summary using your API key.
+
+Endpoint:
+
+```text
+GET /api/users/me/usage-limits
+```
+
+Authentication:
+
+- Include your API key in the `X-API-Key` header.
+
+Response (example):
+
+```json
+{
+  "success": true,
+  "rateLimit": {
+    "sync": { "isLimited": false, "limit": 10, "remaining": 10, "resetAt": "2025-09-08T22:51:55.999Z" },
+    "async": { "isLimited": false, "limit": 50, "remaining": 50, "resetAt": "2025-09-08T22:51:56.155Z" },
+    "authType": "api"
+  },
+  "usage": {
+    "currentPeriodCost": 12.34,
+    "limit": 100,
+    "plan": "pro"
+  }
+}
+```
+
+Example:
+
+```bash
+curl -X GET -H "X-API-Key: YOUR_API_KEY" -H "Content-Type: application/json" https://sim.ai/api/users/me/usage-limits
+```
+
+Notes:
+
+- `currentPeriodCost` reflects usage in the current billing period.
+- `limit` is derived from individual limits (Free/Pro) or pooled organization limits (Team/Enterprise).
+- `plan` is the highest-priority active plan associated with your user.

apps/docs/content/docs/execution/api.mdx

@@ -0,0 +1,532 @@
+---
+title: External API
+description: Query workflow execution logs and set up webhooks for real-time notifications
+---
+
+import { Accordion, Accordions } from 'fumadocs-ui/components/accordion'
+import { Callout } from 'fumadocs-ui/components/callout'
+import { Tab, Tabs } from 'fumadocs-ui/components/tabs'
+import { CodeBlock } from 'fumadocs-ui/components/codeblock'
+
+Sim provides a comprehensive external API for querying workflow execution logs and setting up webhooks for real-time notifications when workflows complete.
+
+## Authentication
+
+All API requests require an API key passed in the `x-api-key` header:
+
+```bash
+curl -H "x-api-key: YOUR_API_KEY" \
+  https://sim.ai/api/v1/logs?workspaceId=YOUR_WORKSPACE_ID
+```
+
+You can generate API keys from your user settings in the Sim dashboard.
+
+## Logs API
+
+All API responses include information about your workflow execution limits and usage:
+
+```json
+"limits": {
+  "workflowExecutionRateLimit": {
+    "sync": {
+      "limit": 60,        // Max sync workflow executions per minute
+      "remaining": 58,    // Remaining sync workflow executions
+      "resetAt": "..."    // When the window resets
+    },
+    "async": {
+      "limit": 60,        // Max async workflow executions per minute
+      "remaining": 59,    // Remaining async workflow executions
+      "resetAt": "..."    // When the window resets
+    }
+  },
+  "usage": {
+    "currentPeriodCost": 1.234,  // Current billing period usage in USD
+    "limit": 10,                  // Usage limit in USD
+    "plan": "pro",                // Current subscription plan
+    "isExceeded": false           // Whether limit is exceeded
+  }
+}
+```
+
+**Note:** The rate limits in the response body are for workflow executions. The rate limits for calling this API endpoint are in the response headers (`X-RateLimit-*`).
+
+### Query Logs
+
+Query workflow execution logs with extensive filtering options.
+
+<Tabs items={['Request', 'Response']}>
+  <Tab value="Request">
+    ```http
+    GET /api/v1/logs
+    ```
+
+    **Required Parameters:**
+    - `workspaceId` - Your workspace ID
+
+    **Optional Filters:**
+    - `workflowIds` - Comma-separated workflow IDs
+    - `folderIds` - Comma-separated folder IDs
+    - `triggers` - Comma-separated trigger types: `api`, `webhook`, `schedule`, `manual`, `chat`
+    - `level` - Filter by level: `info`, `error`
+    - `startDate` - ISO timestamp for date range start
+    - `endDate` - ISO timestamp for date range end
+    - `executionId` - Exact execution ID match
+    - `minDurationMs` - Minimum execution duration in milliseconds
+    - `maxDurationMs` - Maximum execution duration in milliseconds
+    - `minCost` - Minimum execution cost
+    - `maxCost` - Maximum execution cost
+    - `model` - Filter by AI model used
+
+    **Pagination:**
+    - `limit` - Results per page (default: 100)
+    - `cursor` - Cursor for next page
+    - `order` - Sort order: `desc`, `asc` (default: desc)
+
+    **Detail Level:**
+    - `details` - Response detail level: `basic`, `full` (default: basic)
+    - `includeTraceSpans` - Include trace spans (default: false)
+    - `includeFinalOutput` - Include final output (default: false)
+  </Tab>
+  <Tab value="Response">
+    ```json
+    {
+      "data": [
+        {
+          "id": "log_abc123",
+          "workflowId": "wf_xyz789",
+          "executionId": "exec_def456",
+          "level": "info",
+          "trigger": "api",
+          "startedAt": "2025-01-01T12:34:56.789Z",
+          "endedAt": "2025-01-01T12:34:57.123Z",
+          "totalDurationMs": 334,
+          "cost": {
+            "total": 0.00234
+          },
+          "files": null
+        }
+      ],
+      "nextCursor": "eyJzIjoiMjAyNS0wMS0wMVQxMjozNDo1Ni43ODlaIiwiaWQiOiJsb2dfYWJjMTIzIn0",
+      "limits": {
+        "workflowExecutionRateLimit": {
+          "sync": {
+            "limit": 60,
+            "remaining": 58,
+            "resetAt": "2025-01-01T12:35:56.789Z"
+          },
+          "async": {
+            "limit": 60,
+            "remaining": 59,
+            "resetAt": "2025-01-01T12:35:56.789Z"
+          }
+        },
+        "usage": {
+          "currentPeriodCost": 1.234,
+          "limit": 10,
+          "plan": "pro",
+          "isExceeded": false
+        }
+      }
+    }
+    ```
+  </Tab>
+</Tabs>
+
+### Get Log Details
+
+Retrieve detailed information about a specific log entry.
+
+<Tabs items={['Request', 'Response']}>
+  <Tab value="Request">
+    ```http
+    GET /api/v1/logs/{id}
+    ```
+  </Tab>
+  <Tab value="Response">
+    ```json
+    {
+      "data": {
+        "id": "log_abc123",
+        "workflowId": "wf_xyz789",
+        "executionId": "exec_def456",
+        "level": "info",
+        "trigger": "api",
+        "startedAt": "2025-01-01T12:34:56.789Z",
+        "endedAt": "2025-01-01T12:34:57.123Z",
+        "totalDurationMs": 334,
+        "workflow": {
+          "id": "wf_xyz789",
+          "name": "My Workflow",
+          "description": "Process customer data"
+        },
+        "executionData": {
+          "traceSpans": [...],
+          "finalOutput": {...}
+        },
+        "cost": {
+          "total": 0.00234,
+          "tokens": {
+            "prompt": 123,
+            "completion": 456,
+            "total": 579
+          },
+          "models": {
+            "gpt-4o": {
+              "input": 0.001,
+              "output": 0.00134,
+              "total": 0.00234,
+              "tokens": {
+                "prompt": 123,
+                "completion": 456,
+                "total": 579
+              }
+            }
+          }
+        },
+        "limits": {
+          "workflowExecutionRateLimit": {
+            "sync": {
+              "limit": 60,
+              "remaining": 58,
+              "resetAt": "2025-01-01T12:35:56.789Z"
+            },
+            "async": {
+              "limit": 60,
+              "remaining": 59,
+              "resetAt": "2025-01-01T12:35:56.789Z"
+            }
+          },
+          "usage": {
+            "currentPeriodCost": 1.234,
+            "limit": 10,
+            "plan": "pro",
+            "isExceeded": false
+          }
+        }
+      }
+    }
+    ```
+  </Tab>
+</Tabs>
+
+### Get Execution Details
+
+Retrieve execution details including the workflow state snapshot.
+
+<Tabs items={['Request', 'Response']}>
+  <Tab value="Request">
+    ```http
+    GET /api/v1/logs/executions/{executionId}
+    ```
+  </Tab>
+  <Tab value="Response">
+    ```json
+    {
+      "executionId": "exec_def456",
+      "workflowId": "wf_xyz789",
+      "workflowState": {
+        "blocks": {...},
+        "edges": [...],
+        "loops": {...},
+        "parallels": {...}
+      },
+      "executionMetadata": {
+        "trigger": "api",
+        "startedAt": "2025-01-01T12:34:56.789Z",
+        "endedAt": "2025-01-01T12:34:57.123Z",
+        "totalDurationMs": 334,
+        "cost": {...}
+      }
+    }
+    ```
+  </Tab>
+</Tabs>
+
+## Webhook Subscriptions
+
+Get real-time notifications when workflow executions complete. Webhooks are configured through the Sim UI in the workflow editor.
+
+### Configuration
+
+Webhooks can be configured for each workflow through the workflow editor UI. Click the webhook icon in the control bar to set up your webhook subscriptions.
+
+**Available Configuration Options:**
+- `url`: Your webhook endpoint URL
+- `secret`: Optional secret for HMAC signature verification
+- `includeFinalOutput`: Include the workflow's final output in the payload
+- `includeTraceSpans`: Include detailed execution trace spans
+- `includeRateLimits`: Include the workflow owner's rate limit information
+- `includeUsageData`: Include the workflow owner's usage and billing data
+- `levelFilter`: Array of log levels to receive (`info`, `error`)
+- `triggerFilter`: Array of trigger types to receive (`api`, `webhook`, `schedule`, `manual`, `chat`)
+- `active`: Enable/disable the webhook subscription
+
+### Webhook Payload
+
+When a workflow execution completes, Sim sends a POST request to your webhook URL:
+
+```json
+{
+  "id": "evt_123",
+  "type": "workflow.execution.completed",
+  "timestamp": 1735925767890,
+  "data": {
+    "workflowId": "wf_xyz789",
+    "executionId": "exec_def456",
+    "status": "success",
+    "level": "info",
+    "trigger": "api",
+    "startedAt": "2025-01-01T12:34:56.789Z",
+    "endedAt": "2025-01-01T12:34:57.123Z",
+    "totalDurationMs": 334,
+    "cost": {
+      "total": 0.00234,
+      "tokens": {
+        "prompt": 123,
+        "completion": 456,
+        "total": 579
+      },
+      "models": {
+        "gpt-4o": {
+          "input": 0.001,
+          "output": 0.00134,
+          "total": 0.00234,
+          "tokens": {
+            "prompt": 123,
+            "completion": 456,
+            "total": 579
+          }
+        }
+      }
+    },
+    "files": null,
+    "finalOutput": {...},  // Only if includeFinalOutput=true
+    "traceSpans": [...],   // Only if includeTraceSpans=true
+    "rateLimits": {...},   // Only if includeRateLimits=true
+    "usage": {...}         // Only if includeUsageData=true
+  },
+  "links": {
+    "log": "/v1/logs/log_abc123",
+    "execution": "/v1/logs/executions/exec_def456"
+  }
+}
+```
+
+### Webhook Headers
+
+Each webhook request includes these headers:
+
+- `sim-event`: Event type (always `workflow.execution.completed`)
+- `sim-timestamp`: Unix timestamp in milliseconds
+- `sim-delivery-id`: Unique delivery ID for idempotency
+- `sim-signature`: HMAC-SHA256 signature for verification (if secret configured)
+- `Idempotency-Key`: Same as delivery ID for duplicate detection
+
+### Signature Verification
+
+If you configure a webhook secret, verify the signature to ensure the webhook is from Sim:
+
+<Tabs items={['Node.js', 'Python']}>
+  <Tab value="Node.js">
+    ```javascript
+    import crypto from 'crypto';
+
+    function verifyWebhookSignature(body, signature, secret) {
+      const [timestampPart, signaturePart] = signature.split(',');
+      const timestamp = timestampPart.replace('t=', '');
+      const expectedSignature = signaturePart.replace('v1=', '');
+      
+      const signatureBase = `${timestamp}.${body}`;
+      const hmac = crypto.createHmac('sha256', secret);
+      hmac.update(signatureBase);
+      const computedSignature = hmac.digest('hex');
+      
+      return computedSignature === expectedSignature;
+    }
+
+    // In your webhook handler
+    app.post('/webhook', (req, res) => {
+      const signature = req.headers['sim-signature'];
+      const body = JSON.stringify(req.body);
+      
+      if (!verifyWebhookSignature(body, signature, process.env.WEBHOOK_SECRET)) {
+        return res.status(401).send('Invalid signature');
+      }
+      
+      // Process the webhook...
+    });
+    ```
+  </Tab>
+  <Tab value="Python">
+    ```python
+    import hmac
+    import hashlib
+    import json
+
+    def verify_webhook_signature(body: str, signature: str, secret: str) -> bool:
+        timestamp_part, signature_part = signature.split(',')
+        timestamp = timestamp_part.replace('t=', '')
+        expected_signature = signature_part.replace('v1=', '')
+        
+        signature_base = f"{timestamp}.{body}"
+        computed_signature = hmac.new(
+            secret.encode(),
+            signature_base.encode(),
+            hashlib.sha256
+        ).hexdigest()
+        
+        return hmac.compare_digest(computed_signature, expected_signature)
+
+    # In your webhook handler
+    @app.route('/webhook', methods=['POST'])
+    def webhook():
+        signature = request.headers.get('sim-signature')
+        body = json.dumps(request.json)
+        
+        if not verify_webhook_signature(body, signature, os.environ['WEBHOOK_SECRET']):
+            return 'Invalid signature', 401
+        
+        # Process the webhook...
+    ```
+  </Tab>
+</Tabs>
+
+### Retry Policy
+
+Failed webhook deliveries are retried with exponential backoff and jitter:
+
+- Maximum attempts: 5
+- Retry delays: 5 seconds, 15 seconds, 1 minute, 3 minutes, 10 minutes
+- Jitter: Up to 10% additional delay to prevent thundering herd
+- Only HTTP 5xx and 429 responses trigger retries
+- Deliveries timeout after 30 seconds
+
+<Callout type="info">
+  Webhook deliveries are processed asynchronously and don't affect workflow execution performance.
+</Callout>
+
+## Best Practices
+
+1. **Polling Strategy**: When polling for logs, use cursor-based pagination with `order=asc` and `startDate` to fetch new logs efficiently.
+
+2. **Webhook Security**: Always configure a webhook secret and verify signatures to ensure requests are from Sim.
+
+3. **Idempotency**: Use the `Idempotency-Key` header to detect and handle duplicate webhook deliveries.
+
+4. **Privacy**: By default, `finalOutput` and `traceSpans` are excluded from responses. Only enable these if you need the data and understand the privacy implications.
+
+5. **Rate Limiting**: Implement exponential backoff when you receive 429 responses. Check the `Retry-After` header for the recommended wait time.
+
+## Rate Limiting
+
+The API implements rate limiting to ensure fair usage:
+
+- **Free plan**: 10 requests per minute
+- **Pro plan**: 30 requests per minute
+- **Team plan**: 60 requests per minute
+- **Enterprise plan**: Custom limits
+
+Rate limit information is included in response headers:
+- `X-RateLimit-Limit`: Maximum requests per window
+- `X-RateLimit-Remaining`: Requests remaining in current window
+- `X-RateLimit-Reset`: ISO timestamp when the window resets
+
+## Example: Polling for New Logs
+
+```javascript
+let cursor = null;
+const workspaceId = 'YOUR_WORKSPACE_ID';
+const startDate = new Date().toISOString();
+
+async function pollLogs() {
+  const params = new URLSearchParams({
+    workspaceId,
+    startDate,
+    order: 'asc',
+    limit: '100'
+  });
+  
+  if (cursor) {
+    params.append('cursor', cursor);
+  }
+  
+  const response = await fetch(
+    `https://sim.ai/api/v1/logs?${params}`,
+    {
+      headers: {
+        'x-api-key': 'YOUR_API_KEY'
+      }
+    }
+  );
+  
+  if (response.ok) {
+    const data = await response.json();
+    
+    // Process new logs
+    for (const log of data.data) {
+      console.log(`New execution: ${log.executionId}`);
+    }
+    
+    // Update cursor for next poll
+    if (data.nextCursor) {
+      cursor = data.nextCursor;
+    }
+  }
+}
+
+// Poll every 30 seconds
+setInterval(pollLogs, 30000);
+```
+
+## Example: Processing Webhooks
+
+```javascript
+import express from 'express';
+import crypto from 'crypto';
+
+const app = express();
+app.use(express.json());
+
+app.post('/sim-webhook', (req, res) => {
+  // Verify signature
+  const signature = req.headers['sim-signature'];
+  const body = JSON.stringify(req.body);
+  
+  if (!verifyWebhookSignature(body, signature, process.env.WEBHOOK_SECRET)) {
+    return res.status(401).send('Invalid signature');
+  }
+  
+  // Check timestamp to prevent replay attacks
+  const timestamp = parseInt(req.headers['sim-timestamp']);
+  const fiveMinutesAgo = Date.now() - (5 * 60 * 1000);
+  
+  if (timestamp < fiveMinutesAgo) {
+    return res.status(401).send('Timestamp too old');
+  }
+  
+  // Process the webhook
+  const event = req.body;
+  
+  switch (event.type) {
+    case 'workflow.execution.completed':
+      const { workflowId, executionId, status, cost } = event.data;
+      
+      if (status === 'error') {
+        console.error(`Workflow ${workflowId} failed: ${executionId}`);
+        // Handle error...
+      } else {
+        console.log(`Workflow ${workflowId} completed: ${executionId}`);
+        console.log(`Cost: $${cost.total}`);
+        // Process successful execution...
+      }
+      break;
+  }
+  
+  // Return 200 to acknowledge receipt
+  res.status(200).send('OK');
+});
+
+app.listen(3000, () => {
+  console.log('Webhook server listening on port 3000');
+});
+```

apps/docs/content/docs/execution/meta.json

@@ -1,4 +1,4 @@
 {
   "title": "Execution",
-  "pages": ["basics", "advanced"]
+  "pages": ["basics", "advanced", "api"]
 }

apps/docs/content/docs/tools/jira.mdx

@@ -58,7 +58,7 @@ Retrieve detailed information about a specific Jira issue
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) |
-| `projectId` | string | No | Jira project ID to retrieve issues from. If not provided, all issues will be retrieved. |
+| `projectId` | string | No | Jira project ID \(optional; not required to retrieve a single issue\). |
 | `issueKey` | string | Yes | Jira issue key to retrieve \(e.g., PROJ-123\) |
 | `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. |
 

apps/docs/content/docs/tools/meta.json

@@ -33,6 +33,7 @@
     "microsoft_planner",
     "microsoft_teams",
     "mistral_parse",
+    "mongodb",
     "mysql",
     "notion",
     "onedrive",

apps/docs/content/docs/tools/microsoft_excel.mdx

@@ -109,7 +109,7 @@ Read data from a Microsoft Excel spreadsheet
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `spreadsheetId` | string | Yes | The ID of the spreadsheet to read from |
-| `range` | string | No | The range of cells to read from |
+| `range` | string | No | The range of cells to read from. Accepts "SheetName!A1:B2" for explicit ranges or just "SheetName" to read the used range of that sheet. If omitted, reads the used range of the first sheet. |
 
 #### Output
 

apps/docs/content/docs/tools/mongodb.mdx

@@ -0,0 +1,264 @@
+---
+title: MongoDB
+description: Connect to MongoDB database
+---
+
+import { BlockInfoCard } from "@/components/ui/block-info-card"
+
+<BlockInfoCard 
+  type="mongodb"
+  color="#E0E0E0"
+  icon={true}
+  iconSvg={`<svg className="block-icon"  xmlns='http://www.w3.org/2000/svg' viewBox='0 0 128 128'>
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='currentColor'
+        d='M88.038 42.812c1.605 4.643 2.761 9.383 3.141 14.296.472 6.095.256 12.147-1.029 18.142-.035.165-.109.32-.164.48-.403.001-.814-.049-1.208.012-3.329.523-6.655 1.065-9.981 1.604-3.438.557-6.881 1.092-10.313 1.687-1.216.21-2.721-.041-3.212 1.641-.014.046-.154.054-.235.08l.166-10.051-.169-24.252 1.602-.275c2.62-.429 5.24-.864 7.862-1.281 3.129-.497 6.261-.98 9.392-1.465 1.381-.215 2.764-.412 4.148-.618z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#45A538'
+        d='M61.729 110.054c-1.69-1.453-3.439-2.842-5.059-4.37-8.717-8.222-15.093-17.899-18.233-29.566-.865-3.211-1.442-6.474-1.627-9.792-.13-2.322-.318-4.665-.154-6.975.437-6.144 1.325-12.229 3.127-18.147l.099-.138c.175.233.427.439.516.702 1.759 5.18 3.505 10.364 5.242 15.551 5.458 16.3 10.909 32.604 16.376 48.9.107.318.384.579.583.866l-.87 2.969z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#46A037'
+        d='M88.038 42.812c-1.384.206-2.768.403-4.149.616-3.131.485-6.263.968-9.392 1.465-2.622.417-5.242.852-7.862 1.281l-1.602.275-.012-1.045c-.053-.859-.144-1.717-.154-2.576-.069-5.478-.112-10.956-.18-16.434-.042-3.429-.105-6.857-.175-10.285-.043-2.13-.089-4.261-.185-6.388-.052-1.143-.236-2.28-.311-3.423-.042-.657.016-1.319.029-1.979.817 1.583 1.616 3.178 2.456 4.749 1.327 2.484 3.441 4.314 5.344 6.311 7.523 7.892 12.864 17.068 16.193 27.433z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#409433'
+        d='M65.036 80.753c.081-.026.222-.034.235-.08.491-1.682 1.996-1.431 3.212-1.641 3.432-.594 6.875-1.13 10.313-1.687 3.326-.539 6.652-1.081 9.981-1.604.394-.062.805-.011 1.208-.012-.622 2.22-1.112 4.488-1.901 6.647-.896 2.449-1.98 4.839-3.131 7.182a49.142 49.142 0 01-6.353 9.763c-1.919 2.308-4.058 4.441-6.202 6.548-1.185 1.165-2.582 2.114-3.882 3.161l-.337-.23-1.214-1.038-1.256-2.753a41.402 41.402 0 01-1.394-9.838l.023-.561.171-2.426c.057-.828.133-1.655.168-2.485.129-2.982.241-5.964.359-8.946z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#4FAA41'
+        d='M65.036 80.753c-.118 2.982-.23 5.964-.357 8.947-.035.83-.111 1.657-.168 2.485l-.765.289c-1.699-5.002-3.399-9.951-5.062-14.913-2.75-8.209-5.467-16.431-8.213-24.642a4498.887 4498.887 0 00-6.7-19.867c-.105-.31-.407-.552-.617-.826l4.896-9.002c.168.292.39.565.496.879a6167.476 6167.476 0 016.768 20.118c2.916 8.73 5.814 17.467 8.728 26.198.116.349.308.671.491 1.062l.67-.78-.167 10.052z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#4AA73C'
+        d='M43.155 32.227c.21.274.511.516.617.826a4498.887 4498.887 0 016.7 19.867c2.746 8.211 5.463 16.433 8.213 24.642 1.662 4.961 3.362 9.911 5.062 14.913l.765-.289-.171 2.426-.155.559c-.266 2.656-.49 5.318-.814 7.968-.163 1.328-.509 2.632-.772 3.947-.198-.287-.476-.548-.583-.866-5.467-16.297-10.918-32.6-16.376-48.9a3888.972 3888.972 0 00-5.242-15.551c-.089-.263-.34-.469-.516-.702l3.272-8.84z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#57AE47'
+        d='M65.202 70.702l-.67.78c-.183-.391-.375-.714-.491-1.062-2.913-8.731-5.812-17.468-8.728-26.198a6167.476 6167.476 0 00-6.768-20.118c-.105-.314-.327-.588-.496-.879l6.055-7.965c.191.255.463.482.562.769 1.681 4.921 3.347 9.848 5.003 14.778 1.547 4.604 3.071 9.215 4.636 13.813.105.308.47.526.714.786l.012 1.045c.058 8.082.115 16.167.171 24.251z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#60B24F'
+        d='M65.021 45.404c-.244-.26-.609-.478-.714-.786-1.565-4.598-3.089-9.209-4.636-13.813-1.656-4.93-3.322-9.856-5.003-14.778-.099-.287-.371-.514-.562-.769 1.969-1.928 3.877-3.925 5.925-5.764 1.821-1.634 3.285-3.386 3.352-5.968.003-.107.059-.214.145-.514l.519 1.306c-.013.661-.072 1.322-.029 1.979.075 1.143.259 2.28.311 3.423.096 2.127.142 4.258.185 6.388.069 3.428.132 6.856.175 10.285.067 5.478.111 10.956.18 16.434.008.861.098 1.718.152 2.577z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#A9AA88'
+        d='M62.598 107.085c.263-1.315.609-2.62.772-3.947.325-2.649.548-5.312.814-7.968l.066-.01.066.011a41.402 41.402 0 001.394 9.838c-.176.232-.425.439-.518.701-.727 2.05-1.412 4.116-2.143 6.166-.1.28-.378.498-.574.744l-.747-2.566.87-2.969z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#B6B598'
+        d='M62.476 112.621c.196-.246.475-.464.574-.744.731-2.05 1.417-4.115 2.143-6.166.093-.262.341-.469.518-.701l1.255 2.754c-.248.352-.59.669-.728 1.061l-2.404 7.059c-.099.283-.437.483-.663.722l-.695-3.985z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#C2C1A7'
+        d='M63.171 116.605c.227-.238.564-.439.663-.722l2.404-7.059c.137-.391.48-.709.728-1.061l1.215 1.037c-.587.58-.913 1.25-.717 2.097l-.369 1.208c-.168.207-.411.387-.494.624-.839 2.403-1.64 4.819-2.485 7.222-.107.305-.404.544-.614.812-.109-1.387-.22-2.771-.331-4.158z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#CECDB7'
+        d='M63.503 120.763c.209-.269.506-.508.614-.812.845-2.402 1.646-4.818 2.485-7.222.083-.236.325-.417.494-.624l-.509 5.545c-.136.157-.333.294-.398.477-.575 1.614-1.117 3.24-1.694 4.854-.119.333-.347.627-.525.938-.158-.207-.441-.407-.454-.623-.051-.841-.016-1.688-.013-2.533z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#DBDAC7'
+        d='M63.969 123.919c.178-.312.406-.606.525-.938.578-1.613 1.119-3.239 1.694-4.854.065-.183.263-.319.398-.477l.012 3.64-1.218 3.124-1.411-.495z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#EBE9DC'
+        d='M65.38 124.415l1.218-3.124.251 3.696-1.469-.572z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#CECDB7'
+        d='M67.464 110.898c-.196-.847.129-1.518.717-2.097l.337.23-1.054 1.867z'
+      />
+      <path
+        fillRule='evenodd'
+        clipRule='evenodd'
+        fill='#4FAA41'
+        d='M64.316 95.172l-.066-.011-.066.01.155-.559-.023.56z'
+      />
+    </svg>`}
+/>
+
+## Usage Instructions
+
+Connect to any MongoDB database to execute queries, manage data, and perform database operations. Supports find, insert, update, delete, and aggregation operations with secure connection handling.
+
+
+
+## Tools
+
+### `mongodb_query`
+
+Execute find operation on MongoDB collection
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `host` | string | Yes | MongoDB server hostname or IP address |
+| `port` | number | Yes | MongoDB server port \(default: 27017\) |
+| `database` | string | Yes | Database name to connect to |
+| `username` | string | No | MongoDB username |
+| `password` | string | No | MongoDB password |
+| `authSource` | string | No | Authentication database |
+| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
+| `collection` | string | Yes | Collection name to query |
+| `query` | string | No | MongoDB query filter as JSON string |
+| `limit` | number | No | Maximum number of documents to return |
+| `sort` | string | No | Sort criteria as JSON string |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Operation status message |
+| `documents` | array | Array of documents returned from the query |
+| `documentCount` | number | Number of documents returned |
+
+### `mongodb_insert`
+
+Insert documents into MongoDB collection
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `host` | string | Yes | MongoDB server hostname or IP address |
+| `port` | number | Yes | MongoDB server port \(default: 27017\) |
+| `database` | string | Yes | Database name to connect to |
+| `username` | string | No | MongoDB username |
+| `password` | string | No | MongoDB password |
+| `authSource` | string | No | Authentication database |
+| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
+| `collection` | string | Yes | Collection name to insert into |
+| `documents` | array | Yes | Array of documents to insert |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Operation status message |
+| `documentCount` | number | Number of documents inserted |
+| `insertedId` | string | ID of inserted document \(single insert\) |
+| `insertedIds` | array | Array of inserted document IDs \(multiple insert\) |
+
+### `mongodb_update`
+
+Update documents in MongoDB collection
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `host` | string | Yes | MongoDB server hostname or IP address |
+| `port` | number | Yes | MongoDB server port \(default: 27017\) |
+| `database` | string | Yes | Database name to connect to |
+| `username` | string | No | MongoDB username |
+| `password` | string | No | MongoDB password |
+| `authSource` | string | No | Authentication database |
+| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
+| `collection` | string | Yes | Collection name to update |
+| `filter` | string | Yes | Filter criteria as JSON string |
+| `update` | string | Yes | Update operations as JSON string |
+| `upsert` | boolean | No | Create document if not found |
+| `multi` | boolean | No | Update multiple documents |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Operation status message |
+| `matchedCount` | number | Number of documents matched by filter |
+| `modifiedCount` | number | Number of documents modified |
+| `documentCount` | number | Total number of documents affected |
+| `insertedId` | string | ID of inserted document \(if upsert\) |
+
+### `mongodb_delete`
+
+Delete documents from MongoDB collection
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `host` | string | Yes | MongoDB server hostname or IP address |
+| `port` | number | Yes | MongoDB server port \(default: 27017\) |
+| `database` | string | Yes | Database name to connect to |
+| `username` | string | No | MongoDB username |
+| `password` | string | No | MongoDB password |
+| `authSource` | string | No | Authentication database |
+| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
+| `collection` | string | Yes | Collection name to delete from |
+| `filter` | string | Yes | Filter criteria as JSON string |
+| `multi` | boolean | No | Delete multiple documents |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Operation status message |
+| `deletedCount` | number | Number of documents deleted |
+| `documentCount` | number | Total number of documents affected |
+
+### `mongodb_execute`
+
+Execute MongoDB aggregation pipeline
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `host` | string | Yes | MongoDB server hostname or IP address |
+| `port` | number | Yes | MongoDB server port \(default: 27017\) |
+| `database` | string | Yes | Database name to connect to |
+| `username` | string | No | MongoDB username |
+| `password` | string | No | MongoDB password |
+| `authSource` | string | No | Authentication database |
+| `ssl` | string | No | SSL connection mode \(disabled, required, preferred\) |
+| `collection` | string | Yes | Collection name to execute pipeline on |
+| `pipeline` | string | Yes | Aggregation pipeline as JSON string |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Operation status message |
+| `documents` | array | Array of documents returned from aggregation |
+| `documentCount` | number | Number of documents returned |
+
+
+
+## Notes
+
+- Category: `tools`
+- Type: `mongodb`

apps/docs/content/docs/tools/onedrive.mdx

@@ -68,7 +68,7 @@ Upload a file to OneDrive
 | `fileName` | string | Yes | The name of the file to upload |
 | `content` | string | Yes | The content of the file to upload |
 | `folderSelector` | string | No | Select the folder to upload the file to |
-| `folderId` | string | No | The ID of the folder to upload the file to \(internal use\) |
+| `manualFolderId` | string | No | Manually entered folder ID \(advanced mode\) |
 
 #### Output
 
@@ -87,7 +87,7 @@ Create a new folder in OneDrive
 | --------- | ---- | -------- | ----------- |
 | `folderName` | string | Yes | Name of the folder to create |
 | `folderSelector` | string | No | Select the parent folder to create the folder in |
-| `folderId` | string | No | ID of the parent folder \(internal use\) |
+| `manualFolderId` | string | No | Manually entered parent folder ID \(advanced mode\) |
 
 #### Output
 
@@ -105,7 +105,7 @@ List files and folders in OneDrive
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `folderSelector` | string | No | Select the folder to list files from |
-| `folderId` | string | No | The ID of the folder to list files from \(internal use\) |
+| `manualFolderId` | string | No | The manually entered folder ID \(advanced mode\) |
 | `query` | string | No | A query to filter the files |
 | `pageSize` | number | No | The number of files to return |
 

apps/docs/content/docs/tools/outlook.mdx

@@ -211,10 +211,27 @@ Read emails from Outlook
 
 | Parameter | Type | Description |
 | --------- | ---- | ----------- |
-| `success` | boolean | Email read operation success status |
-| `messageCount` | number | Number of emails retrieved |
-| `messages` | array | Array of email message objects |
 | `message` | string | Success or status message |
+| `results` | array | Array of email message objects |
+
+### `outlook_forward`
+
+Forward an existing Outlook message to specified recipients
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `messageId` | string | Yes | The ID of the message to forward |
+| `to` | string | Yes | Recipient email address\(es\), comma-separated |
+| `comment` | string | No | Optional comment to include with the forwarded message |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `message` | string | Success or error message |
+| `results` | object | Delivery result details |
 
 
 

apps/sim/.env.example

@@ -1,6 +1,9 @@
 # Database (Required)
 DATABASE_URL="postgresql://postgres:password@localhost:5432/postgres"
 
+# PostgreSQL Port (Optional) - defaults to 5432 if not specified
+# POSTGRES_PORT=5432
+
 # Authentication (Required)
 BETTER_AUTH_SECRET=your_secret_key  # Use `openssl rand -hex 32` to generate, or visit https://www.better-auth.com/docs/installation
 BETTER_AUTH_URL=http://localhost:3000

apps/sim/app/(auth)/login/login-form.tsx

@@ -49,15 +49,12 @@ const PASSWORD_VALIDATIONS = {
   },
 }
 
-// Validate callback URL to prevent open redirect vulnerabilities
 const validateCallbackUrl = (url: string): boolean => {
   try {
-    // If it's a relative URL, it's safe
     if (url.startsWith('/')) {
       return true
     }
 
-    // If absolute URL, check if it belongs to the same origin
     const currentOrigin = typeof window !== 'undefined' ? window.location.origin : ''
     if (url.startsWith(currentOrigin)) {
       return true
@@ -70,7 +67,6 @@ const validateCallbackUrl = (url: string): boolean => {
   }
 }
 
-// Validate password and return array of error messages
 const validatePassword = (passwordValue: string): string[] => {
   const errors: string[] = []
 
@@ -308,6 +304,15 @@ export default function LoginPage({
       return
     }
 
+    const emailValidation = quickValidateEmail(forgotPasswordEmail.trim().toLowerCase())
+    if (!emailValidation.isValid) {
+      setResetStatus({
+        type: 'error',
+        message: 'Please enter a valid email address',
+      })
+      return
+    }
+
     try {
       setIsSubmittingReset(true)
       setResetStatus({ type: null, message: '' })
@@ -325,7 +330,23 @@ export default function LoginPage({
 
       if (!response.ok) {
         const errorData = await response.json()
-        throw new Error(errorData.message || 'Failed to request password reset')
+        let errorMessage = errorData.message || 'Failed to request password reset'
+
+        if (
+          errorMessage.includes('Invalid body parameters') ||
+          errorMessage.includes('invalid email')
+        ) {
+          errorMessage = 'Please enter a valid email address'
+        } else if (errorMessage.includes('Email is required')) {
+          errorMessage = 'Please enter your email address'
+        } else if (
+          errorMessage.includes('user not found') ||
+          errorMessage.includes('User not found')
+        ) {
+          errorMessage = 'No account found with this email address'
+        }
+
+        throw new Error(errorMessage)
       }
 
       setResetStatus({
@@ -475,6 +496,23 @@ export default function LoginPage({
             Sign up
           </Link>
         </div>
+
+        <div className='text-center text-neutral-500/80 text-xs leading-relaxed'>
+          By signing in, you agree to our{' '}
+          <Link
+            href='/terms'
+            className='text-neutral-400 underline-offset-4 transition hover:text-neutral-300 hover:underline'
+          >
+            Terms of Service
+          </Link>{' '}
+          and{' '}
+          <Link
+            href='/privacy'
+            className='text-neutral-400 underline-offset-4 transition hover:text-neutral-300 hover:underline'
+          >
+            Privacy Policy
+          </Link>
+        </div>
       </div>
 
       <Dialog open={forgotPasswordOpen} onOpenChange={setForgotPasswordOpen}>
@@ -484,7 +522,8 @@ export default function LoginPage({
               Reset Password
             </DialogTitle>
             <DialogDescription className='text-neutral-300 text-sm'>
-              Enter your email address and we'll send you a link to reset your password.
+              Enter your email address and we'll send you a link to reset your password if your
+              account exists.
             </DialogDescription>
           </DialogHeader>
           <div className='space-y-4'>
@@ -499,16 +538,20 @@ export default function LoginPage({
                 placeholder='Enter your email'
                 required
                 type='email'
-                className='border-neutral-700/80 bg-neutral-900 text-white placeholder:text-white/60 focus:border-[var(--brand-primary-hover-hex)]/70 focus:ring-[var(--brand-primary-hover-hex)]/20'
+                className={cn(
+                  'border-neutral-700/80 bg-neutral-900 text-white placeholder:text-white/60 focus:border-[var(--brand-primary-hover-hex)]/70 focus:ring-[var(--brand-primary-hover-hex)]/20',
+                  resetStatus.type === 'error' && 'border-red-500 focus-visible:ring-red-500'
+                )}
               />
+              {resetStatus.type === 'error' && (
+                <div className='mt-1 space-y-1 text-red-400 text-xs'>
+                  <p>{resetStatus.message}</p>
+                </div>
+              )}
             </div>
-            {resetStatus.type && (
-              <div
-                className={`text-sm ${
-                  resetStatus.type === 'success' ? 'text-[#4CAF50]' : 'text-red-500'
-                }`}
-              >
-                {resetStatus.message}
+            {resetStatus.type === 'success' && (
+              <div className='mt-1 space-y-1 text-[#4CAF50] text-xs'>
+                <p>{resetStatus.message}</p>
               </div>
             )}
             <Button

apps/sim/app/(auth)/reset-password/page.tsx

@@ -101,7 +101,7 @@ function ResetPasswordContent() {
           </CardContent>
           <CardFooter>
             <p className='w-full text-center text-gray-500 text-sm'>
-              <Link href='/login' className='text-primary hover:underline'>
+              <Link href='/login' className='text-muted-foreground hover:underline'>
                 Back to login
               </Link>
             </p>

apps/sim/app/(auth)/signup/signup-form.test.tsx

@@ -5,7 +5,7 @@
 import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'
 import { useRouter, useSearchParams } from 'next/navigation'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
-import { client } from '@/lib/auth-client'
+import { client, useSession } from '@/lib/auth-client'
 import SignupPage from '@/app/(auth)/signup/signup-form'
 
 vi.mock('next/navigation', () => ({
@@ -22,6 +22,7 @@ vi.mock('@/lib/auth-client', () => ({
       sendVerificationOtp: vi.fn(),
     },
   },
+  useSession: vi.fn(),
 }))
 
 vi.mock('@/app/(auth)/components/social-login-buttons', () => ({
@@ -43,6 +44,9 @@ describe('SignupPage', () => {
     vi.clearAllMocks()
     ;(useRouter as any).mockReturnValue(mockRouter)
     ;(useSearchParams as any).mockReturnValue(mockSearchParams)
+    ;(useSession as any).mockReturnValue({
+      refetch: vi.fn().mockResolvedValue({}),
+    })
     mockSearchParams.get.mockReturnValue(null)
   })
 
@@ -162,8 +166,9 @@ describe('SignupPage', () => {
       })
     })
 
-    it('should prevent submission with invalid name validation', async () => {
+    it('should automatically trim spaces from name input', async () => {
       const mockSignUp = vi.mocked(client.signUp.email)
+      mockSignUp.mockResolvedValue({ data: null, error: null })
 
       render(<SignupPage {...defaultProps} />)
 
@@ -172,22 +177,20 @@ describe('SignupPage', () => {
       const passwordInput = screen.getByPlaceholderText(/enter your password/i)
       const submitButton = screen.getByRole('button', { name: /create account/i })
 
-      // Use name with leading/trailing spaces which should fail validation
       fireEvent.change(nameInput, { target: { value: '  John Doe  ' } })
       fireEvent.change(emailInput, { target: { value: 'user@company.com' } })
       fireEvent.change(passwordInput, { target: { value: 'Password123!' } })
       fireEvent.click(submitButton)
 
-      // Should not call signUp because validation failed
-      expect(mockSignUp).not.toHaveBeenCalled()
-
-      // Should show validation error
       await waitFor(() => {
-        expect(
-          screen.getByText(
-            /Name cannot contain consecutive spaces|Name cannot start or end with spaces/
-          )
-        ).toBeInTheDocument()
+        expect(mockSignUp).toHaveBeenCalledWith(
+          expect.objectContaining({
+            name: 'John Doe',
+            email: 'user@company.com',
+            password: 'Password123!',
+          }),
+          expect.any(Object)
+        )
       })
     })
 

apps/sim/app/(auth)/signup/signup-form.tsx

@@ -7,7 +7,7 @@ import { useRouter, useSearchParams } from 'next/navigation'
 import { Button } from '@/components/ui/button'
 import { Input } from '@/components/ui/input'
 import { Label } from '@/components/ui/label'
-import { client } from '@/lib/auth-client'
+import { client, useSession } from '@/lib/auth-client'
 import { quickValidateEmail } from '@/lib/email/validation'
 import { createLogger } from '@/lib/logs/console/logger'
 import { cn } from '@/lib/utils'
@@ -49,10 +49,6 @@ const NAME_VALIDATIONS = {
     regex: /^(?!.*\s\s).*$/,
     message: 'Name cannot contain consecutive spaces.',
   },
-  noLeadingTrailingSpaces: {
-    test: (value: string) => value === value.trim(),
-    message: 'Name cannot start or end with spaces.',
-  },
 }
 
 const validateEmailField = (emailValue: string): string[] => {
@@ -82,6 +78,7 @@ function SignupFormContent({
 }) {
   const router = useRouter()
   const searchParams = useSearchParams()
+  const { refetch: refetchSession } = useSession()
   const [isLoading, setIsLoading] = useState(false)
   const [, setMounted] = useState(false)
   const [showPassword, setShowPassword] = useState(false)
@@ -174,10 +171,6 @@ function SignupFormContent({
       errors.push(NAME_VALIDATIONS.noConsecutiveSpaces.message)
     }
 
-    if (!NAME_VALIDATIONS.noLeadingTrailingSpaces.test(nameValue)) {
-      errors.push(NAME_VALIDATIONS.noLeadingTrailingSpaces.message)
-    }
-
     return errors
   }
 
@@ -192,11 +185,10 @@ function SignupFormContent({
   }
 
   const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
-    const newName = e.target.value
-    setName(newName)
+    const rawValue = e.target.value
+    setName(rawValue)
 
-    // Silently validate but don't show errors until submit
-    const errors = validateName(newName)
+    const errors = validateName(rawValue)
     setNameErrors(errors)
     setShowNameValidationError(false)
   }
@@ -223,23 +215,21 @@ function SignupFormContent({
     const formData = new FormData(e.currentTarget)
     const emailValue = formData.get('email') as string
     const passwordValue = formData.get('password') as string
-    const name = formData.get('name') as string
+    const nameValue = formData.get('name') as string
 
-    // Validate name on submit
-    const nameValidationErrors = validateName(name)
+    const trimmedName = nameValue.trim()
+
+    const nameValidationErrors = validateName(trimmedName)
     setNameErrors(nameValidationErrors)
     setShowNameValidationError(nameValidationErrors.length > 0)
 
-    // Validate email on submit
     const emailValidationErrors = validateEmailField(emailValue)
     setEmailErrors(emailValidationErrors)
     setShowEmailValidationError(emailValidationErrors.length > 0)
 
-    // Validate password on submit
     const errors = validatePassword(passwordValue)
     setPasswordErrors(errors)
 
-    // Only show validation errors if there are any
     setShowValidationError(errors.length > 0)
 
     try {
@@ -248,7 +238,6 @@ function SignupFormContent({
         emailValidationErrors.length > 0 ||
         errors.length > 0
       ) {
-        // Prioritize name errors first, then email errors, then password errors
         if (nameValidationErrors.length > 0) {
           setNameErrors([nameValidationErrors[0]])
           setShowNameValidationError(true)
@@ -265,8 +254,6 @@ function SignupFormContent({
         return
       }
 
-      // Check if name will be truncated and warn user
-      const trimmedName = name.trim()
       if (trimmedName.length > 100) {
         setNameErrors(['Name will be truncated to 100 characters. Please shorten your name.'])
         setShowNameValidationError(true)
@@ -330,6 +317,14 @@ function SignupFormContent({
         return
       }
 
+      // Refresh session to get the new user data immediately after signup
+      try {
+        await refetchSession()
+        logger.info('Session refreshed after successful signup')
+      } catch (sessionError) {
+        logger.error('Failed to refresh session after signup:', sessionError)
+      }
+
       // For new signups, always require verification
       if (typeof window !== 'undefined') {
         sessionStorage.setItem('verificationEmail', emailValue)
@@ -507,6 +502,23 @@ function SignupFormContent({
             Sign in
           </Link>
         </div>
+
+        <div className='text-center text-neutral-500/80 text-xs leading-relaxed'>
+          By creating an account, you agree to our{' '}
+          <Link
+            href='/terms'
+            className='text-neutral-400 underline-offset-4 transition hover:text-neutral-300 hover:underline'
+          >
+            Terms of Service
+          </Link>{' '}
+          and{' '}
+          <Link
+            href='/privacy'
+            className='text-neutral-400 underline-offset-4 transition hover:text-neutral-300 hover:underline'
+          >
+            Privacy Policy
+          </Link>
+        </div>
       </div>
     </div>
   )

apps/sim/app/(auth)/verify/use-verification.ts

@@ -2,7 +2,7 @@
 
 import { useEffect, useState } from 'react'
 import { useRouter, useSearchParams } from 'next/navigation'
-import { client } from '@/lib/auth-client'
+import { client, useSession } from '@/lib/auth-client'
 import { env, isTruthy } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
 
@@ -34,6 +34,7 @@ export function useVerification({
 }: UseVerificationParams): UseVerificationReturn {
   const router = useRouter()
   const searchParams = useSearchParams()
+  const { refetch: refetchSession } = useSession()
   const [otp, setOtp] = useState('')
   const [email, setEmail] = useState('')
   const [isLoading, setIsLoading] = useState(false)
@@ -136,16 +137,15 @@ export function useVerification({
           }
         }
 
-        // Redirect to proper page after a short delay
         setTimeout(() => {
           if (isInviteFlow && redirectUrl) {
             // For invitation flow, redirect to the invitation page
-            router.push(redirectUrl)
+            window.location.href = redirectUrl
           } else {
             // Default redirect to dashboard
-            router.push('/workspace')
+            window.location.href = '/workspace'
           }
-        }, 2000)
+        }, 1000)
       } else {
         logger.info('Setting invalid OTP state - API error response')
         const message = 'Invalid verification code. Please check and try again.'
@@ -215,25 +215,33 @@ export function useVerification({
     setOtp(value)
   }
 
+  // Auto-submit when OTP is complete
+  useEffect(() => {
+    if (otp.length === 6 && email && !isLoading && !isVerified) {
+      const timeoutId = setTimeout(() => {
+        verifyCode()
+      }, 300) // Small delay to ensure UI is ready
+
+      return () => clearTimeout(timeoutId)
+    }
+  }, [otp, email, isLoading, isVerified])
+
   useEffect(() => {
     if (typeof window !== 'undefined') {
       if (!isProduction || !hasResendKey) {
         const storedEmail = sessionStorage.getItem('verificationEmail')
-        logger.info('Auto-verifying user', { email: storedEmail })
       }
 
       const isDevOrDocker = !isProduction || isTruthy(env.DOCKER_BUILD)
 
-      // Auto-verify and redirect in development/docker environments
       if (isDevOrDocker || !hasResendKey) {
         setIsVerified(true)
 
-        // Clear verification requirement cookie (same as manual verification)
         document.cookie =
           'requiresEmailVerification=; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT'
 
         const timeoutId = setTimeout(() => {
-          router.push('/workspace')
+          window.location.href = '/workspace'
         }, 1000)
 
         return () => clearTimeout(timeoutId)

apps/sim/app/api/__test-utils__/setup.ts

@@ -3,7 +3,6 @@
  */
 import { afterEach, beforeEach, vi } from 'vitest'
 
-// Mock Next.js implementations
 vi.mock('next/headers', () => ({
   cookies: () => ({
     get: vi.fn().mockReturnValue({ value: 'test-session-token' }),
@@ -13,7 +12,6 @@ vi.mock('next/headers', () => ({
   }),
 }))
 
-// Mock auth utilities
 vi.mock('@/lib/auth/session', () => ({
   getSession: vi.fn().mockResolvedValue({
     user: {
@@ -24,13 +22,10 @@ vi.mock('@/lib/auth/session', () => ({
   }),
 }))
 
-// Configure Vitest environment
 beforeEach(() => {
-  // Clear all mocks before each test
   vi.clearAllMocks()
 })
 
 afterEach(() => {
-  // Ensure all mocks are restored after each test
   vi.restoreAllMocks()
 })

apps/sim/app/api/__test-utils__/utils.ts

@@ -944,12 +944,10 @@ export interface TestSetupOptions {
 export function setupComprehensiveTestMocks(options: TestSetupOptions = {}) {
   const { auth = { authenticated: true }, database = {}, storage, authApi, features = {} } = options
 
-  // Setup basic infrastructure mocks
   setupCommonApiMocks()
   mockUuid()
   mockCryptoUuid()
 
-  // Setup authentication
   const authMocks = mockAuth(auth.user)
   if (auth.authenticated) {
     authMocks.setAuthenticated(auth.user)
@@ -957,22 +955,18 @@ export function setupComprehensiveTestMocks(options: TestSetupOptions = {}) {
     authMocks.setUnauthenticated()
   }
 
-  // Setup database
   const dbMocks = createMockDatabase(database)
 
-  // Setup storage if needed
   let storageMocks
   if (storage) {
     storageMocks = createStorageProviderMocks(storage)
   }
 
-  // Setup auth API if needed
   let authApiMocks
   if (authApi) {
     authApiMocks = createAuthApiMocks(authApi)
   }
 
-  // Setup feature-specific mocks
   const featureMocks: any = {}
   if (features.workflowUtils) {
     featureMocks.workflowUtils = mockWorkflowUtils()
@@ -1008,12 +1002,10 @@ export function createMockDatabase(options: MockDatabaseOptions = {}) {
 
   let selectCallCount = 0
 
-  // Helper to create error
   const createDbError = (operation: string, message?: string) => {
     return new Error(message || `Database ${operation} error`)
   }
 
-  // Create chainable select mock
   const createSelectChain = () => ({
     from: vi.fn().mockReturnThis(),
     leftJoin: vi.fn().mockReturnThis(),
@@ -1038,7 +1030,6 @@ export function createMockDatabase(options: MockDatabaseOptions = {}) {
     }),
   })
 
-  // Create insert chain
   const createInsertChain = () => ({
     values: vi.fn().mockImplementation(() => ({
       returning: vi.fn().mockImplementation(() => {
@@ -1056,7 +1047,6 @@ export function createMockDatabase(options: MockDatabaseOptions = {}) {
     })),
   })
 
-  // Create update chain
   const createUpdateChain = () => ({
     set: vi.fn().mockImplementation(() => ({
       where: vi.fn().mockImplementation(() => {
@@ -1068,7 +1058,6 @@ export function createMockDatabase(options: MockDatabaseOptions = {}) {
     })),
   })
 
-  // Create delete chain
   const createDeleteChain = () => ({
     where: vi.fn().mockImplementation(() => {
       if (deleteOptions.throwError) {
@@ -1078,7 +1067,6 @@ export function createMockDatabase(options: MockDatabaseOptions = {}) {
     }),
   })
 
-  // Create transaction mock
   const createTransactionMock = () => {
     return vi.fn().mockImplementation(async (callback: any) => {
       if (transactionOptions.throwError) {
@@ -1200,7 +1188,6 @@ export function setupKnowledgeMocks(
     mocks.generateEmbedding = vi.fn().mockResolvedValue([0.1, 0.2, 0.3])
   }
 
-  // Mock the knowledge utilities
   vi.doMock('@/app/api/knowledge/utils', () => mocks)
 
   return mocks
@@ -1218,12 +1205,10 @@ export function setupFileApiMocks(
 ) {
   const { authenticated = true, storageProvider = 's3', cloudEnabled = true } = options
 
-  // Setup basic mocks
   setupCommonApiMocks()
   mockUuid()
   mockCryptoUuid()
 
-  // Setup auth
   const authMocks = mockAuth()
   if (authenticated) {
     authMocks.setAuthenticated()
@@ -1231,14 +1216,12 @@ export function setupFileApiMocks(
     authMocks.setUnauthenticated()
   }
 
-  // Setup file system mocks
   mockFileSystem({
     writeFileSuccess: true,
     readFileContent: 'test content',
     existsResult: true,
   })
 
-  // Setup storage provider mocks (this will mock @/lib/uploads)
   let storageMocks
   if (storageProvider) {
     storageMocks = createStorageProviderMocks({
@@ -1246,7 +1229,6 @@ export function setupFileApiMocks(
       isCloudEnabled: cloudEnabled,
     })
   } else {
-    // If no storage provider specified, just mock the base functions
     vi.doMock('@/lib/uploads', () => ({
       getStorageProvider: vi.fn().mockReturnValue('local'),
       isUsingCloudStorage: vi.fn().mockReturnValue(cloudEnabled),

apps/sim/app/api/auth/oauth/connections/route.ts

@@ -3,6 +3,7 @@ import { jwtDecode } from 'jwt-decode'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { account, user } from '@/db/schema'
 
@@ -18,7 +19,7 @@ interface GoogleIdToken {
  * Get all OAuth connections for the current user
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get the session

apps/sim/app/api/auth/oauth/credentials/route.ts

@@ -6,6 +6,7 @@ import { createLogger } from '@/lib/logs/console/logger'
 import type { OAuthService } from '@/lib/oauth/oauth'
 import { parseProvider } from '@/lib/oauth/oauth'
 import { getUserEntityPermissions } from '@/lib/permissions/utils'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { account, user, workflow } from '@/db/schema'
 
@@ -23,7 +24,7 @@ interface GoogleIdToken {
  * Get credentials for a specific provider
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get query params

apps/sim/app/api/auth/oauth/disconnect/route.ts

@@ -2,6 +2,7 @@ import { and, eq, like, or } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { account } from '@/db/schema'
 
@@ -13,7 +14,7 @@ const logger = createLogger('OAuthDisconnectAPI')
  * Disconnect an OAuth provider for the current user
  */
 export async function POST(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get the session

apps/sim/app/api/auth/oauth/microsoft/file/route.ts

@@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 import { db } from '@/db'
 import { account } from '@/db/schema'
@@ -14,7 +15,7 @@ const logger = createLogger('MicrosoftFileAPI')
  * Get a single file from Microsoft OneDrive
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   try {
     // Get the session
     const session = await getSession()

apps/sim/app/api/auth/oauth/microsoft/files/route.ts

@@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 import { db } from '@/db'
 import { account } from '@/db/schema'
@@ -14,7 +15,7 @@ const logger = createLogger('MicrosoftFilesAPI')
  * Get Excel files from Microsoft OneDrive
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8) // Generate a short request ID for correlation
+  const requestId = generateRequestId()
 
   try {
     // Get the session

apps/sim/app/api/auth/oauth/token/route.ts

@@ -2,6 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { authorizeCredentialUse } from '@/lib/auth/credential-access'
 import { checkHybridAuth } from '@/lib/auth/hybrid'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { getCredential, refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 
 export const dynamic = 'force-dynamic'
@@ -14,7 +15,7 @@ const logger = createLogger('OAuthTokenAPI')
  * and workflow-based authentication (for server-side requests)
  */
 export async function POST(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   logger.info(`[${requestId}] OAuth token API POST request received`)
 
@@ -59,7 +60,7 @@ export async function POST(request: NextRequest) {
  * Get the access token for a specific credential
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8) // Short request ID for correlation
+  const requestId = generateRequestId()
 
   try {
     // Get the credential ID from the query params

apps/sim/app/api/auth/oauth/wealthbox/item/route.ts

@@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 import { db } from '@/db'
 import { account } from '@/db/schema'
@@ -14,7 +15,7 @@ const logger = createLogger('WealthboxItemAPI')
  * Get a single item (note, contact, task) from Wealthbox
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get the session

apps/sim/app/api/auth/oauth/wealthbox/items/route.ts

@@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 import { db } from '@/db'
 import { account } from '@/db/schema'
@@ -14,7 +15,7 @@ const logger = createLogger('WealthboxItemsAPI')
  * Get items (notes, contacts, tasks) from Wealthbox
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get the session

apps/sim/app/api/auth/webhook/stripe/route.ts

@@ -0,0 +1,7 @@
+import { toNextJsHandler } from 'better-auth/next-js'
+import { auth } from '@/lib/auth'
+
+export const dynamic = 'force-dynamic'
+
+// Handle Stripe webhooks through better-auth
+export const { GET, POST } = toNextJsHandler(auth.handler)

apps/sim/app/api/billing/portal/route.ts

@@ -0,0 +1,77 @@
+import { and, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { getSession } from '@/lib/auth'
+import { requireStripeClient } from '@/lib/billing/stripe-client'
+import { env } from '@/lib/env'
+import { createLogger } from '@/lib/logs/console/logger'
+import { db } from '@/db'
+import { subscription as subscriptionTable, user } from '@/db/schema'
+
+const logger = createLogger('BillingPortal')
+
+export async function POST(request: NextRequest) {
+  const session = await getSession()
+
+  try {
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const body = await request.json().catch(() => ({}))
+    const context: 'user' | 'organization' =
+      body?.context === 'organization' ? 'organization' : 'user'
+    const organizationId: string | undefined = body?.organizationId || undefined
+    const returnUrl: string =
+      body?.returnUrl || `${env.NEXT_PUBLIC_APP_URL}/workspace?billing=updated`
+
+    const stripe = requireStripeClient()
+
+    let stripeCustomerId: string | null = null
+
+    if (context === 'organization') {
+      if (!organizationId) {
+        return NextResponse.json({ error: 'organizationId is required' }, { status: 400 })
+      }
+
+      const rows = await db
+        .select({ customer: subscriptionTable.stripeCustomerId })
+        .from(subscriptionTable)
+        .where(
+          and(
+            eq(subscriptionTable.referenceId, organizationId),
+            eq(subscriptionTable.status, 'active')
+          )
+        )
+        .limit(1)
+
+      stripeCustomerId = rows.length > 0 ? rows[0].customer || null : null
+    } else {
+      const rows = await db
+        .select({ customer: user.stripeCustomerId })
+        .from(user)
+        .where(eq(user.id, session.user.id))
+        .limit(1)
+
+      stripeCustomerId = rows.length > 0 ? rows[0].customer || null : null
+    }
+
+    if (!stripeCustomerId) {
+      logger.error('Stripe customer not found for portal session', {
+        context,
+        organizationId,
+        userId: session.user.id,
+      })
+      return NextResponse.json({ error: 'Stripe customer not found' }, { status: 404 })
+    }
+
+    const portal = await stripe.billingPortal.sessions.create({
+      customer: stripeCustomerId,
+      return_url: returnUrl,
+    })
+
+    return NextResponse.json({ url: portal.url })
+  } catch (error) {
+    logger.error('Failed to create billing portal session', { error })
+    return NextResponse.json({ error: 'Failed to create billing portal session' }, { status: 500 })
+  }
+}

apps/sim/app/api/billing/route.ts

@@ -2,10 +2,10 @@ import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { getSimplifiedBillingSummary } from '@/lib/billing/core/billing'
-import { getOrganizationBillingData } from '@/lib/billing/core/organization-billing'
+import { getOrganizationBillingData } from '@/lib/billing/core/organization'
 import { createLogger } from '@/lib/logs/console/logger'
 import { db } from '@/db'
-import { member } from '@/db/schema'
+import { member, userStats } from '@/db/schema'
 
 const logger = createLogger('UnifiedBillingAPI')
 
@@ -45,6 +45,16 @@ export async function GET(request: NextRequest) {
     if (context === 'user') {
       // Get user billing (may include organization if they're part of one)
       billingData = await getSimplifiedBillingSummary(session.user.id, contextId || undefined)
+      // Attach billingBlocked status for the current user
+      const stats = await db
+        .select({ blocked: userStats.billingBlocked })
+        .from(userStats)
+        .where(eq(userStats.userId, session.user.id))
+        .limit(1)
+      billingData = {
+        ...billingData,
+        billingBlocked: stats.length > 0 ? !!stats[0].blocked : false,
+      }
     } else {
       // Get user role in organization for permission checks first
       const memberRecord = await db
@@ -78,8 +88,10 @@ export async function GET(request: NextRequest) {
         subscriptionStatus: rawBillingData.subscriptionStatus,
         totalSeats: rawBillingData.totalSeats,
         usedSeats: rawBillingData.usedSeats,
+        seatsCount: rawBillingData.seatsCount,
         totalCurrentUsage: rawBillingData.totalCurrentUsage,
         totalUsageLimit: rawBillingData.totalUsageLimit,
+        minimumBillingAmount: rawBillingData.minimumBillingAmount,
         averageUsagePerMember: rawBillingData.averageUsagePerMember,
         billingPeriodStart: rawBillingData.billingPeriodStart?.toISOString() || null,
         billingPeriodEnd: rawBillingData.billingPeriodEnd?.toISOString() || null,
@@ -92,11 +104,25 @@ export async function GET(request: NextRequest) {
 
       const userRole = memberRecord[0].role
 
+      // Include the requesting user's blocked flag as well so UI can reflect it
+      const stats = await db
+        .select({ blocked: userStats.billingBlocked })
+        .from(userStats)
+        .where(eq(userStats.userId, session.user.id))
+        .limit(1)
+
+      // Merge blocked flag into data for convenience
+      billingData = {
+        ...billingData,
+        billingBlocked: stats.length > 0 ? !!stats[0].blocked : false,
+      }
+
       return NextResponse.json({
         success: true,
         context,
         data: billingData,
         userRole,
+        billingBlocked: billingData.billingBlocked,
       })
     }
 

apps/sim/app/api/billing/update-cost/route.ts

@@ -1,10 +1,10 @@
-import crypto from 'crypto'
 import { eq, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { checkInternalApiKey } from '@/lib/copilot/utils'
 import { isBillingEnabled } from '@/lib/environment'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { userStats } from '@/db/schema'
 import { calculateCost } from '@/providers/utils'
@@ -16,7 +16,8 @@ const UpdateCostSchema = z.object({
   input: z.number().min(0, 'Input tokens must be a non-negative number'),
   output: z.number().min(0, 'Output tokens must be a non-negative number'),
   model: z.string().min(1, 'Model is required'),
-  multiplier: z.number().min(0),
+  inputMultiplier: z.number().min(0),
+  outputMultiplier: z.number().min(0),
 })
 
 /**
@@ -24,7 +25,7 @@ const UpdateCostSchema = z.object({
  * Update user cost based on token usage with internal API key auth
  */
 export async function POST(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const startTime = Date.now()
 
   try {
@@ -75,14 +76,15 @@ export async function POST(req: NextRequest) {
       )
     }
 
-    const { userId, input, output, model, multiplier } = validation.data
+    const { userId, input, output, model, inputMultiplier, outputMultiplier } = validation.data
 
     logger.info(`[${requestId}] Processing cost update`, {
       userId,
       input,
       output,
       model,
-      multiplier,
+      inputMultiplier,
+      outputMultiplier,
     })
 
     const finalPromptTokens = input
@@ -95,7 +97,8 @@ export async function POST(req: NextRequest) {
       finalPromptTokens,
       finalCompletionTokens,
       false,
-      multiplier
+      inputMultiplier,
+      outputMultiplier
     )
 
     logger.info(`[${requestId}] Cost calculation result`, {
@@ -104,7 +107,8 @@ export async function POST(req: NextRequest) {
       promptTokens: finalPromptTokens,
       completionTokens: finalCompletionTokens,
       totalTokens: totalTokens,
-      multiplier,
+      inputMultiplier,
+      outputMultiplier,
       costResult,
     })
 
@@ -115,52 +119,34 @@ export async function POST(req: NextRequest) {
     const userStatsRecords = await db.select().from(userStats).where(eq(userStats.userId, userId))
 
     if (userStatsRecords.length === 0) {
-      // Create new user stats record (same logic as ExecutionLogger)
-      await db.insert(userStats).values({
-        id: crypto.randomUUID(),
-        userId: userId,
-        totalManualExecutions: 0,
-        totalApiCalls: 0,
-        totalWebhookTriggers: 0,
-        totalScheduledExecutions: 0,
-        totalChatExecutions: 0,
-        totalTokensUsed: totalTokens,
-        totalCost: costToStore.toString(),
-        currentPeriodCost: costToStore.toString(),
-        // Copilot usage tracking
-        totalCopilotCost: costToStore.toString(),
-        totalCopilotTokens: totalTokens,
-        totalCopilotCalls: 1,
-        lastActive: new Date(),
-      })
-
-      logger.info(`[${requestId}] Created new user stats record`, {
-        userId,
-        totalCost: costToStore,
-        totalTokens,
-      })
-    } else {
-      // Update existing user stats record (same logic as ExecutionLogger)
-      const updateFields = {
-        totalTokensUsed: sql`total_tokens_used + ${totalTokens}`,
-        totalCost: sql`total_cost + ${costToStore}`,
-        currentPeriodCost: sql`current_period_cost + ${costToStore}`,
-        // Copilot usage tracking increments
-        totalCopilotCost: sql`total_copilot_cost + ${costToStore}`,
-        totalCopilotTokens: sql`total_copilot_tokens + ${totalTokens}`,
-        totalCopilotCalls: sql`total_copilot_calls + 1`,
-        totalApiCalls: sql`total_api_calls`,
-        lastActive: new Date(),
-      }
-
-      await db.update(userStats).set(updateFields).where(eq(userStats.userId, userId))
-
-      logger.info(`[${requestId}] Updated user stats record`, {
-        userId,
-        addedCost: costToStore,
-        addedTokens: totalTokens,
-      })
+      logger.error(
+        `[${requestId}] User stats record not found - should be created during onboarding`,
+        {
+          userId,
+        }
+      )
+      return NextResponse.json({ error: 'User stats record not found' }, { status: 500 })
     }
+    // Update existing user stats record (same logic as ExecutionLogger)
+    const updateFields = {
+      totalTokensUsed: sql`total_tokens_used + ${totalTokens}`,
+      totalCost: sql`total_cost + ${costToStore}`,
+      currentPeriodCost: sql`current_period_cost + ${costToStore}`,
+      // Copilot usage tracking increments
+      totalCopilotCost: sql`total_copilot_cost + ${costToStore}`,
+      totalCopilotTokens: sql`total_copilot_tokens + ${totalTokens}`,
+      totalCopilotCalls: sql`total_copilot_calls + 1`,
+      totalApiCalls: sql`total_api_calls`,
+      lastActive: new Date(),
+    }
+
+    await db.update(userStats).set(updateFields).where(eq(userStats.userId, userId))
+
+    logger.info(`[${requestId}] Updated user stats record`, {
+      userId,
+      addedCost: costToStore,
+      addedTokens: totalTokens,
+    })
 
     const duration = Date.now() - startTime
 

apps/sim/app/api/billing/webhooks/stripe/route.ts

@@ -1,116 +0,0 @@
-import { headers } from 'next/headers'
-import { type NextRequest, NextResponse } from 'next/server'
-import type Stripe from 'stripe'
-import { requireStripeClient } from '@/lib/billing/stripe-client'
-import { handleInvoiceWebhook } from '@/lib/billing/webhooks/stripe-invoice-webhooks'
-import { env } from '@/lib/env'
-import { createLogger } from '@/lib/logs/console/logger'
-
-const logger = createLogger('StripeInvoiceWebhook')
-
-/**
- * Stripe billing webhook endpoint for invoice-related events
- * Endpoint: /api/billing/webhooks/stripe
- * Handles: invoice.payment_succeeded, invoice.payment_failed, invoice.finalized
- */
-export async function POST(request: NextRequest) {
-  try {
-    const body = await request.text()
-    const headersList = await headers()
-    const signature = headersList.get('stripe-signature')
-
-    if (!signature) {
-      logger.error('Missing Stripe signature header')
-      return NextResponse.json({ error: 'Missing Stripe signature' }, { status: 400 })
-    }
-
-    if (!env.STRIPE_BILLING_WEBHOOK_SECRET) {
-      logger.error('Missing Stripe webhook secret configuration')
-      return NextResponse.json({ error: 'Webhook secret not configured' }, { status: 500 })
-    }
-
-    // Check if Stripe client is available
-    let stripe
-    try {
-      stripe = requireStripeClient()
-    } catch (stripeError) {
-      logger.error('Stripe client not available for webhook processing', {
-        error: stripeError,
-      })
-      return NextResponse.json({ error: 'Stripe client not configured' }, { status: 500 })
-    }
-
-    // Verify webhook signature
-    let event: Stripe.Event
-    try {
-      event = stripe.webhooks.constructEvent(body, signature, env.STRIPE_BILLING_WEBHOOK_SECRET)
-    } catch (signatureError) {
-      logger.error('Invalid Stripe webhook signature', {
-        error: signatureError,
-        signature,
-      })
-      return NextResponse.json({ error: 'Invalid signature' }, { status: 400 })
-    }
-
-    logger.info('Received Stripe invoice webhook', {
-      eventId: event.id,
-      eventType: event.type,
-    })
-
-    // Handle specific invoice events
-    const supportedEvents = [
-      'invoice.payment_succeeded',
-      'invoice.payment_failed',
-      'invoice.finalized',
-    ]
-
-    if (supportedEvents.includes(event.type)) {
-      try {
-        await handleInvoiceWebhook(event)
-
-        logger.info('Successfully processed invoice webhook', {
-          eventId: event.id,
-          eventType: event.type,
-        })
-
-        return NextResponse.json({ received: true })
-      } catch (processingError) {
-        logger.error('Failed to process invoice webhook', {
-          eventId: event.id,
-          eventType: event.type,
-          error: processingError,
-        })
-
-        // Return 500 to tell Stripe to retry the webhook
-        return NextResponse.json({ error: 'Failed to process webhook' }, { status: 500 })
-      }
-    } else {
-      // Not a supported invoice event, ignore
-      logger.info('Ignoring unsupported webhook event', {
-        eventId: event.id,
-        eventType: event.type,
-        supportedEvents,
-      })
-
-      return NextResponse.json({ received: true })
-    }
-  } catch (error) {
-    logger.error('Fatal error in invoice webhook handler', {
-      error,
-      url: request.url,
-    })
-
-    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
-  }
-}
-
-/**
- * GET endpoint for webhook health checks
- */
-export async function GET() {
-  return NextResponse.json({
-    status: 'healthy',
-    webhook: 'stripe-invoices',
-    events: ['invoice.payment_succeeded', 'invoice.payment_failed', 'invoice.finalized'],
-  })
-}

apps/sim/app/api/chat/[subdomain]/otp/route.ts

@@ -5,6 +5,7 @@ import { renderOTPEmail } from '@/components/emails/render-email'
 import { sendEmail } from '@/lib/email/mailer'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getRedisClient, markMessageAsProcessed, releaseLock } from '@/lib/redis'
+import { generateRequestId } from '@/lib/utils'
 import { addCorsHeaders, setChatAuthCookie } from '@/app/api/chat/utils'
 import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils'
 import { db } from '@/db'
@@ -115,7 +116,7 @@ export async function POST(
   { params }: { params: Promise<{ subdomain: string }> }
 ) {
   const { subdomain } = await params
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.debug(`[${requestId}] Processing OTP request for subdomain: ${subdomain}`)
@@ -229,7 +230,7 @@ export async function PUT(
   { params }: { params: Promise<{ subdomain: string }> }
 ) {
   const { subdomain } = await params
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.debug(`[${requestId}] Verifying OTP for subdomain: ${subdomain}`)

apps/sim/app/api/chat/[subdomain]/route.ts

@@ -1,6 +1,7 @@
 import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import {
   addCorsHeaders,
   executeWorkflowForChat,
@@ -20,7 +21,7 @@ export async function POST(
   { params }: { params: Promise<{ subdomain: string }> }
 ) {
   const { subdomain } = await params
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.debug(`[${requestId}] Processing chat request for subdomain: ${subdomain}`)
@@ -141,7 +142,7 @@ export async function GET(
   { params }: { params: Promise<{ subdomain: string }> }
 ) {
   const { subdomain } = await params
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.debug(`[${requestId}] Fetching chat info for subdomain: ${subdomain}`)

apps/sim/app/api/chat/subdomains/validate/route.ts

@@ -45,6 +45,7 @@ export async function GET(request: Request) {
       'support',
       'admin',
       'qa',
+      'agent',
     ]
     if (reservedSubdomains.includes(subdomain)) {
       return NextResponse.json(

apps/sim/app/api/chat/utils.test.ts

@@ -14,10 +14,6 @@ vi.mock('@/db', () => ({
   },
 }))
 
-vi.mock('@/lib/utils', () => ({
-  decryptSecret: vi.fn().mockResolvedValue({ decrypted: 'test-secret' }),
-}))
-
 vi.mock('@/lib/logs/execution/logging-session', () => ({
   LoggingSession: vi.fn().mockImplementation(() => ({
     safeStart: vi.fn().mockResolvedValue(undefined),
@@ -38,6 +34,13 @@ vi.mock('@/stores/workflows/server-utils', () => ({
   mergeSubblockState: vi.fn().mockReturnValue({}),
 }))
 
+const mockDecryptSecret = vi.fn()
+
+vi.mock('@/lib/utils', () => ({
+  decryptSecret: mockDecryptSecret,
+  generateRequestId: vi.fn(),
+}))
+
 describe('Chat API Utils', () => {
   beforeEach(() => {
     vi.resetModules()
@@ -177,7 +180,10 @@ describe('Chat API Utils', () => {
   })
 
   describe('Chat auth validation', () => {
-    beforeEach(() => {
+    beforeEach(async () => {
+      vi.clearAllMocks()
+      mockDecryptSecret.mockResolvedValue({ decrypted: 'correct-password' })
+
       vi.doMock('@/app/api/chat/utils', async (importOriginal) => {
         const original = (await importOriginal()) as any
         return {
@@ -190,13 +196,6 @@ describe('Chat API Utils', () => {
           }),
         }
       })
-
-      // Mock decryptSecret globally for all auth tests
-      vi.doMock('@/lib/utils', () => ({
-        decryptSecret: vi.fn((encryptedValue) => {
-          return Promise.resolve({ decrypted: 'correct-password' })
-        }),
-      }))
     })
 
     it.concurrent('should allow access to public chats', async () => {

apps/sim/app/api/chat/utils.ts

@@ -3,16 +3,17 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { v4 as uuidv4 } from 'uuid'
 import { checkServerSideUsageLimits } from '@/lib/billing'
 import { isDev } from '@/lib/environment'
+import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
 import { createLogger } from '@/lib/logs/console/logger'
 import { LoggingSession } from '@/lib/logs/execution/logging-session'
 import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
 import { hasAdminPermission } from '@/lib/permissions/utils'
 import { processStreamingBlockLogs } from '@/lib/tokenization'
 import { getEmailDomain } from '@/lib/urls/utils'
-import { decryptSecret } from '@/lib/utils'
+import { decryptSecret, generateRequestId } from '@/lib/utils'
 import { getBlock } from '@/blocks'
 import { db } from '@/db'
-import { chat, environment as envTable, userStats, workflow } from '@/db/schema'
+import { chat, userStats, workflow } from '@/db/schema'
 import { Executor } from '@/executor'
 import type { BlockLog, ExecutionResult } from '@/executor/types'
 import { Serializer } from '@/serializer'
@@ -302,7 +303,7 @@ export async function executeWorkflowForChat(
   input: string,
   conversationId?: string
 ): Promise<any> {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   logger.debug(
     `[${requestId}] Executing workflow for chat: ${chatId}${
@@ -453,18 +454,21 @@ export async function executeWorkflowForChat(
     {} as Record<string, Record<string, any>>
   )
 
-  // Get user environment variables for this workflow
+  // Get user environment variables with workspace precedence
   let envVars: Record<string, string> = {}
   try {
-    const envResult = await db
-      .select()
-      .from(envTable)
-      .where(eq(envTable.userId, deployment.userId))
+    const wfWorkspaceRow = await db
+      .select({ workspaceId: workflow.workspaceId })
+      .from(workflow)
+      .where(eq(workflow.id, workflowId))
       .limit(1)
 
-    if (envResult.length > 0 && envResult[0].variables) {
-      envVars = envResult[0].variables as Record<string, string>
-    }
+    const workspaceId = wfWorkspaceRow[0]?.workspaceId || undefined
+    const { personalEncrypted, workspaceEncrypted } = await getPersonalAndWorkspaceEnv(
+      deployment.userId,
+      workspaceId
+    )
+    envVars = { ...personalEncrypted, ...workspaceEncrypted }
   } catch (error) {
     logger.warn(`[${requestId}] Could not fetch environment variables:`, error)
   }

apps/sim/app/api/copilot/api-keys/generate/route.ts

@@ -1,34 +1,12 @@
-import { createCipheriv, createHash, createHmac, randomBytes } from 'crypto'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
-import { generateApiKey } from '@/lib/utils'
-import { db } from '@/db'
-import { copilotApiKeys } from '@/db/schema'
+import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
 
 const logger = createLogger('CopilotApiKeysGenerate')
 
-function deriveKey(keyString: string): Buffer {
-  return createHash('sha256').update(keyString, 'utf8').digest()
-}
-
-function encryptRandomIv(plaintext: string, keyString: string): string {
-  const key = deriveKey(keyString)
-  const iv = randomBytes(16)
-  const cipher = createCipheriv('aes-256-gcm', key, iv)
-  let encrypted = cipher.update(plaintext, 'utf8', 'hex')
-  encrypted += cipher.final('hex')
-  const authTag = cipher.getAuthTag().toString('hex')
-  return `${iv.toString('hex')}:${encrypted}:${authTag}`
-}
-
-function computeLookup(plaintext: string, keyString: string): string {
-  // Deterministic, constant-time comparable MAC: HMAC-SHA256(DB_KEY, plaintext)
-  return createHmac('sha256', Buffer.from(keyString, 'utf8'))
-    .update(plaintext, 'utf8')
-    .digest('hex')
-}
+const SIM_AGENT_API_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
 
 export async function POST(req: NextRequest) {
   try {
@@ -37,34 +15,39 @@ export async function POST(req: NextRequest) {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    if (!env.AGENT_API_DB_ENCRYPTION_KEY) {
-      logger.error('AGENT_API_DB_ENCRYPTION_KEY is not set')
-      return NextResponse.json({ error: 'Server not configured' }, { status: 500 })
-    }
-
     const userId = session.user.id
 
-    // Generate and prefix the key (strip the generic sim_ prefix from the random part)
-    const rawKey = generateApiKey().replace(/^sim_/, '')
-    const plaintextKey = `sk-sim-copilot-${rawKey}`
+    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/generate`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
+      },
+      body: JSON.stringify({ userId }),
+    })
 
-    // Encrypt with random IV for confidentiality
-    const dbEncrypted = encryptRandomIv(plaintextKey, env.AGENT_API_DB_ENCRYPTION_KEY)
+    if (!res.ok) {
+      const errorBody = await res.text().catch(() => '')
+      logger.error('Sim Agent generate key error', { status: res.status, error: errorBody })
+      return NextResponse.json(
+        { error: 'Failed to generate copilot API key' },
+        { status: res.status || 500 }
+      )
+    }
 
-    // Compute deterministic lookup value for O(1) search
-    const lookup = computeLookup(plaintextKey, env.AGENT_API_DB_ENCRYPTION_KEY)
+    const data = (await res.json().catch(() => null)) as { apiKey?: string } | null
 
-    const [inserted] = await db
-      .insert(copilotApiKeys)
-      .values({ userId, apiKeyEncrypted: dbEncrypted, apiKeyLookup: lookup })
-      .returning({ id: copilotApiKeys.id })
+    if (!data?.apiKey) {
+      logger.error('Sim Agent generate key returned invalid payload')
+      return NextResponse.json({ error: 'Invalid response from Sim Agent' }, { status: 500 })
+    }
 
     return NextResponse.json(
-      { success: true, key: { id: inserted.id, apiKey: plaintextKey } },
+      { success: true, key: { id: 'new', apiKey: data.apiKey } },
       { status: 201 }
     )
   } catch (error) {
-    logger.error('Failed to generate copilot API key', { error })
+    logger.error('Failed to proxy generate copilot API key', { error })
     return NextResponse.json({ error: 'Failed to generate copilot API key' }, { status: 500 })
   }
 }

apps/sim/app/api/copilot/api-keys/route.ts

@@ -1,32 +1,12 @@
-import { createDecipheriv, createHash } from 'crypto'
-import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
-import { db } from '@/db'
-import { copilotApiKeys } from '@/db/schema'
+import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
 
 const logger = createLogger('CopilotApiKeys')
 
-function deriveKey(keyString: string): Buffer {
-  return createHash('sha256').update(keyString, 'utf8').digest()
-}
-
-function decryptWithKey(encryptedValue: string, keyString: string): string {
-  const parts = encryptedValue.split(':')
-  if (parts.length !== 3) {
-    throw new Error('Invalid encrypted value format')
-  }
-  const [ivHex, encryptedHex, authTagHex] = parts
-  const key = deriveKey(keyString)
-  const iv = Buffer.from(ivHex, 'hex')
-  const decipher = createDecipheriv('aes-256-gcm', key, iv)
-  decipher.setAuthTag(Buffer.from(authTagHex, 'hex'))
-  let decrypted = decipher.update(encryptedHex, 'hex', 'utf8')
-  decrypted += decipher.final('utf8')
-  return decrypted
-}
+const SIM_AGENT_API_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
 
 export async function GET(request: NextRequest) {
   try {
@@ -35,22 +15,31 @@ export async function GET(request: NextRequest) {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    if (!env.AGENT_API_DB_ENCRYPTION_KEY) {
-      logger.error('AGENT_API_DB_ENCRYPTION_KEY is not set')
-      return NextResponse.json({ error: 'Server not configured' }, { status: 500 })
-    }
-
     const userId = session.user.id
 
-    const rows = await db
-      .select({ id: copilotApiKeys.id, apiKeyEncrypted: copilotApiKeys.apiKeyEncrypted })
-      .from(copilotApiKeys)
-      .where(eq(copilotApiKeys.userId, userId))
+    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/get-api-keys`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
+      },
+      body: JSON.stringify({ userId }),
+    })
 
-    const keys = rows.map((row) => ({
-      id: row.id,
-      apiKey: decryptWithKey(row.apiKeyEncrypted, env.AGENT_API_DB_ENCRYPTION_KEY as string),
-    }))
+    if (!res.ok) {
+      const errorBody = await res.text().catch(() => '')
+      logger.error('Sim Agent get-api-keys error', { status: res.status, error: errorBody })
+      return NextResponse.json({ error: 'Failed to get keys' }, { status: res.status || 500 })
+    }
+
+    const apiKeys = (await res.json().catch(() => null)) as { id: string; apiKey: string }[] | null
+
+    if (!Array.isArray(apiKeys)) {
+      logger.error('Sim Agent get-api-keys returned invalid payload')
+      return NextResponse.json({ error: 'Invalid response from Sim Agent' }, { status: 500 })
+    }
+
+    const keys = apiKeys
 
     return NextResponse.json({ keys }, { status: 200 })
   } catch (error) {
@@ -73,9 +62,26 @@ export async function DELETE(request: NextRequest) {
       return NextResponse.json({ error: 'id is required' }, { status: 400 })
     }
 
-    await db
-      .delete(copilotApiKeys)
-      .where(and(eq(copilotApiKeys.userId, userId), eq(copilotApiKeys.id, id)))
+    const res = await fetch(`${SIM_AGENT_API_URL}/api/validate-key/delete`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
+      },
+      body: JSON.stringify({ userId, apiKeyId: id }),
+    })
+
+    if (!res.ok) {
+      const errorBody = await res.text().catch(() => '')
+      logger.error('Sim Agent delete key error', { status: res.status, error: errorBody })
+      return NextResponse.json({ error: 'Failed to delete key' }, { status: res.status || 500 })
+    }
+
+    const data = (await res.json().catch(() => null)) as { success?: boolean } | null
+    if (!data?.success) {
+      logger.error('Sim Agent delete key returned invalid payload')
+      return NextResponse.json({ error: 'Invalid response from Sim Agent' }, { status: 500 })
+    }
 
     return NextResponse.json({ success: true }, { status: 200 })
   } catch (error) {

apps/sim/app/api/copilot/api-keys/validate/route.ts

@@ -1,50 +1,29 @@
-import { createHmac } from 'crypto'
 import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
-import { env } from '@/lib/env'
+import { checkInternalApiKey } from '@/lib/copilot/utils'
 import { createLogger } from '@/lib/logs/console/logger'
 import { db } from '@/db'
-import { copilotApiKeys, userStats } from '@/db/schema'
+import { userStats } from '@/db/schema'
 
 const logger = createLogger('CopilotApiKeysValidate')
 
-function computeLookup(plaintext: string, keyString: string): string {
-  // Deterministic MAC: HMAC-SHA256(DB_KEY, plaintext)
-  return createHmac('sha256', Buffer.from(keyString, 'utf8'))
-    .update(plaintext, 'utf8')
-    .digest('hex')
-}
-
 export async function POST(req: NextRequest) {
   try {
-    if (!env.AGENT_API_DB_ENCRYPTION_KEY) {
-      logger.error('AGENT_API_DB_ENCRYPTION_KEY is not set')
-      return NextResponse.json({ error: 'Server not configured' }, { status: 500 })
+    // Authenticate via internal API key header
+    const auth = checkInternalApiKey(req)
+    if (!auth.success) {
+      return new NextResponse(null, { status: 401 })
     }
 
     const body = await req.json().catch(() => null)
-    const apiKey = typeof body?.apiKey === 'string' ? body.apiKey : undefined
+    const userId = typeof body?.userId === 'string' ? body.userId : undefined
 
-    if (!apiKey) {
-      return new NextResponse(null, { status: 401 })
+    if (!userId) {
+      return NextResponse.json({ error: 'userId is required' }, { status: 400 })
     }
 
-    const lookup = computeLookup(apiKey, env.AGENT_API_DB_ENCRYPTION_KEY)
+    logger.info('[API VALIDATION] Validating usage limit', { userId })
 
-    // Find matching API key and its user
-    const rows = await db
-      .select({ id: copilotApiKeys.id, userId: copilotApiKeys.userId })
-      .from(copilotApiKeys)
-      .where(eq(copilotApiKeys.apiKeyLookup, lookup))
-      .limit(1)
-
-    if (rows.length === 0) {
-      return new NextResponse(null, { status: 401 })
-    }
-
-    const { userId } = rows[0]
-
-    // Check usage for the associated user
     const usage = await db
       .select({
         currentPeriodCost: userStats.currentPeriodCost,
@@ -55,6 +34,8 @@ export async function POST(req: NextRequest) {
       .where(eq(userStats.userId, userId))
       .limit(1)
 
+    logger.info('[API VALIDATION] Usage limit validated', { userId, usage })
+
     if (usage.length > 0) {
       const currentUsage = Number.parseFloat(
         (usage[0].currentPeriodCost?.toString() as string) ||
@@ -64,16 +45,14 @@ export async function POST(req: NextRequest) {
       const limit = Number.parseFloat((usage[0].currentUsageLimit as unknown as string) || '0')
 
       if (!Number.isNaN(limit) && limit > 0 && currentUsage >= limit) {
-        // Usage exceeded
         logger.info('[API VALIDATION] Usage exceeded', { userId, currentUsage, limit })
         return new NextResponse(null, { status: 402 })
       }
     }
 
-    // Valid and within usage limits
     return new NextResponse(null, { status: 200 })
   } catch (error) {
-    logger.error('Error validating copilot API key', { error })
-    return NextResponse.json({ error: 'Failed to validate key' }, { status: 500 })
+    logger.error('Error validating usage limit', { error })
+    return NextResponse.json({ error: 'Failed to validate usage' }, { status: 500 })
   }
 }

apps/sim/app/api/copilot/chat/route.test.ts

@@ -99,6 +99,7 @@ describe('Copilot Chat API Route', () => {
 
     vi.doMock('@/lib/utils', () => ({
       getRotatingApiKey: mockGetRotatingApiKey,
+      generateRequestId: vi.fn(() => 'test-request-id'),
     }))
 
     vi.doMock('@/lib/env', () => ({
@@ -224,9 +225,9 @@ describe('Copilot Chat API Route', () => {
             stream: true,
             streamToolCalls: true,
             mode: 'agent',
-            provider: 'openai',
+            messageId: 'mock-uuid-1234-5678',
             depth: 0,
-            origin: 'http://localhost:3000',
+            chatId: 'chat-123',
           }),
         })
       )
@@ -288,9 +289,9 @@ describe('Copilot Chat API Route', () => {
             stream: true,
             streamToolCalls: true,
             mode: 'agent',
-            provider: 'openai',
+            messageId: 'mock-uuid-1234-5678',
             depth: 0,
-            origin: 'http://localhost:3000',
+            chatId: 'chat-123',
           }),
         })
       )
@@ -300,7 +301,6 @@ describe('Copilot Chat API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock new chat creation
       const newChat = {
         id: 'chat-123',
         userId: 'user-123',
@@ -309,8 +309,6 @@ describe('Copilot Chat API Route', () => {
       }
       mockReturning.mockResolvedValue([newChat])
 
-      // Mock sim agent response
-
       ;(global.fetch as any).mockResolvedValue({
         ok: true,
         body: new ReadableStream({
@@ -344,9 +342,9 @@ describe('Copilot Chat API Route', () => {
             stream: true,
             streamToolCalls: true,
             mode: 'agent',
-            provider: 'openai',
+            messageId: 'mock-uuid-1234-5678',
             depth: 0,
-            origin: 'http://localhost:3000',
+            chatId: 'chat-123',
           }),
         })
       )
@@ -356,11 +354,8 @@ describe('Copilot Chat API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock new chat creation
       mockReturning.mockResolvedValue([{ id: 'chat-123', messages: [] }])
 
-      // Mock sim agent error
-
       ;(global.fetch as any).mockResolvedValue({
         ok: false,
         status: 500,
@@ -406,11 +401,8 @@ describe('Copilot Chat API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock new chat creation
       mockReturning.mockResolvedValue([{ id: 'chat-123', messages: [] }])
 
-      // Mock sim agent response
-
       ;(global.fetch as any).mockResolvedValue({
         ok: true,
         body: new ReadableStream({
@@ -440,9 +432,9 @@ describe('Copilot Chat API Route', () => {
             stream: true,
             streamToolCalls: true,
             mode: 'ask',
-            provider: 'openai',
+            messageId: 'mock-uuid-1234-5678',
             depth: 0,
-            origin: 'http://localhost:3000',
+            chatId: 'chat-123',
           }),
         })
       )

apps/sim/app/api/copilot/chat/route.ts

@@ -1,4 +1,3 @@
-import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'crypto'
 import { and, desc, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
@@ -11,77 +10,36 @@ import {
   createUnauthorizedResponse,
 } from '@/lib/copilot/auth'
 import { getCopilotModel } from '@/lib/copilot/config'
-import { TITLE_GENERATION_SYSTEM_PROMPT, TITLE_GENERATION_USER_PROMPT } from '@/lib/copilot/prompts'
+import type { CopilotProviderConfig } from '@/lib/copilot/types'
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
 import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
-import { downloadFile } from '@/lib/uploads'
-import { downloadFromS3WithConfig } from '@/lib/uploads/s3/s3-client'
-import { S3_COPILOT_CONFIG, USE_S3_STORAGE } from '@/lib/uploads/setup'
+import { generateChatTitle } from '@/lib/sim-agent/utils'
+import { createFileContent, isSupportedFileType } from '@/lib/uploads/file-utils'
+import { S3_COPILOT_CONFIG } from '@/lib/uploads/setup'
+import { downloadFile, getStorageProvider } from '@/lib/uploads/storage-client'
 import { db } from '@/db'
 import { copilotChats } from '@/db/schema'
-import { executeProviderRequest } from '@/providers'
-import { createAnthropicFileContent, isSupportedFileType } from './file-utils'
 
 const logger = createLogger('CopilotChatAPI')
 
-// Sim Agent API configuration
 const SIM_AGENT_API_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
 
-function getRequestOrigin(_req: NextRequest): string {
-  try {
-    // Strictly use configured Better Auth URL
-    return env.BETTER_AUTH_URL || ''
-  } catch (_) {
-    return ''
-  }
-}
-
-function deriveKey(keyString: string): Buffer {
-  return createHash('sha256').update(keyString, 'utf8').digest()
-}
-
-function decryptWithKey(encryptedValue: string, keyString: string): string {
-  const [ivHex, encryptedHex, authTagHex] = encryptedValue.split(':')
-  if (!ivHex || !encryptedHex || !authTagHex) {
-    throw new Error('Invalid encrypted format')
-  }
-  const key = deriveKey(keyString)
-  const iv = Buffer.from(ivHex, 'hex')
-  const decipher = createDecipheriv('aes-256-gcm', key, iv)
-  decipher.setAuthTag(Buffer.from(authTagHex, 'hex'))
-  let decrypted = decipher.update(encryptedHex, 'hex', 'utf8')
-  decrypted += decipher.final('utf8')
-  return decrypted
-}
-
-function encryptWithKey(plaintext: string, keyString: string): string {
-  const key = deriveKey(keyString)
-  const iv = randomBytes(16)
-  const cipher = createCipheriv('aes-256-gcm', key, iv)
-  let encrypted = cipher.update(plaintext, 'utf8', 'hex')
-  encrypted += cipher.final('hex')
-  const authTag = cipher.getAuthTag().toString('hex')
-  return `${iv.toString('hex')}:${encrypted}:${authTag}`
-}
-
-// Schema for file attachments
 const FileAttachmentSchema = z.object({
   id: z.string(),
-  s3_key: z.string(),
+  key: z.string(),
   filename: z.string(),
   media_type: z.string(),
   size: z.number(),
 })
 
-// Schema for chat messages
 const ChatMessageSchema = z.object({
   message: z.string().min(1, 'Message is required'),
   userMessageId: z.string().optional(), // ID from frontend for the user message
   chatId: z.string().optional(),
   workflowId: z.string().min(1, 'Workflow ID is required'),
   mode: z.enum(['ask', 'agent']).optional().default('agent'),
-  depth: z.number().int().min(-2).max(3).optional().default(0),
+  depth: z.number().int().min(0).max(3).optional().default(0),
   prefetch: z.boolean().optional(),
   createNewChat: z.boolean().optional().default(false),
   stream: z.boolean().optional().default(true),
@@ -89,90 +47,32 @@ const ChatMessageSchema = z.object({
   fileAttachments: z.array(FileAttachmentSchema).optional(),
   provider: z.string().optional().default('openai'),
   conversationId: z.string().optional(),
-})
-
-/**
- * Generate a chat title using LLM
- */
-async function generateChatTitle(userMessage: string): Promise<string> {
-  try {
-    const { provider, model } = getCopilotModel('title')
-
-    // Get the appropriate API key for the provider
-    let apiKey: string | undefined
-    if (provider === 'anthropic') {
-      // Use rotating API key for Anthropic
-      const { getRotatingApiKey } = require('@/lib/utils')
-      try {
-        apiKey = getRotatingApiKey('anthropic')
-        logger.debug(`Using rotating API key for Anthropic title generation`)
-      } catch (e) {
-        // If rotation fails, let the provider handle it
-        logger.warn(`Failed to get rotating API key for Anthropic:`, e)
-      }
-    }
-
-    const response = await executeProviderRequest(provider, {
-      model,
-      systemPrompt: TITLE_GENERATION_SYSTEM_PROMPT,
-      context: TITLE_GENERATION_USER_PROMPT(userMessage),
-      temperature: 0.3,
-      maxTokens: 50,
-      apiKey: apiKey || '',
-      stream: false,
-    })
-
-    if (typeof response === 'object' && 'content' in response) {
-      return response.content?.trim() || 'New Chat'
-    }
-
-    return 'New Chat'
-  } catch (error) {
-    logger.error('Failed to generate chat title:', error)
-    return 'New Chat'
-  }
-}
-
-/**
- * Generate chat title asynchronously and update the database
- */
-async function generateChatTitleAsync(
-  chatId: string,
-  userMessage: string,
-  requestId: string,
-  streamController?: ReadableStreamDefaultController<Uint8Array>
-): Promise<void> {
-  try {
-    // logger.info(`[${requestId}] Starting async title generation for chat ${chatId}`)
-
-    const title = await generateChatTitle(userMessage)
-
-    // Update the chat with the generated title
-    await db
-      .update(copilotChats)
-      .set({
-        title,
-        updatedAt: new Date(),
+  contexts: z
+    .array(
+      z.object({
+        kind: z.enum([
+          'past_chat',
+          'workflow',
+          'current_workflow',
+          'blocks',
+          'logs',
+          'workflow_block',
+          'knowledge',
+          'templates',
+          'docs',
+        ]),
+        label: z.string(),
+        chatId: z.string().optional(),
+        workflowId: z.string().optional(),
+        knowledgeId: z.string().optional(),
+        blockId: z.string().optional(),
+        templateId: z.string().optional(),
+        executionId: z.string().optional(),
+        // For workflow_block, provide both workflowId and blockId
       })
-      .where(eq(copilotChats.id, chatId))
-
-    // Send title_updated event to client if streaming
-    if (streamController) {
-      const encoder = new TextEncoder()
-      const titleEvent = `data: ${JSON.stringify({
-        type: 'title_updated',
-        title: title,
-      })}\n\n`
-      streamController.enqueue(encoder.encode(titleEvent))
-      logger.debug(`[${requestId}] Sent title_updated event to client: "${title}"`)
-    }
-
-    // logger.info(`[${requestId}] Generated title for chat ${chatId}: "${title}"`)
-  } catch (error) {
-    logger.error(`[${requestId}] Failed to generate title for chat ${chatId}:`, error)
-    // Don't throw - this is a background operation
-  }
-}
+    )
+    .optional(),
+})
 
 /**
  * POST /api/copilot/chat
@@ -206,14 +106,45 @@ export async function POST(req: NextRequest) {
       fileAttachments,
       provider,
       conversationId,
+      contexts,
     } = ChatMessageSchema.parse(body)
-
-    // Derive request origin for downstream service
-    const requestOrigin = getRequestOrigin(req)
-
-    if (!requestOrigin) {
-      logger.error(`[${tracker.requestId}] Missing required configuration: BETTER_AUTH_URL`)
-      return createInternalServerErrorResponse('Missing required configuration: BETTER_AUTH_URL')
+    // Ensure we have a consistent user message ID for this request
+    const userMessageIdToUse = userMessageId || crypto.randomUUID()
+    try {
+      logger.info(`[${tracker.requestId}] Received chat POST`, {
+        hasContexts: Array.isArray(contexts),
+        contextsCount: Array.isArray(contexts) ? contexts.length : 0,
+        contextsPreview: Array.isArray(contexts)
+          ? contexts.map((c: any) => ({
+              kind: c?.kind,
+              chatId: c?.chatId,
+              workflowId: c?.workflowId,
+              executionId: (c as any)?.executionId,
+              label: c?.label,
+            }))
+          : undefined,
+      })
+    } catch {}
+    // Preprocess contexts server-side
+    let agentContexts: Array<{ type: string; content: string }> = []
+    if (Array.isArray(contexts) && contexts.length > 0) {
+      try {
+        const { processContextsServer } = await import('@/lib/copilot/process-contents')
+        const processed = await processContextsServer(contexts as any, authenticatedUserId, message)
+        agentContexts = processed
+        logger.info(`[${tracker.requestId}] Contexts processed for request`, {
+          processedCount: agentContexts.length,
+          kinds: agentContexts.map((c) => c.type),
+          lengthPreview: agentContexts.map((c) => c.content?.length ?? 0),
+        })
+        if (Array.isArray(contexts) && contexts.length > 0 && agentContexts.length === 0) {
+          logger.warn(
+            `[${tracker.requestId}] Contexts provided but none processed. Check executionId for logs contexts.`
+          )
+        }
+      } catch (e) {
+        logger.error(`[${tracker.requestId}] Failed to process contexts`, e)
+      }
     }
 
     // Consolidation mapping: map negative depths to base depth with prefetch=true
@@ -229,22 +160,6 @@ export async function POST(req: NextRequest) {
       }
     }
 
-    // logger.info(`[${tracker.requestId}] Processing copilot chat request`, {
-    //   userId: authenticatedUserId,
-    //   workflowId,
-    //   chatId,
-    //   mode,
-    //   stream,
-    //   createNewChat,
-    //   messageLength: message.length,
-    //   hasImplicitFeedback: !!implicitFeedback,
-    //   provider: provider || 'openai',
-    //   hasConversationId: !!conversationId,
-    //   depth,
-    //   prefetch,
-    //   origin: requestOrigin,
-    // })
-
     // Handle chat context
     let currentChat: any = null
     let conversationHistory: any[] = []
@@ -285,8 +200,6 @@ export async function POST(req: NextRequest) {
     // Process file attachments if present
     const processedFileContents: any[] = []
     if (fileAttachments && fileAttachments.length > 0) {
-      // logger.info(`[${tracker.requestId}] Processing ${fileAttachments.length} file attachments`)
-
       for (const attachment of fileAttachments) {
         try {
           // Check if file type is supported
@@ -295,23 +208,30 @@ export async function POST(req: NextRequest) {
             continue
           }
 
-          // Download file from S3
-          // logger.info(`[${tracker.requestId}] Downloading file: ${attachment.s3_key}`)
+          const storageProvider = getStorageProvider()
           let fileBuffer: Buffer
-          if (USE_S3_STORAGE) {
-            fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG)
+
+          if (storageProvider === 's3') {
+            fileBuffer = await downloadFile(attachment.key, {
+              bucket: S3_COPILOT_CONFIG.bucket,
+              region: S3_COPILOT_CONFIG.region,
+            })
+          } else if (storageProvider === 'blob') {
+            const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup')
+            fileBuffer = await downloadFile(attachment.key, {
+              containerName: BLOB_COPILOT_CONFIG.containerName,
+              accountName: BLOB_COPILOT_CONFIG.accountName,
+              accountKey: BLOB_COPILOT_CONFIG.accountKey,
+              connectionString: BLOB_COPILOT_CONFIG.connectionString,
+            })
           } else {
-            // Fallback to generic downloadFile for other storage providers
-            fileBuffer = await downloadFile(attachment.s3_key)
+            fileBuffer = await downloadFile(attachment.key)
           }
 
-          // Convert to Anthropic format
-          const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type)
+          // Convert to format
+          const fileContent = createFileContent(fileBuffer, attachment.media_type)
           if (fileContent) {
             processedFileContents.push(fileContent)
-            // logger.info(
-            //   `[${tracker.requestId}] Processed file: ${attachment.filename} (${attachment.media_type})`
-            // )
           }
         } catch (error) {
           logger.error(
@@ -336,14 +256,26 @@ export async function POST(req: NextRequest) {
         for (const attachment of msg.fileAttachments) {
           try {
             if (isSupportedFileType(attachment.media_type)) {
+              const storageProvider = getStorageProvider()
               let fileBuffer: Buffer
-              if (USE_S3_STORAGE) {
-                fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG)
+
+              if (storageProvider === 's3') {
+                fileBuffer = await downloadFile(attachment.key, {
+                  bucket: S3_COPILOT_CONFIG.bucket,
+                  region: S3_COPILOT_CONFIG.region,
+                })
+              } else if (storageProvider === 'blob') {
+                const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup')
+                fileBuffer = await downloadFile(attachment.key, {
+                  containerName: BLOB_COPILOT_CONFIG.containerName,
+                  accountName: BLOB_COPILOT_CONFIG.accountName,
+                  accountKey: BLOB_COPILOT_CONFIG.accountKey,
+                  connectionString: BLOB_COPILOT_CONFIG.connectionString,
+                })
               } else {
-                // Fallback to generic downloadFile for other storage providers
-                fileBuffer = await downloadFile(attachment.s3_key)
+                fileBuffer = await downloadFile(attachment.key)
               }
-              const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type)
+              const fileContent = createFileContent(fileBuffer, attachment.media_type)
               if (fileContent) {
                 content.push(fileContent)
               }
@@ -399,8 +331,31 @@ export async function POST(req: NextRequest) {
       })
     }
 
+    const defaults = getCopilotModel('chat')
+    const modelToUse = env.COPILOT_MODEL || defaults.model
+
+    let providerConfig: CopilotProviderConfig | undefined
+    const providerEnv = env.COPILOT_PROVIDER as any
+
+    if (providerEnv) {
+      if (providerEnv === 'azure-openai') {
+        providerConfig = {
+          provider: 'azure-openai',
+          model: modelToUse,
+          apiKey: env.AZURE_OPENAI_API_KEY,
+          apiVersion: 'preview',
+          endpoint: env.AZURE_OPENAI_ENDPOINT,
+        }
+      } else {
+        providerConfig = {
+          provider: providerEnv,
+          model: modelToUse,
+          apiKey: env.COPILOT_API_KEY,
+        }
+      }
+    }
+
     // Determine provider and conversationId to use for this request
-    const providerToUse = provider || 'openai'
     const effectiveConversationId =
       (currentChat?.conversationId as string | undefined) || conversationId
 
@@ -416,15 +371,21 @@ export async function POST(req: NextRequest) {
       stream: stream,
       streamToolCalls: true,
       mode: mode,
-      provider: providerToUse,
+      messageId: userMessageIdToUse,
+      ...(providerConfig ? { provider: providerConfig } : {}),
       ...(effectiveConversationId ? { conversationId: effectiveConversationId } : {}),
       ...(typeof effectiveDepth === 'number' ? { depth: effectiveDepth } : {}),
       ...(typeof effectivePrefetch === 'boolean' ? { prefetch: effectivePrefetch } : {}),
       ...(session?.user?.name && { userName: session.user.name }),
-      ...(requestOrigin ? { origin: requestOrigin } : {}),
+      ...(agentContexts.length > 0 && { context: agentContexts }),
+      ...(actualChatId ? { chatId: actualChatId } : {}),
     }
 
-    // Log the payload being sent to the streaming endpoint (logs currently disabled)
+    try {
+      logger.info(`[${tracker.requestId}] About to call Sim Agent with context`, {
+        context: (requestPayload as any).context,
+      })
+    } catch {}
 
     const simAgentResponse = await fetch(`${SIM_AGENT_API_URL}/api/chat-completion-streaming`, {
       method: 'POST',
@@ -455,15 +416,18 @@ export async function POST(req: NextRequest) {
 
     // If streaming is requested, forward the stream and update chat later
     if (stream && simAgentResponse.body) {
-      // logger.info(`[${tracker.requestId}] Streaming response from sim agent`)
-
       // Create user message to save
       const userMessage = {
-        id: userMessageId || crypto.randomUUID(), // Use frontend ID if provided
+        id: userMessageIdToUse, // Consistent ID used for request and persistence
         role: 'user',
         content: message,
         timestamp: new Date().toISOString(),
         ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }),
+        ...(Array.isArray(contexts) && contexts.length > 0 && { contexts }),
+        ...(Array.isArray(contexts) &&
+          contexts.length > 0 && {
+            contentBlocks: [{ type: 'contexts', contexts: contexts as any, timestamp: Date.now() }],
+          }),
       }
 
       // Create a pass-through stream that captures the response
@@ -495,30 +459,30 @@ export async function POST(req: NextRequest) {
 
           // Start title generation in parallel if needed
           if (actualChatId && !currentChat?.title && conversationHistory.length === 0) {
-            // logger.info(`[${tracker.requestId}] Starting title generation with stream updates`, {
-            //   chatId: actualChatId,
-            //   hasTitle: !!currentChat?.title,
-            //   conversationLength: conversationHistory.length,
-            //   message: message.substring(0, 100) + (message.length > 100 ? '...' : ''),
-            // })
-            generateChatTitleAsync(actualChatId, message, tracker.requestId, controller).catch(
-              (error) => {
+            generateChatTitle(message)
+              .then(async (title) => {
+                if (title) {
+                  await db
+                    .update(copilotChats)
+                    .set({
+                      title,
+                      updatedAt: new Date(),
+                    })
+                    .where(eq(copilotChats.id, actualChatId!))
+
+                  const titleEvent = `data: ${JSON.stringify({
+                    type: 'title_updated',
+                    title: title,
+                  })}\n\n`
+                  controller.enqueue(encoder.encode(titleEvent))
+                  logger.info(`[${tracker.requestId}] Generated and saved title: ${title}`)
+                }
+              })
+              .catch((error) => {
                 logger.error(`[${tracker.requestId}] Title generation failed:`, error)
-              }
-            )
+              })
           } else {
-            // logger.debug(`[${tracker.requestId}] Skipping title generation`, {
-            //   chatId: actualChatId,
-            //   hasTitle: !!currentChat?.title,
-            //   conversationLength: conversationHistory.length,
-            //   reason: !actualChatId
-            //     ? 'no chatId'
-            //     : currentChat?.title
-            //       ? 'already has title'
-            //       : conversationHistory.length > 0
-            //         ? 'not first message'
-            //         : 'unknown',
-            // })
+            logger.debug(`[${tracker.requestId}] Skipping title generation`)
           }
 
           // Forward the sim agent stream and capture assistant response
@@ -529,24 +493,9 @@ export async function POST(req: NextRequest) {
             while (true) {
               const { done, value } = await reader.read()
               if (done) {
-                // logger.info(`[${tracker.requestId}] Stream reading completed`)
                 break
               }
 
-              // Check if client disconnected before processing chunk
-              try {
-                // Forward the chunk to client immediately
-                controller.enqueue(value)
-              } catch (error) {
-                // Client disconnected - stop reading from sim agent
-                // logger.info(
-                //   `[${tracker.requestId}] Client disconnected, stopping stream processing`
-                // )
-                reader.cancel() // Stop reading from sim agent
-                break
-              }
-              const chunkSize = value.byteLength
-
               // Decode and parse SSE events for logging and capturing content
               const decodedChunk = decoder.decode(value, { stream: true })
               buffer += decodedChunk
@@ -581,22 +530,12 @@ export async function POST(req: NextRequest) {
                         break
 
                       case 'reasoning':
-                        // Treat like thinking: do not add to assistantContent to avoid leaking
                         logger.debug(
                           `[${tracker.requestId}] Reasoning chunk received (${(event.data || event.content || '').length} chars)`
                         )
                         break
 
                       case 'tool_call':
-                        // logger.info(
-                        //   `[${tracker.requestId}] Tool call ${event.data?.partial ? '(partial)' : '(complete)'}:`,
-                        //   {
-                        //     id: event.data?.id,
-                        //     name: event.data?.name,
-                        //     arguments: event.data?.arguments,
-                        //     blockIndex: event.data?._blockIndex,
-                        //   }
-                        // )
                         if (!event.data?.partial) {
                           toolCalls.push(event.data)
                           if (event.data?.id) {
@@ -606,23 +545,12 @@ export async function POST(req: NextRequest) {
                         break
 
                       case 'tool_generating':
-                        // logger.info(`[${tracker.requestId}] Tool generating:`, {
-                        //   toolCallId: event.toolCallId,
-                        //   toolName: event.toolName,
-                        // })
                         if (event.toolCallId) {
                           startedToolExecutionIds.add(event.toolCallId)
                         }
                         break
 
                       case 'tool_result':
-                        // logger.info(`[${tracker.requestId}] Tool result received:`, {
-                        //   toolCallId: event.toolCallId,
-                        //   toolName: event.toolName,
-                        //   success: event.success,
-                        //   result: `${JSON.stringify(event.result).substring(0, 200)}...`,
-                        //   resultSize: JSON.stringify(event.result).length,
-                        // })
                         if (event.toolCallId) {
                           completedToolExecutionIds.add(event.toolCallId)
                         }
@@ -667,6 +595,47 @@ export async function POST(req: NextRequest) {
 
                       default:
                     }
+
+                    // Emit to client: rewrite 'error' events into user-friendly assistant message
+                    if (event?.type === 'error') {
+                      try {
+                        const displayMessage: string =
+                          (event?.data && (event.data.displayMessage as string)) ||
+                          'Sorry, I encountered an error. Please try again.'
+                        const formatted = `_${displayMessage}_`
+                        // Accumulate so it persists to DB as assistant content
+                        assistantContent += formatted
+                        // Send as content chunk
+                        try {
+                          controller.enqueue(
+                            encoder.encode(
+                              `data: ${JSON.stringify({ type: 'content', data: formatted })}\n\n`
+                            )
+                          )
+                        } catch (enqueueErr) {
+                          reader.cancel()
+                          break
+                        }
+                        // Then close this response cleanly for the client
+                        try {
+                          controller.enqueue(
+                            encoder.encode(`data: ${JSON.stringify({ type: 'done' })}\n\n`)
+                          )
+                        } catch (enqueueErr) {
+                          reader.cancel()
+                          break
+                        }
+                      } catch {}
+                      // Do not forward the original error event
+                    } else {
+                      // Forward original event to client
+                      try {
+                        controller.enqueue(encoder.encode(`data: ${jsonStr}\n\n`))
+                      } catch (enqueueErr) {
+                        reader.cancel()
+                        break
+                      }
+                    }
                   } catch (e) {
                     // Enhanced error handling for large payloads and parsing issues
                     const lineLength = line.length
@@ -699,10 +668,37 @@ export async function POST(req: NextRequest) {
               logger.debug(`[${tracker.requestId}] Processing remaining buffer: "${buffer}"`)
               if (buffer.startsWith('data: ')) {
                 try {
-                  const event = JSON.parse(buffer.slice(6))
+                  const jsonStr = buffer.slice(6)
+                  const event = JSON.parse(jsonStr)
                   if (event.type === 'content' && event.data) {
                     assistantContent += event.data
                   }
+                  // Forward remaining event, applying same error rewrite behavior
+                  if (event?.type === 'error') {
+                    const displayMessage: string =
+                      (event?.data && (event.data.displayMessage as string)) ||
+                      'Sorry, I encountered an error. Please try again.'
+                    const formatted = `_${displayMessage}_`
+                    assistantContent += formatted
+                    try {
+                      controller.enqueue(
+                        encoder.encode(
+                          `data: ${JSON.stringify({ type: 'content', data: formatted })}\n\n`
+                        )
+                      )
+                      controller.enqueue(
+                        encoder.encode(`data: ${JSON.stringify({ type: 'done' })}\n\n`)
+                      )
+                    } catch (enqueueErr) {
+                      reader.cancel()
+                    }
+                  } else {
+                    try {
+                      controller.enqueue(encoder.encode(`data: ${jsonStr}\n\n`))
+                    } catch (enqueueErr) {
+                      reader.cancel()
+                    }
+                  }
                 } catch (e) {
                   logger.warn(`[${tracker.requestId}] Failed to parse final buffer: "${buffer}"`)
                 }
@@ -818,11 +814,16 @@ export async function POST(req: NextRequest) {
     // Save messages if we have a chat
     if (currentChat && responseData.content) {
       const userMessage = {
-        id: userMessageId || crypto.randomUUID(), // Use frontend ID if provided
+        id: userMessageIdToUse, // Consistent ID used for request and persistence
         role: 'user',
         content: message,
         timestamp: new Date().toISOString(),
         ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }),
+        ...(Array.isArray(contexts) && contexts.length > 0 && { contexts }),
+        ...(Array.isArray(contexts) &&
+          contexts.length > 0 && {
+            contentBlocks: [{ type: 'contexts', contexts: contexts as any, timestamp: Date.now() }],
+          }),
       }
 
       const assistantMessage = {
@@ -837,9 +838,22 @@ export async function POST(req: NextRequest) {
       // Start title generation in parallel if this is first message (non-streaming)
       if (actualChatId && !currentChat.title && conversationHistory.length === 0) {
         logger.info(`[${tracker.requestId}] Starting title generation for non-streaming response`)
-        generateChatTitleAsync(actualChatId, message, tracker.requestId).catch((error) => {
-          logger.error(`[${tracker.requestId}] Title generation failed:`, error)
-        })
+        generateChatTitle(message)
+          .then(async (title) => {
+            if (title) {
+              await db
+                .update(copilotChats)
+                .set({
+                  title,
+                  updatedAt: new Date(),
+                })
+                .where(eq(copilotChats.id, actualChatId!))
+              logger.info(`[${tracker.requestId}] Generated and saved title: ${title}`)
+            }
+          })
+          .catch((error) => {
+            logger.error(`[${tracker.requestId}] Title generation failed:`, error)
+          })
       }
 
       // Update chat in database immediately (without blocking for title)

apps/sim/app/api/copilot/chat/update-messages/route.test.ts

@@ -229,7 +229,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists - override the default empty array
       const existingChat = {
         id: 'chat-123',
         userId: 'user-123',
@@ -267,7 +266,6 @@ describe('Copilot Chat Update Messages API Route', () => {
         messageCount: 2,
       })
 
-      // Verify database operations
       expect(mockSelect).toHaveBeenCalled()
       expect(mockUpdate).toHaveBeenCalled()
       expect(mockSet).toHaveBeenCalledWith({
@@ -280,7 +278,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists
       const existingChat = {
         id: 'chat-456',
         userId: 'user-123',
@@ -341,7 +338,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists
       const existingChat = {
         id: 'chat-789',
         userId: 'user-123',
@@ -374,7 +370,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock database error during chat lookup
       mockLimit.mockRejectedValueOnce(new Error('Database connection failed'))
 
       const req = createMockRequest('POST', {
@@ -401,7 +396,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists
       const existingChat = {
         id: 'chat-123',
         userId: 'user-123',
@@ -409,7 +403,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       }
       mockLimit.mockResolvedValueOnce([existingChat])
 
-      // Mock database error during update
       mockSet.mockReturnValueOnce({
         where: vi.fn().mockRejectedValue(new Error('Update operation failed')),
       })
@@ -438,7 +431,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Create a request with invalid JSON
       const req = new NextRequest('http://localhost:3000/api/copilot/chat/update-messages', {
         method: 'POST',
         body: '{invalid-json',
@@ -459,7 +451,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists
       const existingChat = {
         id: 'chat-large',
         userId: 'user-123',
@@ -467,7 +458,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       }
       mockLimit.mockResolvedValueOnce([existingChat])
 
-      // Create a large array of messages
       const messages = Array.from({ length: 100 }, (_, i) => ({
         id: `msg-${i + 1}`,
         role: i % 2 === 0 ? 'user' : 'assistant',
@@ -500,7 +490,6 @@ describe('Copilot Chat Update Messages API Route', () => {
       const authMocks = mockAuth()
       authMocks.setAuthenticated()
 
-      // Mock chat exists
       const existingChat = {
         id: 'chat-mixed',
         userId: 'user-123',

apps/sim/app/api/copilot/chat/update-messages/route.ts

@@ -28,7 +28,7 @@ const UpdateMessagesSchema = z.object({
         .array(
           z.object({
             id: z.string(),
-            s3_key: z.string(),
+            key: z.string(),
             filename: z.string(),
             media_type: z.string(),
             size: z.number(),

apps/sim/app/api/copilot/chats/route.ts

@@ -0,0 +1,39 @@
+import { desc, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import {
+  authenticateCopilotRequestSessionOnly,
+  createInternalServerErrorResponse,
+  createUnauthorizedResponse,
+} from '@/lib/copilot/auth'
+import { createLogger } from '@/lib/logs/console/logger'
+import { db } from '@/db'
+import { copilotChats } from '@/db/schema'
+
+const logger = createLogger('CopilotChatsListAPI')
+
+export async function GET(_req: NextRequest) {
+  try {
+    const { userId, isAuthenticated } = await authenticateCopilotRequestSessionOnly()
+    if (!isAuthenticated || !userId) {
+      return createUnauthorizedResponse()
+    }
+
+    const chats = await db
+      .select({
+        id: copilotChats.id,
+        title: copilotChats.title,
+        workflowId: copilotChats.workflowId,
+        updatedAt: copilotChats.updatedAt,
+      })
+      .from(copilotChats)
+      .where(eq(copilotChats.userId, userId))
+      .orderBy(desc(copilotChats.updatedAt))
+
+    logger.info(`Retrieved ${chats.length} chats for user ${userId}`)
+
+    return NextResponse.json({ success: true, chats })
+  } catch (error) {
+    logger.error('Error fetching user copilot chats:', error)
+    return createInternalServerErrorResponse('Failed to fetch user chats')
+  }
+}

apps/sim/app/api/copilot/stats/route.ts

@@ -0,0 +1,68 @@
+import { type NextRequest, NextResponse } from 'next/server'
+import { z } from 'zod'
+import {
+  authenticateCopilotRequestSessionOnly,
+  createBadRequestResponse,
+  createInternalServerErrorResponse,
+  createRequestTracker,
+  createUnauthorizedResponse,
+} from '@/lib/copilot/auth'
+import { env } from '@/lib/env'
+import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
+
+const SIM_AGENT_API_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
+
+const BodySchema = z.object({
+  messageId: z.string(),
+  diffCreated: z.boolean(),
+  diffAccepted: z.boolean(),
+})
+
+export async function POST(req: NextRequest) {
+  const tracker = createRequestTracker()
+  try {
+    const { userId, isAuthenticated } = await authenticateCopilotRequestSessionOnly()
+    if (!isAuthenticated || !userId) {
+      return createUnauthorizedResponse()
+    }
+
+    const json = await req.json().catch(() => ({}))
+    const parsed = BodySchema.safeParse(json)
+    if (!parsed.success) {
+      return createBadRequestResponse('Invalid request body for copilot stats')
+    }
+
+    const { messageId, diffCreated, diffAccepted } = parsed.data as any
+
+    // Build outgoing payload for Sim Agent with only required fields
+    const payload: Record<string, any> = {
+      messageId,
+      diffCreated,
+      diffAccepted,
+    }
+
+    const agentRes = await fetch(`${SIM_AGENT_API_URL}/api/stats`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
+      },
+      body: JSON.stringify(payload),
+    })
+
+    // Prefer not to block clients; still relay status
+    let agentJson: any = null
+    try {
+      agentJson = await agentRes.json()
+    } catch {}
+
+    if (!agentRes.ok) {
+      const message = (agentJson && (agentJson.error || agentJson.message)) || 'Upstream error'
+      return NextResponse.json({ success: false, error: message }, { status: 400 })
+    }
+
+    return NextResponse.json({ success: true })
+  } catch (error) {
+    return createInternalServerErrorResponse('Failed to forward copilot stats')
+  }
+}

apps/sim/app/api/environment/route.ts

@@ -3,20 +3,19 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
-import { decryptSecret, encryptSecret } from '@/lib/utils'
+import { decryptSecret, encryptSecret, generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { environment } from '@/db/schema'
 import type { EnvironmentVariable } from '@/stores/settings/environment/types'
 
 const logger = createLogger('EnvironmentAPI')
 
-// Schema for environment variable updates
 const EnvVarSchema = z.object({
   variables: z.record(z.string()),
 })
 
 export async function POST(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const session = await getSession()
@@ -30,17 +29,13 @@ export async function POST(req: NextRequest) {
     try {
       const { variables } = EnvVarSchema.parse(body)
 
-      // Encrypt all variables
-      const encryptedVariables = await Object.entries(variables).reduce(
-        async (accPromise, [key, value]) => {
-          const acc = await accPromise
+      const encryptedVariables = await Promise.all(
+        Object.entries(variables).map(async ([key, value]) => {
           const { encrypted } = await encryptSecret(value)
-          return { ...acc, [key]: encrypted }
-        },
-        Promise.resolve({})
-      )
+          return [key, encrypted] as const
+        })
+      ).then((entries) => Object.fromEntries(entries))
 
-      // Replace all environment variables for user
       await db
         .insert(environment)
         .values({
@@ -77,10 +72,9 @@ export async function POST(req: NextRequest) {
 }
 
 export async function GET(request: Request) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
-    // Get the session directly in the API route
     const session = await getSession()
     if (!session?.user?.id) {
       logger.warn(`[${requestId}] Unauthorized environment variables access attempt`)
@@ -99,18 +93,15 @@ export async function GET(request: Request) {
       return NextResponse.json({ data: {} }, { status: 200 })
     }
 
-    // Decrypt the variables for client-side use
     const encryptedVariables = result[0].variables as Record<string, string>
     const decryptedVariables: Record<string, EnvironmentVariable> = {}
 
-    // Decrypt each variable
     for (const [key, encryptedValue] of Object.entries(encryptedVariables)) {
       try {
         const { decrypted } = await decryptSecret(encryptedValue)
         decryptedVariables[key] = { key, value: decrypted }
       } catch (error) {
         logger.error(`[${requestId}] Error decrypting variable ${key}`, error)
-        // If decryption fails, provide a placeholder
         decryptedVariables[key] = { key, value: '' }
       }
     }

apps/sim/app/api/environment/variables/route.ts

@@ -1,223 +0,0 @@
-import { eq } from 'drizzle-orm'
-import { type NextRequest, NextResponse } from 'next/server'
-import { z } from 'zod'
-import { getEnvironmentVariableKeys } from '@/lib/environment/utils'
-import { createLogger } from '@/lib/logs/console/logger'
-import { decryptSecret, encryptSecret } from '@/lib/utils'
-import { getUserId } from '@/app/api/auth/oauth/utils'
-import { db } from '@/db'
-import { environment } from '@/db/schema'
-
-const logger = createLogger('EnvironmentVariablesAPI')
-
-// Schema for environment variable updates
-const EnvVarSchema = z.object({
-  variables: z.record(z.string()),
-})
-
-export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
-
-  try {
-    // For GET requests, check for workflowId in query params
-    const { searchParams } = new URL(request.url)
-    const workflowId = searchParams.get('workflowId')
-
-    // Use dual authentication pattern like other copilot tools
-    const userId = await getUserId(requestId, workflowId || undefined)
-
-    if (!userId) {
-      logger.warn(`[${requestId}] Unauthorized environment variables access attempt`)
-      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
-    }
-
-    // Get only the variable names (keys), not values
-    const result = await getEnvironmentVariableKeys(userId)
-
-    return NextResponse.json(
-      {
-        success: true,
-        output: result,
-      },
-      { status: 200 }
-    )
-  } catch (error: any) {
-    logger.error(`[${requestId}] Environment variables fetch error`, error)
-    return NextResponse.json(
-      {
-        success: false,
-        error: error.message || 'Failed to get environment variables',
-      },
-      { status: 500 }
-    )
-  }
-}
-
-export async function PUT(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
-
-  try {
-    const body = await request.json()
-    const { workflowId, variables } = body
-
-    // Use dual authentication pattern like other copilot tools
-    const userId = await getUserId(requestId, workflowId)
-
-    if (!userId) {
-      logger.warn(`[${requestId}] Unauthorized environment variables set attempt`)
-      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
-    }
-
-    try {
-      const { variables: validatedVariables } = EnvVarSchema.parse({ variables })
-
-      // Get existing environment variables for this user
-      const existingData = await db
-        .select()
-        .from(environment)
-        .where(eq(environment.userId, userId))
-        .limit(1)
-
-      // Start with existing encrypted variables or empty object
-      const existingEncryptedVariables =
-        (existingData[0]?.variables as Record<string, string>) || {}
-
-      // Determine which variables are new or changed by comparing with decrypted existing values
-      const variablesToEncrypt: Record<string, string> = {}
-      const addedVariables: string[] = []
-      const updatedVariables: string[] = []
-
-      for (const [key, newValue] of Object.entries(validatedVariables)) {
-        if (!(key in existingEncryptedVariables)) {
-          // New variable
-          variablesToEncrypt[key] = newValue
-          addedVariables.push(key)
-        } else {
-          // Check if the value has actually changed by decrypting the existing value
-          try {
-            const { decrypted: existingValue } = await decryptSecret(
-              existingEncryptedVariables[key]
-            )
-
-            if (existingValue !== newValue) {
-              // Value changed, needs re-encryption
-              variablesToEncrypt[key] = newValue
-              updatedVariables.push(key)
-            }
-            // If values are the same, keep the existing encrypted value
-          } catch (decryptError) {
-            // If we can't decrypt the existing value, treat as changed and re-encrypt
-            logger.warn(
-              `[${requestId}] Could not decrypt existing variable ${key}, re-encrypting`,
-              { error: decryptError }
-            )
-            variablesToEncrypt[key] = newValue
-            updatedVariables.push(key)
-          }
-        }
-      }
-
-      // Only encrypt the variables that are new or changed
-      const newlyEncryptedVariables = await Object.entries(variablesToEncrypt).reduce(
-        async (accPromise, [key, value]) => {
-          const acc = await accPromise
-          const { encrypted } = await encryptSecret(value)
-          return { ...acc, [key]: encrypted }
-        },
-        Promise.resolve({})
-      )
-
-      // Merge existing encrypted variables with newly encrypted ones
-      const finalEncryptedVariables = { ...existingEncryptedVariables, ...newlyEncryptedVariables }
-
-      // Update or insert environment variables for user
-      await db
-        .insert(environment)
-        .values({
-          id: crypto.randomUUID(),
-          userId: userId,
-          variables: finalEncryptedVariables,
-          updatedAt: new Date(),
-        })
-        .onConflictDoUpdate({
-          target: [environment.userId],
-          set: {
-            variables: finalEncryptedVariables,
-            updatedAt: new Date(),
-          },
-        })
-
-      return NextResponse.json(
-        {
-          success: true,
-          output: {
-            message: `Successfully processed ${Object.keys(validatedVariables).length} environment variable(s): ${addedVariables.length} added, ${updatedVariables.length} updated`,
-            variableCount: Object.keys(validatedVariables).length,
-            variableNames: Object.keys(validatedVariables),
-            totalVariableCount: Object.keys(finalEncryptedVariables).length,
-            addedVariables,
-            updatedVariables,
-          },
-        },
-        { status: 200 }
-      )
-    } catch (validationError) {
-      if (validationError instanceof z.ZodError) {
-        logger.warn(`[${requestId}] Invalid environment variables data`, {
-          errors: validationError.errors,
-        })
-        return NextResponse.json(
-          { error: 'Invalid request data', details: validationError.errors },
-          { status: 400 }
-        )
-      }
-      throw validationError
-    }
-  } catch (error: any) {
-    logger.error(`[${requestId}] Environment variables set error`, error)
-    return NextResponse.json(
-      {
-        success: false,
-        error: error.message || 'Failed to set environment variables',
-      },
-      { status: 500 }
-    )
-  }
-}
-
-export async function POST(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
-
-  try {
-    const body = await request.json()
-    const { workflowId } = body
-
-    // Use dual authentication pattern like other copilot tools
-    const userId = await getUserId(requestId, workflowId)
-
-    if (!userId) {
-      logger.warn(`[${requestId}] Unauthorized environment variables access attempt`)
-      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
-    }
-
-    // Get only the variable names (keys), not values
-    const result = await getEnvironmentVariableKeys(userId)
-
-    return NextResponse.json(
-      {
-        success: true,
-        output: result,
-      },
-      { status: 200 }
-    )
-  } catch (error: any) {
-    logger.error(`[${requestId}] Environment variables fetch error`, error)
-    return NextResponse.json(
-      {
-        success: false,
-        error: error.message || 'Failed to get environment variables',
-      },
-      { status: 500 }
-    )
-  }
-}

apps/sim/app/api/files/multipart/route.ts

@@ -1,16 +1,8 @@
-import {
-  AbortMultipartUploadCommand,
-  CompleteMultipartUploadCommand,
-  CreateMultipartUploadCommand,
-  UploadPartCommand,
-} from '@aws-sdk/client-s3'
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import { type NextRequest, NextResponse } from 'next/server'
-import { v4 as uuidv4 } from 'uuid'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
-import { S3_KB_CONFIG } from '@/lib/uploads/setup'
+import { BLOB_KB_CONFIG } from '@/lib/uploads/setup'
 
 const logger = createLogger('MultipartUploadAPI')
 
@@ -26,15 +18,6 @@ interface GetPartUrlsRequest {
   partNumbers: number[]
 }
 
-interface CompleteMultipartRequest {
-  uploadId: string
-  key: string
-  parts: Array<{
-    ETag: string
-    PartNumber: number
-  }>
-}
-
 export async function POST(request: NextRequest) {
   try {
     const session = await getSession()
@@ -44,106 +27,214 @@ export async function POST(request: NextRequest) {
 
     const action = request.nextUrl.searchParams.get('action')
 
-    if (!isUsingCloudStorage() || getStorageProvider() !== 's3') {
+    if (!isUsingCloudStorage()) {
       return NextResponse.json(
-        { error: 'Multipart upload is only available with S3 storage' },
+        { error: 'Multipart upload is only available with cloud storage (S3 or Azure Blob)' },
         { status: 400 }
       )
     }
 
-    const { getS3Client } = await import('@/lib/uploads/s3/s3-client')
-    const s3Client = getS3Client()
+    const storageProvider = getStorageProvider()
 
     switch (action) {
       case 'initiate': {
         const data: InitiateMultipartRequest = await request.json()
-        const { fileName, contentType } = data
+        const { fileName, contentType, fileSize } = data
 
-        const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_')
-        const uniqueKey = `kb/${uuidv4()}-${safeFileName}`
+        if (storageProvider === 's3') {
+          const { initiateS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client')
 
-        const command = new CreateMultipartUploadCommand({
-          Bucket: S3_KB_CONFIG.bucket,
-          Key: uniqueKey,
-          ContentType: contentType,
-          Metadata: {
-            originalName: fileName,
-            uploadedAt: new Date().toISOString(),
-            purpose: 'knowledge-base',
-          },
-        })
+          const result = await initiateS3MultipartUpload({
+            fileName,
+            contentType,
+            fileSize,
+          })
 
-        const response = await s3Client.send(command)
+          logger.info(`Initiated S3 multipart upload for ${fileName}: ${result.uploadId}`)
 
-        logger.info(`Initiated multipart upload for ${fileName}: ${response.UploadId}`)
+          return NextResponse.json({
+            uploadId: result.uploadId,
+            key: result.key,
+          })
+        }
+        if (storageProvider === 'blob') {
+          const { initiateMultipartUpload } = await import('@/lib/uploads/blob/blob-client')
 
-        return NextResponse.json({
-          uploadId: response.UploadId,
-          key: uniqueKey,
-        })
+          const result = await initiateMultipartUpload({
+            fileName,
+            contentType,
+            fileSize,
+            customConfig: {
+              containerName: BLOB_KB_CONFIG.containerName,
+              accountName: BLOB_KB_CONFIG.accountName,
+              accountKey: BLOB_KB_CONFIG.accountKey,
+              connectionString: BLOB_KB_CONFIG.connectionString,
+            },
+          })
+
+          logger.info(`Initiated Azure multipart upload for ${fileName}: ${result.uploadId}`)
+
+          return NextResponse.json({
+            uploadId: result.uploadId,
+            key: result.key,
+          })
+        }
+
+        return NextResponse.json(
+          { error: `Unsupported storage provider: ${storageProvider}` },
+          { status: 400 }
+        )
       }
 
       case 'get-part-urls': {
         const data: GetPartUrlsRequest = await request.json()
         const { uploadId, key, partNumbers } = data
 
-        const presignedUrls = await Promise.all(
-          partNumbers.map(async (partNumber) => {
-            const command = new UploadPartCommand({
-              Bucket: S3_KB_CONFIG.bucket,
-              Key: key,
-              PartNumber: partNumber,
-              UploadId: uploadId,
-            })
+        if (storageProvider === 's3') {
+          const { getS3MultipartPartUrls } = await import('@/lib/uploads/s3/s3-client')
 
-            const url = await getSignedUrl(s3Client, command, { expiresIn: 3600 })
-            return { partNumber, url }
+          const presignedUrls = await getS3MultipartPartUrls(key, uploadId, partNumbers)
+
+          return NextResponse.json({ presignedUrls })
+        }
+        if (storageProvider === 'blob') {
+          const { getMultipartPartUrls } = await import('@/lib/uploads/blob/blob-client')
+
+          const presignedUrls = await getMultipartPartUrls(key, uploadId, partNumbers, {
+            containerName: BLOB_KB_CONFIG.containerName,
+            accountName: BLOB_KB_CONFIG.accountName,
+            accountKey: BLOB_KB_CONFIG.accountKey,
+            connectionString: BLOB_KB_CONFIG.connectionString,
           })
-        )
 
-        return NextResponse.json({ presignedUrls })
+          return NextResponse.json({ presignedUrls })
+        }
+
+        return NextResponse.json(
+          { error: `Unsupported storage provider: ${storageProvider}` },
+          { status: 400 }
+        )
       }
 
       case 'complete': {
-        const data: CompleteMultipartRequest = await request.json()
+        const data = await request.json()
+
+        // Handle batch completion
+        if ('uploads' in data) {
+          const results = await Promise.all(
+            data.uploads.map(async (upload: any) => {
+              const { uploadId, key } = upload
+
+              if (storageProvider === 's3') {
+                const { completeS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client')
+                const parts = upload.parts // S3 format: { ETag, PartNumber }
+
+                const result = await completeS3MultipartUpload(key, uploadId, parts)
+
+                return {
+                  success: true,
+                  location: result.location,
+                  path: result.path,
+                  key: result.key,
+                }
+              }
+              if (storageProvider === 'blob') {
+                const { completeMultipartUpload } = await import('@/lib/uploads/blob/blob-client')
+                const parts = upload.parts // Azure format: { blockId, partNumber }
+
+                const result = await completeMultipartUpload(key, uploadId, parts, {
+                  containerName: BLOB_KB_CONFIG.containerName,
+                  accountName: BLOB_KB_CONFIG.accountName,
+                  accountKey: BLOB_KB_CONFIG.accountKey,
+                  connectionString: BLOB_KB_CONFIG.connectionString,
+                })
+
+                return {
+                  success: true,
+                  location: result.location,
+                  path: result.path,
+                  key: result.key,
+                }
+              }
+
+              throw new Error(`Unsupported storage provider: ${storageProvider}`)
+            })
+          )
+
+          logger.info(`Completed ${data.uploads.length} multipart uploads`)
+          return NextResponse.json({ results })
+        }
+
+        // Handle single completion
         const { uploadId, key, parts } = data
 
-        const command = new CompleteMultipartUploadCommand({
-          Bucket: S3_KB_CONFIG.bucket,
-          Key: key,
-          UploadId: uploadId,
-          MultipartUpload: {
-            Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
-          },
-        })
+        if (storageProvider === 's3') {
+          const { completeS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client')
 
-        const response = await s3Client.send(command)
+          const result = await completeS3MultipartUpload(key, uploadId, parts)
 
-        logger.info(`Completed multipart upload for key ${key}`)
+          logger.info(`Completed S3 multipart upload for key ${key}`)
 
-        const finalPath = `/api/files/serve/s3/${encodeURIComponent(key)}`
+          return NextResponse.json({
+            success: true,
+            location: result.location,
+            path: result.path,
+            key: result.key,
+          })
+        }
+        if (storageProvider === 'blob') {
+          const { completeMultipartUpload } = await import('@/lib/uploads/blob/blob-client')
 
-        return NextResponse.json({
-          success: true,
-          location: response.Location,
-          path: finalPath,
-          key,
-        })
+          const result = await completeMultipartUpload(key, uploadId, parts, {
+            containerName: BLOB_KB_CONFIG.containerName,
+            accountName: BLOB_KB_CONFIG.accountName,
+            accountKey: BLOB_KB_CONFIG.accountKey,
+            connectionString: BLOB_KB_CONFIG.connectionString,
+          })
+
+          logger.info(`Completed Azure multipart upload for key ${key}`)
+
+          return NextResponse.json({
+            success: true,
+            location: result.location,
+            path: result.path,
+            key: result.key,
+          })
+        }
+
+        return NextResponse.json(
+          { error: `Unsupported storage provider: ${storageProvider}` },
+          { status: 400 }
+        )
       }
 
       case 'abort': {
         const data = await request.json()
         const { uploadId, key } = data
 
-        const command = new AbortMultipartUploadCommand({
-          Bucket: S3_KB_CONFIG.bucket,
-          Key: key,
-          UploadId: uploadId,
-        })
+        if (storageProvider === 's3') {
+          const { abortS3MultipartUpload } = await import('@/lib/uploads/s3/s3-client')
 
-        await s3Client.send(command)
+          await abortS3MultipartUpload(key, uploadId)
 
-        logger.info(`Aborted multipart upload for key ${key}`)
+          logger.info(`Aborted S3 multipart upload for key ${key}`)
+        } else if (storageProvider === 'blob') {
+          const { abortMultipartUpload } = await import('@/lib/uploads/blob/blob-client')
+
+          await abortMultipartUpload(key, uploadId, {
+            containerName: BLOB_KB_CONFIG.containerName,
+            accountName: BLOB_KB_CONFIG.accountName,
+            accountKey: BLOB_KB_CONFIG.accountKey,
+            connectionString: BLOB_KB_CONFIG.connectionString,
+          })
+
+          logger.info(`Aborted Azure multipart upload for key ${key}`)
+        } else {
+          return NextResponse.json(
+            { error: `Unsupported storage provider: ${storageProvider}` },
+            { status: 400 }
+          )
+        }
 
         return NextResponse.json({ success: true })
       }

apps/sim/app/api/files/parse/route.ts

@@ -76,11 +76,9 @@ export async function POST(request: NextRequest) {
 
     logger.info('File parse request received:', { filePath, fileType })
 
-    // Handle multiple files
     if (Array.isArray(filePath)) {
       const results = []
       for (const path of filePath) {
-        // Skip empty or invalid paths
         if (!path || (typeof path === 'string' && path.trim() === '')) {
           results.push({
             success: false,
@@ -91,12 +89,10 @@ export async function POST(request: NextRequest) {
         }
 
         const result = await parseFileSingle(path, fileType)
-        // Add processing time to metadata
         if (result.metadata) {
           result.metadata.processingTime = Date.now() - startTime
         }
 
-        // Transform each result to match expected frontend format
         if (result.success) {
           results.push({
             success: true,
@@ -105,7 +101,7 @@ export async function POST(request: NextRequest) {
               name: result.filePath.split('/').pop() || 'unknown',
               fileType: result.metadata?.fileType || 'application/octet-stream',
               size: result.metadata?.size || 0,
-              binary: false, // We only return text content
+              binary: false,
             },
             filePath: result.filePath,
           })
@@ -120,15 +116,12 @@ export async function POST(request: NextRequest) {
       })
     }
 
-    // Handle single file
     const result = await parseFileSingle(filePath, fileType)
 
-    // Add processing time to metadata
     if (result.metadata) {
       result.metadata.processingTime = Date.now() - startTime
     }
 
-    // Transform single file result to match expected frontend format
     if (result.success) {
       return NextResponse.json({
         success: true,
@@ -142,8 +135,6 @@ export async function POST(request: NextRequest) {
       })
     }
 
-    // Only return 500 for actual server errors, not file processing failures
-    // File processing failures (like file not found, parsing errors) should return 200 with success:false
     return NextResponse.json(result)
   } catch (error) {
     logger.error('Error in file parse API:', error)
@@ -164,7 +155,6 @@ export async function POST(request: NextRequest) {
 async function parseFileSingle(filePath: string, fileType?: string): Promise<ParseResult> {
   logger.info('Parsing file:', filePath)
 
-  // Validate that filePath is not empty
   if (!filePath || filePath.trim() === '') {
     return {
       success: false,
@@ -173,7 +163,6 @@ async function parseFileSingle(filePath: string, fileType?: string): Promise<Par
     }
   }
 
-  // Validate path for security before any processing
   const pathValidation = validateFilePath(filePath)
   if (!pathValidation.isValid) {
     return {
@@ -183,49 +172,40 @@ async function parseFileSingle(filePath: string, fileType?: string): Promise<Par
     }
   }
 
-  // Check if this is an external URL
   if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
     return handleExternalUrl(filePath, fileType)
   }
 
-  // Check if this is a cloud storage path (S3 or Blob)
   const isS3Path = filePath.includes('/api/files/serve/s3/')
   const isBlobPath = filePath.includes('/api/files/serve/blob/')
 
-  // Use cloud handler if it's a cloud path or we're in cloud mode
   if (isS3Path || isBlobPath || isUsingCloudStorage()) {
     return handleCloudFile(filePath, fileType)
   }
 
-  // Use local handler for local files
   return handleLocalFile(filePath, fileType)
 }
 
 /**
- * Validate file path for security
+ * Validate file path for security - prevents null byte injection and path traversal attacks
  */
 function validateFilePath(filePath: string): { isValid: boolean; error?: string } {
-  // Check for null bytes
   if (filePath.includes('\0')) {
     return { isValid: false, error: 'Invalid path: null byte detected' }
   }
 
-  // Check for path traversal attempts
   if (filePath.includes('..')) {
     return { isValid: false, error: 'Access denied: path traversal detected' }
   }
 
-  // Check for tilde characters (home directory access)
   if (filePath.includes('~')) {
     return { isValid: false, error: 'Invalid path: tilde character not allowed' }
   }
 
-  // Check for absolute paths outside allowed directories
   if (filePath.startsWith('/') && !filePath.startsWith('/api/files/serve/')) {
     return { isValid: false, error: 'Path outside allowed directory' }
   }
 
-  // Check for Windows absolute paths
   if (/^[A-Za-z]:\\/.test(filePath)) {
     return { isValid: false, error: 'Path outside allowed directory' }
   }
@@ -260,12 +240,10 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
 
     logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`)
 
-    // Extract filename from URL
     const urlPath = new URL(url).pathname
     const filename = urlPath.split('/').pop() || 'download'
     const extension = path.extname(filename).toLowerCase().substring(1)
 
-    // Process the file based on its content type
     if (extension === 'pdf') {
       return await handlePdfBuffer(buffer, filename, fileType, url)
     }
@@ -276,7 +254,6 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
       return await handleGenericTextBuffer(buffer, filename, extension, fileType, url)
     }
 
-    // For binary or unknown files
     return handleGenericBuffer(buffer, filename, extension, fileType)
   } catch (error) {
     logger.error(`Error handling external URL ${url}:`, error)
@@ -289,35 +266,29 @@ async function handleExternalUrl(url: string, fileType?: string): Promise<ParseR
 }
 
 /**
- * Handle file stored in cloud storage (S3 or Azure Blob)
+ * Handle file stored in cloud storage
  */
 async function handleCloudFile(filePath: string, fileType?: string): Promise<ParseResult> {
   try {
-    // Extract the cloud key from the path
     let cloudKey: string
     if (filePath.includes('/api/files/serve/s3/')) {
       cloudKey = decodeURIComponent(filePath.split('/api/files/serve/s3/')[1])
     } else if (filePath.includes('/api/files/serve/blob/')) {
       cloudKey = decodeURIComponent(filePath.split('/api/files/serve/blob/')[1])
     } else if (filePath.startsWith('/api/files/serve/')) {
-      // Backwards-compatibility: path like "/api/files/serve/<key>"
       cloudKey = decodeURIComponent(filePath.substring('/api/files/serve/'.length))
     } else {
-      // Assume raw key provided
       cloudKey = filePath
     }
 
     logger.info('Extracted cloud key:', cloudKey)
 
-    // Download the file from cloud storage - this can throw for access errors
     const fileBuffer = await downloadFile(cloudKey)
     logger.info(`Downloaded file from cloud storage: ${cloudKey}, size: ${fileBuffer.length} bytes`)
 
-    // Extract the filename from the cloud key
     const filename = cloudKey.split('/').pop() || cloudKey
     const extension = path.extname(filename).toLowerCase().substring(1)
 
-    // Process the file based on its content type
     if (extension === 'pdf') {
       return await handlePdfBuffer(fileBuffer, filename, fileType, filePath)
     }
@@ -325,22 +296,19 @@ async function handleCloudFile(filePath: string, fileType?: string): Promise<Par
       return await handleCsvBuffer(fileBuffer, filename, fileType, filePath)
     }
     if (isSupportedFileType(extension)) {
-      // For other supported types that we have parsers for
       return await handleGenericTextBuffer(fileBuffer, filename, extension, fileType, filePath)
     }
-    // For binary or unknown files
     return handleGenericBuffer(fileBuffer, filename, extension, fileType)
   } catch (error) {
     logger.error(`Error handling cloud file ${filePath}:`, error)
 
-    // Check if this is a download/access error that should trigger a 500 response
+    // For download/access errors, throw to trigger 500 response
     const errorMessage = (error as Error).message
     if (errorMessage.includes('Access denied') || errorMessage.includes('Forbidden')) {
-      // For access errors, throw to trigger 500 response
       throw new Error(`Error accessing file from cloud storage: ${errorMessage}`)
     }
 
-    // For other errors (parsing, processing), return success:false
+    // For other errors (parsing, processing), return success:false and an error message
     return {
       success: false,
       error: `Error accessing file from cloud storage: ${errorMessage}`,
@@ -354,28 +322,23 @@ async function handleCloudFile(filePath: string, fileType?: string): Promise<Par
  */
 async function handleLocalFile(filePath: string, fileType?: string): Promise<ParseResult> {
   try {
-    // Extract filename from path
     const filename = filePath.split('/').pop() || filePath
     const fullPath = path.join(UPLOAD_DIR_SERVER, filename)
 
     logger.info('Processing local file:', fullPath)
 
-    // Check if file exists
     try {
       await fsPromises.access(fullPath)
     } catch {
       throw new Error(`File not found: ${filename}`)
     }
 
-    // Parse the file directly
     const result = await parseFile(fullPath)
 
-    // Get file stats for metadata
     const stats = await fsPromises.stat(fullPath)
     const fileBuffer = await readFile(fullPath)
     const hash = createHash('md5').update(fileBuffer).digest('hex')
 
-    // Extract file extension for type detection
     const extension = path.extname(filename).toLowerCase().substring(1)
 
     return {
@@ -386,7 +349,7 @@ async function handleLocalFile(filePath: string, fileType?: string): Promise<Par
         fileType: fileType || getMimeType(extension),
         size: stats.size,
         hash,
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   } catch (error) {
@@ -425,15 +388,14 @@ async function handlePdfBuffer(
         fileType: fileType || 'application/pdf',
         size: fileBuffer.length,
         hash: createHash('md5').update(fileBuffer).digest('hex'),
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   } catch (error) {
     logger.error('Failed to parse PDF in memory:', error)
 
-    // Create fallback message for PDF parsing failure
     const content = createPdfFailureMessage(
-      0, // We can't determine page count without parsing
+      0,
       fileBuffer.length,
       originalPath || filename,
       (error as Error).message
@@ -447,7 +409,7 @@ async function handlePdfBuffer(
         fileType: fileType || 'application/pdf',
         size: fileBuffer.length,
         hash: createHash('md5').update(fileBuffer).digest('hex'),
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   }
@@ -465,7 +427,6 @@ async function handleCsvBuffer(
   try {
     logger.info(`Parsing CSV in memory: ${filename}`)
 
-    // Use the parseBuffer function from our library
     const { parseBuffer } = await import('@/lib/file-parsers')
     const result = await parseBuffer(fileBuffer, 'csv')
 
@@ -477,7 +438,7 @@ async function handleCsvBuffer(
         fileType: fileType || 'text/csv',
         size: fileBuffer.length,
         hash: createHash('md5').update(fileBuffer).digest('hex'),
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   } catch (error) {
@@ -490,7 +451,7 @@ async function handleCsvBuffer(
         fileType: 'text/csv',
         size: 0,
         hash: '',
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   }
@@ -509,7 +470,6 @@ async function handleGenericTextBuffer(
   try {
     logger.info(`Parsing text file in memory: ${filename}`)
 
-    // Try to use a specialized parser if available
     try {
       const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers')
 
@@ -524,7 +484,7 @@ async function handleGenericTextBuffer(
             fileType: fileType || getMimeType(extension),
             size: fileBuffer.length,
             hash: createHash('md5').update(fileBuffer).digest('hex'),
-            processingTime: 0, // Will be set by caller
+            processingTime: 0,
           },
         }
       }
@@ -532,7 +492,6 @@ async function handleGenericTextBuffer(
       logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
     }
 
-    // Fallback to generic text parsing
     const content = fileBuffer.toString('utf-8')
 
     return {
@@ -543,7 +502,7 @@ async function handleGenericTextBuffer(
         fileType: fileType || getMimeType(extension),
         size: fileBuffer.length,
         hash: createHash('md5').update(fileBuffer).digest('hex'),
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   } catch (error) {
@@ -556,7 +515,7 @@ async function handleGenericTextBuffer(
         fileType: 'text/plain',
         size: 0,
         hash: '',
-        processingTime: 0, // Will be set by caller
+        processingTime: 0,
       },
     }
   }
@@ -584,7 +543,7 @@ function handleGenericBuffer(
       fileType: fileType || getMimeType(extension),
       size: fileBuffer.length,
       hash: createHash('md5').update(fileBuffer).digest('hex'),
-      processingTime: 0, // Will be set by caller
+      processingTime: 0,
     },
   }
 }
@@ -594,25 +553,11 @@ function handleGenericBuffer(
  */
 async function parseBufferAsPdf(buffer: Buffer) {
   try {
-    // Import parsers dynamically to avoid initialization issues in tests
-    // First try to use the main PDF parser
-    try {
-      const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
-      const parser = new PdfParser()
-      logger.info('Using main PDF parser for buffer')
+    const { PdfParser } = await import('@/lib/file-parsers/pdf-parser')
+    const parser = new PdfParser()
+    logger.info('Using main PDF parser for buffer')
 
-      if (parser.parseBuffer) {
-        return await parser.parseBuffer(buffer)
-      }
-      throw new Error('PDF parser does not support buffer parsing')
-    } catch (error) {
-      // Fallback to raw PDF parser
-      logger.warn('Main PDF parser failed, using raw parser for buffer:', error)
-      const { RawPdfParser } = await import('@/lib/file-parsers/raw-pdf-parser')
-      const rawParser = new RawPdfParser()
-
-      return await rawParser.parseBuffer(buffer)
-    }
+    return await parser.parseBuffer(buffer)
   } catch (error) {
     throw new Error(`PDF parsing failed: ${(error as Error).message}`)
   }
@@ -655,7 +600,7 @@ Please use a PDF viewer for best results.`
 }
 
 /**
- * Create error message for PDF parsing failure
+ * Create error message for PDF parsing failure and make it more readable
  */
 function createPdfFailureMessage(
   pageCount: number,

apps/sim/app/api/files/presigned/batch/route.ts

@@ -0,0 +1,361 @@
+import { PutObjectCommand } from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { type NextRequest, NextResponse } from 'next/server'
+import { v4 as uuidv4 } from 'uuid'
+import { getSession } from '@/lib/auth'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
+import {
+  BLOB_CHAT_CONFIG,
+  BLOB_CONFIG,
+  BLOB_COPILOT_CONFIG,
+  BLOB_KB_CONFIG,
+  S3_CHAT_CONFIG,
+  S3_CONFIG,
+  S3_COPILOT_CONFIG,
+  S3_KB_CONFIG,
+} from '@/lib/uploads/setup'
+import { validateFileType } from '@/lib/uploads/validation'
+import { createErrorResponse, createOptionsResponse } from '@/app/api/files/utils'
+
+const logger = createLogger('BatchPresignedUploadAPI')
+
+interface BatchFileRequest {
+  fileName: string
+  contentType: string
+  fileSize: number
+}
+
+interface BatchPresignedUrlRequest {
+  files: BatchFileRequest[]
+}
+
+type UploadType = 'general' | 'knowledge-base' | 'chat' | 'copilot'
+
+export async function POST(request: NextRequest) {
+  try {
+    const session = await getSession()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    let data: BatchPresignedUrlRequest
+    try {
+      data = await request.json()
+    } catch {
+      return NextResponse.json({ error: 'Invalid JSON in request body' }, { status: 400 })
+    }
+
+    const { files } = data
+
+    if (!files || !Array.isArray(files) || files.length === 0) {
+      return NextResponse.json(
+        { error: 'files array is required and cannot be empty' },
+        { status: 400 }
+      )
+    }
+
+    if (files.length > 100) {
+      return NextResponse.json(
+        { error: 'Cannot process more than 100 files at once' },
+        { status: 400 }
+      )
+    }
+
+    const uploadTypeParam = request.nextUrl.searchParams.get('type')
+    const uploadType: UploadType =
+      uploadTypeParam === 'knowledge-base'
+        ? 'knowledge-base'
+        : uploadTypeParam === 'chat'
+          ? 'chat'
+          : uploadTypeParam === 'copilot'
+            ? 'copilot'
+            : 'general'
+
+    const MAX_FILE_SIZE = 100 * 1024 * 1024
+    for (const file of files) {
+      if (!file.fileName?.trim()) {
+        return NextResponse.json({ error: 'fileName is required for all files' }, { status: 400 })
+      }
+      if (!file.contentType?.trim()) {
+        return NextResponse.json(
+          { error: 'contentType is required for all files' },
+          { status: 400 }
+        )
+      }
+      if (!file.fileSize || file.fileSize <= 0) {
+        return NextResponse.json(
+          { error: 'fileSize must be positive for all files' },
+          { status: 400 }
+        )
+      }
+      if (file.fileSize > MAX_FILE_SIZE) {
+        return NextResponse.json(
+          { error: `File ${file.fileName} exceeds maximum size of ${MAX_FILE_SIZE} bytes` },
+          { status: 400 }
+        )
+      }
+
+      if (uploadType === 'knowledge-base') {
+        const fileValidationError = validateFileType(file.fileName, file.contentType)
+        if (fileValidationError) {
+          return NextResponse.json(
+            {
+              error: fileValidationError.message,
+              code: fileValidationError.code,
+              supportedTypes: fileValidationError.supportedTypes,
+            },
+            { status: 400 }
+          )
+        }
+      }
+    }
+
+    const sessionUserId = session.user.id
+
+    if (uploadType === 'copilot' && !sessionUserId?.trim()) {
+      return NextResponse.json(
+        { error: 'Authenticated user session is required for copilot uploads' },
+        { status: 400 }
+      )
+    }
+
+    if (!isUsingCloudStorage()) {
+      return NextResponse.json(
+        { error: 'Direct uploads are only available when cloud storage is enabled' },
+        { status: 400 }
+      )
+    }
+
+    const storageProvider = getStorageProvider()
+    logger.info(
+      `Generating batch ${uploadType} presigned URLs for ${files.length} files using ${storageProvider}`
+    )
+
+    const startTime = Date.now()
+
+    let result
+    switch (storageProvider) {
+      case 's3':
+        result = await handleBatchS3PresignedUrls(files, uploadType, sessionUserId)
+        break
+      case 'blob':
+        result = await handleBatchBlobPresignedUrls(files, uploadType, sessionUserId)
+        break
+      default:
+        return NextResponse.json(
+          { error: `Unknown storage provider: ${storageProvider}` },
+          { status: 500 }
+        )
+    }
+
+    const duration = Date.now() - startTime
+    logger.info(
+      `Generated ${files.length} presigned URLs in ${duration}ms (avg ${Math.round(duration / files.length)}ms per file)`
+    )
+
+    return NextResponse.json(result)
+  } catch (error) {
+    logger.error('Error generating batch presigned URLs:', error)
+    return createErrorResponse(
+      error instanceof Error ? error : new Error('Failed to generate batch presigned URLs')
+    )
+  }
+}
+
+async function handleBatchS3PresignedUrls(
+  files: BatchFileRequest[],
+  uploadType: UploadType,
+  userId?: string
+) {
+  const config =
+    uploadType === 'knowledge-base'
+      ? S3_KB_CONFIG
+      : uploadType === 'chat'
+        ? S3_CHAT_CONFIG
+        : uploadType === 'copilot'
+          ? S3_COPILOT_CONFIG
+          : S3_CONFIG
+
+  if (!config.bucket || !config.region) {
+    throw new Error(`S3 configuration missing for ${uploadType} uploads`)
+  }
+
+  const { getS3Client, sanitizeFilenameForMetadata } = await import('@/lib/uploads/s3/s3-client')
+  const s3Client = getS3Client()
+
+  let prefix = ''
+  if (uploadType === 'knowledge-base') {
+    prefix = 'kb/'
+  } else if (uploadType === 'chat') {
+    prefix = 'chat/'
+  } else if (uploadType === 'copilot') {
+    prefix = `${userId}/`
+  }
+
+  const baseMetadata: Record<string, string> = {
+    uploadedAt: new Date().toISOString(),
+  }
+
+  if (uploadType === 'knowledge-base') {
+    baseMetadata.purpose = 'knowledge-base'
+  } else if (uploadType === 'chat') {
+    baseMetadata.purpose = 'chat'
+  } else if (uploadType === 'copilot') {
+    baseMetadata.purpose = 'copilot'
+    baseMetadata.userId = userId || ''
+  }
+
+  const results = await Promise.all(
+    files.map(async (file) => {
+      const safeFileName = file.fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_')
+      const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
+      const sanitizedOriginalName = sanitizeFilenameForMetadata(file.fileName)
+
+      const metadata = {
+        ...baseMetadata,
+        originalName: sanitizedOriginalName,
+      }
+
+      const command = new PutObjectCommand({
+        Bucket: config.bucket,
+        Key: uniqueKey,
+        ContentType: file.contentType,
+        Metadata: metadata,
+      })
+
+      const presignedUrl = await getSignedUrl(s3Client, command, { expiresIn: 3600 })
+
+      const finalPath =
+        uploadType === 'chat'
+          ? `https://${config.bucket}.s3.${config.region}.amazonaws.com/${uniqueKey}`
+          : `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}`
+
+      return {
+        fileName: file.fileName,
+        presignedUrl,
+        fileInfo: {
+          path: finalPath,
+          key: uniqueKey,
+          name: file.fileName,
+          size: file.fileSize,
+          type: file.contentType,
+        },
+      }
+    })
+  )
+
+  return {
+    files: results,
+    directUploadSupported: true,
+  }
+}
+
+async function handleBatchBlobPresignedUrls(
+  files: BatchFileRequest[],
+  uploadType: UploadType,
+  userId?: string
+) {
+  const config =
+    uploadType === 'knowledge-base'
+      ? BLOB_KB_CONFIG
+      : uploadType === 'chat'
+        ? BLOB_CHAT_CONFIG
+        : uploadType === 'copilot'
+          ? BLOB_COPILOT_CONFIG
+          : BLOB_CONFIG
+
+  if (
+    !config.accountName ||
+    !config.containerName ||
+    (!config.accountKey && !config.connectionString)
+  ) {
+    throw new Error(`Azure Blob configuration missing for ${uploadType} uploads`)
+  }
+
+  const { getBlobServiceClient } = await import('@/lib/uploads/blob/blob-client')
+  const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } =
+    await import('@azure/storage-blob')
+
+  const blobServiceClient = getBlobServiceClient()
+  const containerClient = blobServiceClient.getContainerClient(config.containerName)
+
+  let prefix = ''
+  if (uploadType === 'knowledge-base') {
+    prefix = 'kb/'
+  } else if (uploadType === 'chat') {
+    prefix = 'chat/'
+  } else if (uploadType === 'copilot') {
+    prefix = `${userId}/`
+  }
+
+  const baseUploadHeaders: Record<string, string> = {
+    'x-ms-blob-type': 'BlockBlob',
+    'x-ms-meta-uploadedat': new Date().toISOString(),
+  }
+
+  if (uploadType === 'knowledge-base') {
+    baseUploadHeaders['x-ms-meta-purpose'] = 'knowledge-base'
+  } else if (uploadType === 'chat') {
+    baseUploadHeaders['x-ms-meta-purpose'] = 'chat'
+  } else if (uploadType === 'copilot') {
+    baseUploadHeaders['x-ms-meta-purpose'] = 'copilot'
+    baseUploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '')
+  }
+
+  const results = await Promise.all(
+    files.map(async (file) => {
+      const safeFileName = file.fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_')
+      const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
+      const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey)
+
+      const sasOptions = {
+        containerName: config.containerName,
+        blobName: uniqueKey,
+        permissions: BlobSASPermissions.parse('w'),
+        startsOn: new Date(),
+        expiresOn: new Date(Date.now() + 3600 * 1000),
+      }
+
+      const sasToken = generateBlobSASQueryParameters(
+        sasOptions,
+        new StorageSharedKeyCredential(config.accountName, config.accountKey || '')
+      ).toString()
+
+      const presignedUrl = `${blockBlobClient.url}?${sasToken}`
+
+      const finalPath =
+        uploadType === 'chat'
+          ? blockBlobClient.url
+          : `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}`
+
+      const uploadHeaders = {
+        ...baseUploadHeaders,
+        'x-ms-blob-content-type': file.contentType,
+        'x-ms-meta-originalname': encodeURIComponent(file.fileName),
+      }
+
+      return {
+        fileName: file.fileName,
+        presignedUrl,
+        fileInfo: {
+          path: finalPath,
+          key: uniqueKey,
+          name: file.fileName,
+          size: file.fileSize,
+          type: file.contentType,
+        },
+        uploadHeaders,
+      }
+    })
+  )
+
+  return {
+    files: results,
+    directUploadSupported: true,
+  }
+}
+
+export async function OPTIONS() {
+  return createOptionsResponse()
+}

apps/sim/app/api/files/presigned/route.test.ts

@@ -1,7 +1,13 @@
 import { NextRequest } from 'next/server'
-import { beforeEach, describe, expect, test, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { setupFileApiMocks } from '@/app/api/__test-utils__/utils'
 
+/**
+ * Tests for file presigned API route
+ *
+ * @vitest-environment node
+ */
+
 describe('/api/files/presigned', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -19,7 +25,7 @@ describe('/api/files/presigned', () => {
   })
 
   describe('POST', () => {
-    test('should return error when cloud storage is not enabled', async () => {
+    it('should return error when cloud storage is not enabled', async () => {
       setupFileApiMocks({
         cloudEnabled: false,
         storageProvider: 's3',
@@ -39,7 +45,7 @@ describe('/api/files/presigned', () => {
       const response = await POST(request)
       const data = await response.json()
 
-      expect(response.status).toBe(500) // Changed from 400 to 500 (StorageConfigError)
+      expect(response.status).toBe(500)
       expect(data.error).toBe('Direct uploads are only available when cloud storage is enabled')
       expect(data.code).toBe('STORAGE_CONFIG_ERROR')
       expect(data.directUploadSupported).toBe(false)

apps/sim/app/api/files/presigned/route.ts

@@ -5,17 +5,21 @@ import { v4 as uuidv4 } from 'uuid'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
+import { isImageFileType } from '@/lib/uploads/file-utils'
 // Dynamic imports for storage clients to avoid client-side bundling
 import {
   BLOB_CHAT_CONFIG,
   BLOB_CONFIG,
   BLOB_COPILOT_CONFIG,
   BLOB_KB_CONFIG,
+  BLOB_PROFILE_PICTURES_CONFIG,
   S3_CHAT_CONFIG,
   S3_CONFIG,
   S3_COPILOT_CONFIG,
   S3_KB_CONFIG,
+  S3_PROFILE_PICTURES_CONFIG,
 } from '@/lib/uploads/setup'
+import { validateFileType } from '@/lib/uploads/validation'
 import { createErrorResponse, createOptionsResponse } from '@/app/api/files/utils'
 
 const logger = createLogger('PresignedUploadAPI')
@@ -28,7 +32,7 @@ interface PresignedUrlRequest {
   chatId?: string
 }
 
-type UploadType = 'general' | 'knowledge-base' | 'chat' | 'copilot'
+type UploadType = 'general' | 'knowledge-base' | 'chat' | 'copilot' | 'profile-pictures'
 
 class PresignedUrlError extends Error {
   constructor(
@@ -94,7 +98,16 @@ export async function POST(request: NextRequest) {
           ? 'chat'
           : uploadTypeParam === 'copilot'
             ? 'copilot'
-            : 'general'
+            : uploadTypeParam === 'profile-pictures'
+              ? 'profile-pictures'
+              : 'general'
+
+    if (uploadType === 'knowledge-base') {
+      const fileValidationError = validateFileType(fileName, contentType)
+      if (fileValidationError) {
+        throw new ValidationError(`${fileValidationError.message}`)
+      }
+    }
 
     // Evaluate user id from session for copilot uploads
     const sessionUserId = session.user.id
@@ -104,6 +117,27 @@ export async function POST(request: NextRequest) {
       if (!sessionUserId?.trim()) {
         throw new ValidationError('Authenticated user session is required for copilot uploads')
       }
+      // Only allow image uploads for copilot
+      if (!isImageFileType(contentType)) {
+        throw new ValidationError(
+          'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for copilot uploads'
+        )
+      }
+    }
+
+    // Validate profile picture requirements
+    if (uploadType === 'profile-pictures') {
+      if (!sessionUserId?.trim()) {
+        throw new ValidationError(
+          'Authenticated user session is required for profile picture uploads'
+        )
+      }
+      // Only allow image uploads for profile pictures
+      if (!isImageFileType(contentType)) {
+        throw new ValidationError(
+          'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for profile picture uploads'
+        )
+      }
     }
 
     if (!isUsingCloudStorage()) {
@@ -170,7 +204,9 @@ async function handleS3PresignedUrl(
           ? S3_CHAT_CONFIG
           : uploadType === 'copilot'
             ? S3_COPILOT_CONFIG
-            : S3_CONFIG
+            : uploadType === 'profile-pictures'
+              ? S3_PROFILE_PICTURES_CONFIG
+              : S3_CONFIG
 
     if (!config.bucket || !config.region) {
       throw new StorageConfigError(`S3 configuration missing for ${uploadType} uploads`)
@@ -185,6 +221,8 @@ async function handleS3PresignedUrl(
       prefix = 'chat/'
     } else if (uploadType === 'copilot') {
       prefix = `${userId}/`
+    } else if (uploadType === 'profile-pictures') {
+      prefix = `${userId}/`
     }
 
     const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
@@ -204,6 +242,9 @@ async function handleS3PresignedUrl(
     } else if (uploadType === 'copilot') {
       metadata.purpose = 'copilot'
       metadata.userId = userId || ''
+    } else if (uploadType === 'profile-pictures') {
+      metadata.purpose = 'profile-pictures'
+      metadata.userId = userId || ''
     }
 
     const command = new PutObjectCommand({
@@ -224,10 +265,9 @@ async function handleS3PresignedUrl(
       )
     }
 
-    // For chat images, use direct S3 URLs since they need to be permanently accessible
-    // For other files, use serve path for access control
+    // For chat images, knowledge base files, and profile pictures, use direct URLs since they need to be accessible by external services
     const finalPath =
-      uploadType === 'chat'
+      uploadType === 'chat' || uploadType === 'knowledge-base' || uploadType === 'profile-pictures'
         ? `https://${config.bucket}.s3.${config.region}.amazonaws.com/${uniqueKey}`
         : `/api/files/serve/s3/${encodeURIComponent(uniqueKey)}`
 
@@ -271,7 +311,9 @@ async function handleBlobPresignedUrl(
           ? BLOB_CHAT_CONFIG
           : uploadType === 'copilot'
             ? BLOB_COPILOT_CONFIG
-            : BLOB_CONFIG
+            : uploadType === 'profile-pictures'
+              ? BLOB_PROFILE_PICTURES_CONFIG
+              : BLOB_CONFIG
 
     if (
       !config.accountName ||
@@ -290,6 +332,8 @@ async function handleBlobPresignedUrl(
       prefix = 'chat/'
     } else if (uploadType === 'copilot') {
       prefix = `${userId}/`
+    } else if (uploadType === 'profile-pictures') {
+      prefix = `${userId}/`
     }
 
     const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}`
@@ -325,10 +369,10 @@ async function handleBlobPresignedUrl(
 
     const presignedUrl = `${blockBlobClient.url}?${sasToken}`
 
-    // For chat images, use direct Blob URLs since they need to be permanently accessible
+    // For chat images and profile pictures, use direct Blob URLs since they need to be permanently accessible
     // For other files, use serve path for access control
     const finalPath =
-      uploadType === 'chat'
+      uploadType === 'chat' || uploadType === 'profile-pictures'
         ? blockBlobClient.url
         : `/api/files/serve/blob/${encodeURIComponent(uniqueKey)}`
 
@@ -348,6 +392,9 @@ async function handleBlobPresignedUrl(
     } else if (uploadType === 'copilot') {
       uploadHeaders['x-ms-meta-purpose'] = 'copilot'
       uploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '')
+    } else if (uploadType === 'profile-pictures') {
+      uploadHeaders['x-ms-meta-purpose'] = 'profile-pictures'
+      uploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '')
     }
 
     return NextResponse.json({

apps/sim/app/api/files/serve/[...path]/route.ts

@@ -2,7 +2,7 @@ import { readFile } from 'fs/promises'
 import type { NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console/logger'
 import { downloadFile, getStorageProvider, isUsingCloudStorage } from '@/lib/uploads'
-import { BLOB_KB_CONFIG, S3_KB_CONFIG } from '@/lib/uploads/setup'
+import { S3_KB_CONFIG } from '@/lib/uploads/setup'
 import '@/lib/uploads/setup.server'
 
 import {
@@ -15,19 +15,6 @@ import {
 
 const logger = createLogger('FilesServeAPI')
 
-async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer> {
-  return new Promise((resolve, reject) => {
-    const chunks: Buffer[] = []
-    readableStream.on('data', (data) => {
-      chunks.push(data instanceof Buffer ? data : Buffer.from(data))
-    })
-    readableStream.on('end', () => {
-      resolve(Buffer.concat(chunks))
-    })
-    readableStream.on('error', reject)
-  })
-}
-
 /**
  * Main API route handler for serving files
  */
@@ -102,49 +89,23 @@ async function handleLocalFile(filename: string): Promise<NextResponse> {
 }
 
 async function downloadKBFile(cloudKey: string): Promise<Buffer> {
+  logger.info(`Downloading KB file: ${cloudKey}`)
   const storageProvider = getStorageProvider()
 
   if (storageProvider === 'blob') {
-    logger.info(`Downloading KB file from Azure Blob Storage: ${cloudKey}`)
-    // Use KB-specific blob configuration
-    const { getBlobServiceClient } = await import('@/lib/uploads/blob/blob-client')
-    const blobServiceClient = getBlobServiceClient()
-    const containerClient = blobServiceClient.getContainerClient(BLOB_KB_CONFIG.containerName)
-    const blockBlobClient = containerClient.getBlockBlobClient(cloudKey)
-
-    const downloadBlockBlobResponse = await blockBlobClient.download()
-    if (!downloadBlockBlobResponse.readableStreamBody) {
-      throw new Error('Failed to get readable stream from blob download')
-    }
-
-    // Convert stream to buffer
-    return await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)
+    const { BLOB_KB_CONFIG } = await import('@/lib/uploads/setup')
+    return downloadFile(cloudKey, {
+      containerName: BLOB_KB_CONFIG.containerName,
+      accountName: BLOB_KB_CONFIG.accountName,
+      accountKey: BLOB_KB_CONFIG.accountKey,
+      connectionString: BLOB_KB_CONFIG.connectionString,
+    })
   }
 
   if (storageProvider === 's3') {
-    logger.info(`Downloading KB file from S3: ${cloudKey}`)
-    // Use KB-specific S3 configuration
-    const { getS3Client } = await import('@/lib/uploads/s3/s3-client')
-    const { GetObjectCommand } = await import('@aws-sdk/client-s3')
-
-    const s3Client = getS3Client()
-    const command = new GetObjectCommand({
-      Bucket: S3_KB_CONFIG.bucket,
-      Key: cloudKey,
-    })
-
-    const response = await s3Client.send(command)
-    if (!response.Body) {
-      throw new Error('No body in S3 response')
-    }
-
-    // Convert stream to buffer using the same method as the regular S3 client
-    const stream = response.Body as any
-    return new Promise<Buffer>((resolve, reject) => {
-      const chunks: Buffer[] = []
-      stream.on('data', (chunk: Buffer) => chunks.push(chunk))
-      stream.on('end', () => resolve(Buffer.concat(chunks)))
-      stream.on('error', reject)
+    return downloadFile(cloudKey, {
+      bucket: S3_KB_CONFIG.bucket,
+      region: S3_KB_CONFIG.region,
     })
   }
 
@@ -167,17 +128,22 @@ async function handleCloudProxy(
     if (isKBFile) {
       fileBuffer = await downloadKBFile(cloudKey)
     } else if (bucketType === 'copilot') {
-      // Download from copilot-specific bucket
       const storageProvider = getStorageProvider()
 
       if (storageProvider === 's3') {
-        const { downloadFromS3WithConfig } = await import('@/lib/uploads/s3/s3-client')
         const { S3_COPILOT_CONFIG } = await import('@/lib/uploads/setup')
-        fileBuffer = await downloadFromS3WithConfig(cloudKey, S3_COPILOT_CONFIG)
+        fileBuffer = await downloadFile(cloudKey, {
+          bucket: S3_COPILOT_CONFIG.bucket,
+          region: S3_COPILOT_CONFIG.region,
+        })
       } else if (storageProvider === 'blob') {
-        // For Azure Blob, use the default downloadFile for now
-        // TODO: Add downloadFromBlobWithConfig when needed
-        fileBuffer = await downloadFile(cloudKey)
+        const { BLOB_COPILOT_CONFIG } = await import('@/lib/uploads/setup')
+        fileBuffer = await downloadFile(cloudKey, {
+          containerName: BLOB_COPILOT_CONFIG.containerName,
+          accountName: BLOB_COPILOT_CONFIG.accountName,
+          accountKey: BLOB_COPILOT_CONFIG.accountKey,
+          connectionString: BLOB_COPILOT_CONFIG.connectionString,
+        })
       } else {
         fileBuffer = await downloadFile(cloudKey)
       }

apps/sim/app/api/files/upload/route.test.ts

@@ -186,3 +186,190 @@ describe('File Upload API Route', () => {
     expect(response.headers.get('Access-Control-Allow-Headers')).toBe('Content-Type')
   })
 })
+
+describe('File Upload Security Tests', () => {
+  beforeEach(() => {
+    vi.resetModules()
+    vi.clearAllMocks()
+
+    vi.doMock('@/lib/auth', () => ({
+      getSession: vi.fn().mockResolvedValue({
+        user: { id: 'test-user-id' },
+      }),
+    }))
+
+    vi.doMock('@/lib/uploads', () => ({
+      isUsingCloudStorage: vi.fn().mockReturnValue(false),
+      uploadFile: vi.fn().mockResolvedValue({
+        key: 'test-key',
+        path: '/test/path',
+      }),
+    }))
+
+    vi.doMock('@/lib/uploads/setup.server', () => ({}))
+  })
+
+  afterEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('File Extension Validation', () => {
+    it('should accept allowed file types', async () => {
+      const allowedTypes = [
+        'pdf',
+        'doc',
+        'docx',
+        'txt',
+        'md',
+        'png',
+        'jpg',
+        'jpeg',
+        'gif',
+        'csv',
+        'xlsx',
+        'xls',
+      ]
+
+      for (const ext of allowedTypes) {
+        const formData = new FormData()
+        const file = new File(['test content'], `test.${ext}`, { type: 'application/octet-stream' })
+        formData.append('file', file)
+
+        const req = new Request('http://localhost/api/files/upload', {
+          method: 'POST',
+          body: formData,
+        })
+
+        const { POST } = await import('@/app/api/files/upload/route')
+        const response = await POST(req as any)
+
+        expect(response.status).toBe(200)
+      }
+    })
+
+    it('should reject HTML files to prevent XSS', async () => {
+      const formData = new FormData()
+      const maliciousContent = '<script>alert("XSS")</script>'
+      const file = new File([maliciousContent], 'malicious.html', { type: 'text/html' })
+      formData.append('file', file)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.message).toContain("File type 'html' is not allowed")
+    })
+
+    it('should reject SVG files to prevent XSS', async () => {
+      const formData = new FormData()
+      const maliciousSvg = '<svg onload="alert(\'XSS\')" xmlns="http://www.w3.org/2000/svg"></svg>'
+      const file = new File([maliciousSvg], 'malicious.svg', { type: 'image/svg+xml' })
+      formData.append('file', file)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.message).toContain("File type 'svg' is not allowed")
+    })
+
+    it('should reject JavaScript files', async () => {
+      const formData = new FormData()
+      const maliciousJs = 'alert("XSS")'
+      const file = new File([maliciousJs], 'malicious.js', { type: 'application/javascript' })
+      formData.append('file', file)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.message).toContain("File type 'js' is not allowed")
+    })
+
+    it('should reject files without extensions', async () => {
+      const formData = new FormData()
+      const file = new File(['test content'], 'noextension', { type: 'application/octet-stream' })
+      formData.append('file', file)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.message).toContain("File type 'noextension' is not allowed")
+    })
+
+    it('should handle multiple files with mixed valid/invalid types', async () => {
+      const formData = new FormData()
+
+      // Valid file
+      const validFile = new File(['valid content'], 'valid.pdf', { type: 'application/pdf' })
+      formData.append('file', validFile)
+
+      // Invalid file (should cause rejection of entire request)
+      const invalidFile = new File(['<script>alert("XSS")</script>'], 'malicious.html', {
+        type: 'text/html',
+      })
+      formData.append('file', invalidFile)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.message).toContain("File type 'html' is not allowed")
+    })
+  })
+
+  describe('Authentication Requirements', () => {
+    it('should reject uploads without authentication', async () => {
+      vi.doMock('@/lib/auth', () => ({
+        getSession: vi.fn().mockResolvedValue(null),
+      }))
+
+      const formData = new FormData()
+      const file = new File(['test content'], 'test.pdf', { type: 'application/pdf' })
+      formData.append('file', file)
+
+      const req = new Request('http://localhost/api/files/upload', {
+        method: 'POST',
+        body: formData,
+      })
+
+      const { POST } = await import('@/app/api/files/upload/route')
+      const response = await POST(req as any)
+
+      expect(response.status).toBe(401)
+      const data = await response.json()
+      expect(data.error).toBe('Unauthorized')
+    })
+  })
+})

apps/sim/app/api/files/upload/route.ts

@@ -9,6 +9,34 @@ import {
   InvalidRequestError,
 } from '@/app/api/files/utils'
 
+// Allowlist of permitted file extensions for security
+const ALLOWED_EXTENSIONS = new Set([
+  // Documents
+  'pdf',
+  'doc',
+  'docx',
+  'txt',
+  'md',
+  // Images (safe formats)
+  'png',
+  'jpg',
+  'jpeg',
+  'gif',
+  // Data files
+  'csv',
+  'xlsx',
+  'xls',
+])
+
+/**
+ * Validates file extension against allowlist
+ */
+function validateFileExtension(filename: string): boolean {
+  const extension = filename.split('.').pop()?.toLowerCase()
+  if (!extension) return false
+  return ALLOWED_EXTENSIONS.has(extension)
+}
+
 export const dynamic = 'force-dynamic'
 
 const logger = createLogger('FilesUploadAPI')
@@ -49,6 +77,14 @@ export async function POST(request: NextRequest) {
     // Process each file
     for (const file of files) {
       const originalName = file.name
+
+      if (!validateFileExtension(originalName)) {
+        const extension = originalName.split('.').pop()?.toLowerCase() || 'unknown'
+        throw new InvalidRequestError(
+          `File type '${extension}' is not allowed. Allowed types: ${Array.from(ALLOWED_EXTENSIONS).join(', ')}`
+        )
+      }
+
       const bytes = await file.arrayBuffer()
       const buffer = Buffer.from(bytes)
 

apps/sim/app/api/files/utils.test.ts

@@ -0,0 +1,327 @@
+import { describe, expect, it } from 'vitest'
+import { createFileResponse, extractFilename } from './utils'
+
+describe('extractFilename', () => {
+  describe('legitimate file paths', () => {
+    it('should extract filename from standard serve path', () => {
+      expect(extractFilename('/api/files/serve/test-file.txt')).toBe('test-file.txt')
+    })
+
+    it('should extract filename from serve path with special characters', () => {
+      expect(extractFilename('/api/files/serve/document-with-dashes_and_underscores.pdf')).toBe(
+        'document-with-dashes_and_underscores.pdf'
+      )
+    })
+
+    it('should handle simple filename without serve path', () => {
+      expect(extractFilename('simple-file.txt')).toBe('simple-file.txt')
+    })
+
+    it('should extract last segment from nested path', () => {
+      expect(extractFilename('nested/path/file.txt')).toBe('file.txt')
+    })
+  })
+
+  describe('cloud storage paths', () => {
+    it('should preserve S3 path structure', () => {
+      expect(extractFilename('/api/files/serve/s3/1234567890-test-file.txt')).toBe(
+        's3/1234567890-test-file.txt'
+      )
+    })
+
+    it('should preserve S3 path with nested folders', () => {
+      expect(extractFilename('/api/files/serve/s3/folder/subfolder/document.pdf')).toBe(
+        's3/folder/subfolder/document.pdf'
+      )
+    })
+
+    it('should preserve Azure Blob path structure', () => {
+      expect(extractFilename('/api/files/serve/blob/1234567890-test-document.pdf')).toBe(
+        'blob/1234567890-test-document.pdf'
+      )
+    })
+
+    it('should preserve Blob path with nested folders', () => {
+      expect(extractFilename('/api/files/serve/blob/uploads/user-files/report.xlsx')).toBe(
+        'blob/uploads/user-files/report.xlsx'
+      )
+    })
+  })
+
+  describe('security - path traversal prevention', () => {
+    it('should sanitize basic path traversal attempt', () => {
+      expect(extractFilename('/api/files/serve/../config.txt')).toBe('config.txt')
+    })
+
+    it('should sanitize deep path traversal attempt', () => {
+      expect(extractFilename('/api/files/serve/../../../../../etc/passwd')).toBe('etcpasswd')
+    })
+
+    it('should sanitize multiple path traversal patterns', () => {
+      expect(extractFilename('/api/files/serve/../../secret.txt')).toBe('secret.txt')
+    })
+
+    it('should sanitize path traversal with forward slashes', () => {
+      expect(extractFilename('/api/files/serve/../../../system/file')).toBe('systemfile')
+    })
+
+    it('should sanitize mixed path traversal patterns', () => {
+      expect(extractFilename('/api/files/serve/../folder/../file.txt')).toBe('folderfile.txt')
+    })
+
+    it('should remove directory separators from local filenames', () => {
+      expect(extractFilename('/api/files/serve/folder/with/separators.txt')).toBe(
+        'folderwithseparators.txt'
+      )
+    })
+
+    it('should handle backslash path separators (Windows style)', () => {
+      expect(extractFilename('/api/files/serve/folder\\file.txt')).toBe('folderfile.txt')
+    })
+  })
+
+  describe('cloud storage path traversal prevention', () => {
+    it('should sanitize S3 path traversal attempts while preserving structure', () => {
+      expect(extractFilename('/api/files/serve/s3/../config')).toBe('s3/config')
+    })
+
+    it('should sanitize S3 path with nested traversal attempts', () => {
+      expect(extractFilename('/api/files/serve/s3/folder/../sensitive/../file.txt')).toBe(
+        's3/folder/sensitive/file.txt'
+      )
+    })
+
+    it('should sanitize Blob path traversal attempts while preserving structure', () => {
+      expect(extractFilename('/api/files/serve/blob/../system.txt')).toBe('blob/system.txt')
+    })
+
+    it('should remove leading dots from cloud path segments', () => {
+      expect(extractFilename('/api/files/serve/s3/.hidden/../file.txt')).toBe('s3/hidden/file.txt')
+    })
+  })
+
+  describe('edge cases and error handling', () => {
+    it('should handle filename with dots (but not traversal)', () => {
+      expect(extractFilename('/api/files/serve/file.with.dots.txt')).toBe('file.with.dots.txt')
+    })
+
+    it('should handle filename with multiple extensions', () => {
+      expect(extractFilename('/api/files/serve/archive.tar.gz')).toBe('archive.tar.gz')
+    })
+
+    it('should throw error for empty filename after sanitization', () => {
+      expect(() => extractFilename('/api/files/serve/')).toThrow(
+        'Invalid or empty filename after sanitization'
+      )
+    })
+
+    it('should throw error for filename that becomes empty after path traversal removal', () => {
+      expect(() => extractFilename('/api/files/serve/../..')).toThrow(
+        'Invalid or empty filename after sanitization'
+      )
+    })
+
+    it('should handle single character filenames', () => {
+      expect(extractFilename('/api/files/serve/a')).toBe('a')
+    })
+
+    it('should handle numeric filenames', () => {
+      expect(extractFilename('/api/files/serve/123')).toBe('123')
+    })
+  })
+
+  describe('backward compatibility', () => {
+    it('should match old behavior for legitimate local files', () => {
+      // These test cases verify that our security fix maintains exact backward compatibility
+      // for all legitimate use cases found in the existing codebase
+      expect(extractFilename('/api/files/serve/test-file.txt')).toBe('test-file.txt')
+      expect(extractFilename('/api/files/serve/nonexistent.txt')).toBe('nonexistent.txt')
+    })
+
+    it('should match old behavior for legitimate cloud files', () => {
+      // These test cases are from the actual delete route tests
+      expect(extractFilename('/api/files/serve/s3/1234567890-test-file.txt')).toBe(
+        's3/1234567890-test-file.txt'
+      )
+      expect(extractFilename('/api/files/serve/blob/1234567890-test-document.pdf')).toBe(
+        'blob/1234567890-test-document.pdf'
+      )
+    })
+
+    it('should match old behavior for simple paths', () => {
+      // These match the mock implementations in serve route tests
+      expect(extractFilename('simple-file.txt')).toBe('simple-file.txt')
+      expect(extractFilename('nested/path/file.txt')).toBe('file.txt')
+    })
+  })
+
+  describe('File Serving Security Tests', () => {
+    describe('createFileResponse security headers', () => {
+      it('should serve safe images inline with proper headers', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('fake-image-data'),
+          contentType: 'image/png',
+          filename: 'safe-image.png',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('image/png')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'inline; filename="safe-image.png"'
+        )
+        expect(response.headers.get('X-Content-Type-Options')).toBe('nosniff')
+        expect(response.headers.get('Content-Security-Policy')).toBe(
+          "default-src 'none'; style-src 'unsafe-inline'; sandbox;"
+        )
+      })
+
+      it('should serve PDFs inline safely', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('fake-pdf-data'),
+          contentType: 'application/pdf',
+          filename: 'document.pdf',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/pdf')
+        expect(response.headers.get('Content-Disposition')).toBe('inline; filename="document.pdf"')
+        expect(response.headers.get('X-Content-Type-Options')).toBe('nosniff')
+      })
+
+      it('should force attachment for HTML files to prevent XSS', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('<script>alert("XSS")</script>'),
+          contentType: 'text/html',
+          filename: 'malicious.html',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/octet-stream')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="malicious.html"'
+        )
+        expect(response.headers.get('X-Content-Type-Options')).toBe('nosniff')
+      })
+
+      it('should force attachment for SVG files to prevent XSS', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from(
+            '<svg onload="alert(\'XSS\')" xmlns="http://www.w3.org/2000/svg"></svg>'
+          ),
+          contentType: 'image/svg+xml',
+          filename: 'malicious.svg',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/octet-stream')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="malicious.svg"'
+        )
+      })
+
+      it('should override dangerous content types to safe alternatives', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('<svg>safe content</svg>'),
+          contentType: 'image/svg+xml',
+          filename: 'image.png', // Extension doesn't match content-type
+        })
+
+        expect(response.status).toBe(200)
+        // Should override SVG content type to plain text for safety
+        expect(response.headers.get('Content-Type')).toBe('text/plain')
+        expect(response.headers.get('Content-Disposition')).toBe('inline; filename="image.png"')
+      })
+
+      it('should force attachment for JavaScript files', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('alert("XSS")'),
+          contentType: 'application/javascript',
+          filename: 'malicious.js',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/octet-stream')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="malicious.js"'
+        )
+      })
+
+      it('should force attachment for CSS files', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('body { background: url(javascript:alert("XSS")) }'),
+          contentType: 'text/css',
+          filename: 'malicious.css',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/octet-stream')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="malicious.css"'
+        )
+      })
+
+      it('should force attachment for XML files', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('<?xml version="1.0"?><root><script>alert("XSS")</script></root>'),
+          contentType: 'application/xml',
+          filename: 'malicious.xml',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/octet-stream')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="malicious.xml"'
+        )
+      })
+
+      it('should serve text files safely', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('Safe text content'),
+          contentType: 'text/plain',
+          filename: 'document.txt',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('text/plain')
+        expect(response.headers.get('Content-Disposition')).toBe('inline; filename="document.txt"')
+      })
+
+      it('should force attachment for unknown/unsafe content types', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('unknown content'),
+          contentType: 'application/unknown',
+          filename: 'unknown.bin',
+        })
+
+        expect(response.status).toBe(200)
+        expect(response.headers.get('Content-Type')).toBe('application/unknown')
+        expect(response.headers.get('Content-Disposition')).toBe(
+          'attachment; filename="unknown.bin"'
+        )
+      })
+    })
+
+    describe('Content Security Policy', () => {
+      it('should include CSP header in all responses', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('test'),
+          contentType: 'text/plain',
+          filename: 'test.txt',
+        })
+
+        const csp = response.headers.get('Content-Security-Policy')
+        expect(csp).toBe("default-src 'none'; style-src 'unsafe-inline'; sandbox;")
+      })
+
+      it('should include X-Content-Type-Options header', () => {
+        const response = createFileResponse({
+          buffer: Buffer.from('test'),
+          contentType: 'text/plain',
+          filename: 'test.txt',
+        })
+
+        expect(response.headers.get('X-Content-Type-Options')).toBe('nosniff')
+      })
+    })
+  })
+})

apps/sim/app/api/files/utils.ts

@@ -70,7 +70,6 @@ export const contentTypeMap: Record<string, string> = {
   jpg: 'image/jpeg',
   jpeg: 'image/jpeg',
   gif: 'image/gif',
-  svg: 'image/svg+xml',
   // Archive formats
   zip: 'application/zip',
   // Folder format
@@ -153,10 +152,43 @@ export function extractBlobKey(path: string): string {
  * Extract filename from a serve path
  */
 export function extractFilename(path: string): string {
+  let filename: string
+
   if (path.startsWith('/api/files/serve/')) {
-    return path.substring('/api/files/serve/'.length)
+    filename = path.substring('/api/files/serve/'.length)
+  } else {
+    filename = path.split('/').pop() || path
   }
-  return path.split('/').pop() || path
+
+  filename = filename
+    .replace(/\.\./g, '')
+    .replace(/\/\.\./g, '')
+    .replace(/\.\.\//g, '')
+
+  // Handle cloud storage paths (s3/key, blob/key) - preserve forward slashes for these
+  if (filename.startsWith('s3/') || filename.startsWith('blob/')) {
+    // For cloud paths, only sanitize the key portion after the prefix
+    const parts = filename.split('/')
+    const prefix = parts[0] // 's3' or 'blob'
+    const keyParts = parts.slice(1)
+
+    // Sanitize each part of the key to prevent traversal
+    const sanitizedKeyParts = keyParts
+      .map((part) => part.replace(/\.\./g, '').replace(/^\./g, '').trim())
+      .filter((part) => part.length > 0)
+
+    filename = `${prefix}/${sanitizedKeyParts.join('/')}`
+  } else {
+    // For regular filenames, remove any remaining path separators
+    filename = filename.replace(/[/\\]/g, '')
+  }
+
+  // Additional validation: ensure filename is not empty after sanitization
+  if (!filename || filename.trim().length === 0) {
+    throw new Error('Invalid or empty filename after sanitization')
+  }
+
+  return filename
 }
 
 /**
@@ -174,16 +206,65 @@ export function findLocalFile(filename: string): string | null {
   return null
 }
 
+const SAFE_INLINE_TYPES = new Set([
+  'image/png',
+  'image/jpeg',
+  'image/jpg',
+  'image/gif',
+  'application/pdf',
+  'text/plain',
+  'text/csv',
+  'application/json',
+])
+
+// File extensions that should always be served as attachment for security
+const FORCE_ATTACHMENT_EXTENSIONS = new Set(['html', 'htm', 'svg', 'js', 'css', 'xml'])
+
 /**
- * Create a file response with appropriate headers
+ * Determines safe content type and disposition for file serving
+ */
+function getSecureFileHeaders(filename: string, originalContentType: string) {
+  const extension = filename.split('.').pop()?.toLowerCase() || ''
+
+  // Force attachment for potentially dangerous file types
+  if (FORCE_ATTACHMENT_EXTENSIONS.has(extension)) {
+    return {
+      contentType: 'application/octet-stream', // Force download
+      disposition: 'attachment',
+    }
+  }
+
+  // Override content type for safety while preserving legitimate use cases
+  let safeContentType = originalContentType
+
+  // Handle potentially dangerous content types
+  if (originalContentType === 'text/html' || originalContentType === 'image/svg+xml') {
+    safeContentType = 'text/plain' // Prevent browser rendering
+  }
+
+  // Use inline only for verified safe content types
+  const disposition = SAFE_INLINE_TYPES.has(safeContentType) ? 'inline' : 'attachment'
+
+  return {
+    contentType: safeContentType,
+    disposition,
+  }
+}
+
+/**
+ * Create a file response with appropriate security headers
  */
 export function createFileResponse(file: FileResponse): NextResponse {
+  const { contentType, disposition } = getSecureFileHeaders(file.filename, file.contentType)
+
   return new NextResponse(file.buffer as BodyInit, {
     status: 200,
     headers: {
-      'Content-Type': file.contentType,
-      'Content-Disposition': `inline; filename="${file.filename}"`,
+      'Content-Type': contentType,
+      'Content-Disposition': `${disposition}; filename="${file.filename}"`,
       'Cache-Control': 'public, max-age=31536000', // Cache for 1 year
+      'X-Content-Type-Options': 'nosniff',
+      'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; sandbox;",
     },
   })
 }

apps/sim/app/api/function/execute/route.test.ts

@@ -32,6 +32,14 @@ describe('Function Execute API Route', () => {
       createLogger: vi.fn().mockReturnValue(mockLogger),
     }))
 
+    vi.doMock('@/lib/execution/e2b', () => ({
+      executeInE2B: vi.fn().mockResolvedValue({
+        result: 'e2b success',
+        stdout: 'e2b output',
+        sandboxId: 'test-sandbox-id',
+      }),
+    }))
+
     mockRunInContext.mockResolvedValue('vm success')
     mockCreateContext.mockReturnValue({})
   })
@@ -45,6 +53,7 @@ describe('Function Execute API Route', () => {
       const req = createMockRequest('POST', {
         code: 'return "Hello World"',
         timeout: 5000,
+        useLocalVM: true,
       })
 
       const { POST } = await import('@/app/api/function/execute/route')
@@ -74,6 +83,7 @@ describe('Function Execute API Route', () => {
     it('should use default timeout when not provided', async () => {
       const req = createMockRequest('POST', {
         code: 'return "test"',
+        useLocalVM: true,
       })
 
       const { POST } = await import('@/app/api/function/execute/route')
@@ -93,6 +103,7 @@ describe('Function Execute API Route', () => {
     it('should resolve environment variables with {{var_name}} syntax', async () => {
       const req = createMockRequest('POST', {
         code: 'return {{API_KEY}}',
+        useLocalVM: true,
         envVars: {
           API_KEY: 'secret-key-123',
         },
@@ -108,6 +119,7 @@ describe('Function Execute API Route', () => {
     it('should resolve tag variables with <tag_name> syntax', async () => {
       const req = createMockRequest('POST', {
         code: 'return <email>',
+        useLocalVM: true,
         params: {
           email: { id: '123', subject: 'Test Email' },
         },
@@ -123,6 +135,7 @@ describe('Function Execute API Route', () => {
     it('should NOT treat email addresses as template variables', async () => {
       const req = createMockRequest('POST', {
         code: 'return "Email sent to user"',
+        useLocalVM: true,
         params: {
           email: {
             from: 'Waleed Latif <waleed@sim.ai>',
@@ -141,6 +154,7 @@ describe('Function Execute API Route', () => {
     it('should only match valid variable names in angle brackets', async () => {
       const req = createMockRequest('POST', {
         code: 'return <validVar> + "<invalid@email.com>" + <another_valid>',
+        useLocalVM: true,
         params: {
           validVar: 'hello',
           another_valid: 'world',
@@ -178,6 +192,7 @@ describe('Function Execute API Route', () => {
 
       const req = createMockRequest('POST', {
         code: 'return <email>',
+        useLocalVM: true,
         params: gmailData,
       })
 
@@ -200,6 +215,7 @@ describe('Function Execute API Route', () => {
 
       const req = createMockRequest('POST', {
         code: 'return <email>',
+        useLocalVM: true,
         params: complexEmailData,
       })
 
@@ -214,6 +230,7 @@ describe('Function Execute API Route', () => {
     it('should handle custom tool execution with direct parameter access', async () => {
       const req = createMockRequest('POST', {
         code: 'return location + " weather is sunny"',
+        useLocalVM: true,
         params: {
           location: 'San Francisco',
         },
@@ -245,6 +262,7 @@ describe('Function Execute API Route', () => {
     it('should handle timeout parameter', async () => {
       const req = createMockRequest('POST', {
         code: 'return "test"',
+        useLocalVM: true,
         timeout: 10000,
       })
 
@@ -262,6 +280,7 @@ describe('Function Execute API Route', () => {
     it('should handle empty parameters object', async () => {
       const req = createMockRequest('POST', {
         code: 'return "no params"',
+        useLocalVM: true,
         params: {},
       })
 
@@ -295,6 +314,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'const obj = {\n  name: "test",\n  description: "This has a missing closing quote\n};\nreturn obj;',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -338,6 +358,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'const obj = null;\nreturn obj.someMethod();',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -379,6 +400,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'const x = 42;\nreturn undefinedVariable + x;',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -409,6 +431,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'return "test";',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -445,6 +468,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'const a = 1;\nconst b = 2;\nconst c = 3;\nconst d = 4;\nreturn a + b + c + d;',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -476,6 +500,7 @@ SyntaxError: Invalid or unexpected token
 
       const req = createMockRequest('POST', {
         code: 'const obj = {\n  name: "test"\n// Missing closing brace',
+        useLocalVM: true,
         timeout: 5000,
       })
 
@@ -496,6 +521,7 @@ SyntaxError: Invalid or unexpected token
       // This tests the escapeRegExp function indirectly
       const req = createMockRequest('POST', {
         code: 'return {{special.chars+*?}}',
+        useLocalVM: true,
         envVars: {
           'special.chars+*?': 'escaped-value',
         },
@@ -512,6 +538,7 @@ SyntaxError: Invalid or unexpected token
       // Test with complex but not circular data first
       const req = createMockRequest('POST', {
         code: 'return <complexData>',
+        useLocalVM: true,
         params: {
           complexData: {
             special: 'chars"with\'quotes',

apps/sim/app/api/function/execute/route.ts

@@ -1,13 +1,20 @@
 import { createContext, Script } from 'vm'
 import { type NextRequest, NextResponse } from 'next/server'
+import { env, isTruthy } from '@/lib/env'
+import { executeInE2B } from '@/lib/execution/e2b'
+import { CodeLanguage, DEFAULT_CODE_LANGUAGE, isValidCodeLanguage } from '@/lib/execution/languages'
 import { createLogger } from '@/lib/logs/console/logger'
-
+import { generateRequestId } from '@/lib/utils'
 export const dynamic = 'force-dynamic'
 export const runtime = 'nodejs'
 export const maxDuration = 60
 
 const logger = createLogger('FunctionExecuteAPI')
 
+// Constants for E2B code wrapping line counts
+const E2B_JS_WRAPPER_LINES = 3 // Lines before user code: ';(async () => {', '  try {', '    const __sim_result = await (async () => {'
+const E2B_PYTHON_WRAPPER_LINES = 1 // Lines before user code: 'def __sim_main__():'
+
 /**
  * Enhanced error information interface
  */
@@ -124,6 +131,103 @@ function extractEnhancedError(
   return enhanced
 }
 
+/**
+ * Parse and format E2B error message
+ * Removes E2B-specific line references and adds correct user line numbers
+ */
+function formatE2BError(
+  errorMessage: string,
+  errorOutput: string,
+  language: CodeLanguage,
+  userCode: string,
+  prologueLineCount: number
+): { formattedError: string; cleanedOutput: string } {
+  // Calculate line offset based on language and prologue
+  const wrapperLines =
+    language === CodeLanguage.Python ? E2B_PYTHON_WRAPPER_LINES : E2B_JS_WRAPPER_LINES
+  const totalOffset = prologueLineCount + wrapperLines
+
+  let userLine: number | undefined
+  let cleanErrorType = ''
+  let cleanErrorMsg = ''
+
+  if (language === CodeLanguage.Python) {
+    // Python error format: "Cell In[X], line Y" followed by error details
+    // Extract line number from the Cell reference
+    const cellMatch = errorOutput.match(/Cell In\[\d+\], line (\d+)/)
+    if (cellMatch) {
+      const originalLine = Number.parseInt(cellMatch[1], 10)
+      userLine = originalLine - totalOffset
+    }
+
+    // Extract clean error message from the error string
+    // Remove file references like "(detected at line X) (file.py, line Y)"
+    cleanErrorMsg = errorMessage
+      .replace(/\s*\(detected at line \d+\)/g, '')
+      .replace(/\s*\([^)]+\.py, line \d+\)/g, '')
+      .trim()
+  } else if (language === CodeLanguage.JavaScript) {
+    // JavaScript error format from E2B: "SyntaxError: /path/file.ts: Message. (line:col)\n\n   9 | ..."
+    // First, extract the error type and message from the first line
+    const firstLineEnd = errorMessage.indexOf('\n')
+    const firstLine = firstLineEnd > 0 ? errorMessage.substring(0, firstLineEnd) : errorMessage
+
+    // Parse: "SyntaxError: /home/user/index.ts: Missing semicolon. (11:9)"
+    const jsErrorMatch = firstLine.match(/^(\w+Error):\s*[^:]+:\s*([^(]+)\.\s*\((\d+):(\d+)\)/)
+    if (jsErrorMatch) {
+      cleanErrorType = jsErrorMatch[1]
+      cleanErrorMsg = jsErrorMatch[2].trim()
+      const originalLine = Number.parseInt(jsErrorMatch[3], 10)
+      userLine = originalLine - totalOffset
+    } else {
+      // Fallback: look for line number in the arrow pointer line (> 11 |)
+      const arrowMatch = errorMessage.match(/^>\s*(\d+)\s*\|/m)
+      if (arrowMatch) {
+        const originalLine = Number.parseInt(arrowMatch[1], 10)
+        userLine = originalLine - totalOffset
+      }
+      // Try to extract error type and message
+      const errorMatch = firstLine.match(/^(\w+Error):\s*(.+)/)
+      if (errorMatch) {
+        cleanErrorType = errorMatch[1]
+        cleanErrorMsg = errorMatch[2]
+          .replace(/^[^:]+:\s*/, '') // Remove file path
+          .replace(/\s*\(\d+:\d+\)\s*$/, '') // Remove line:col at end
+          .trim()
+      } else {
+        cleanErrorMsg = firstLine
+      }
+    }
+  }
+
+  // Build the final clean error message
+  const finalErrorMsg =
+    cleanErrorType && cleanErrorMsg
+      ? `${cleanErrorType}: ${cleanErrorMsg}`
+      : cleanErrorMsg || errorMessage
+
+  // Format with line number if available
+  let formattedError = finalErrorMsg
+  if (userLine && userLine > 0) {
+    const codeLines = userCode.split('\n')
+    // Clamp userLine to the actual user code range
+    const actualUserLine = Math.min(userLine, codeLines.length)
+    if (actualUserLine > 0 && actualUserLine <= codeLines.length) {
+      const lineContent = codeLines[actualUserLine - 1]?.trim()
+      if (lineContent) {
+        formattedError = `Line ${actualUserLine}: \`${lineContent}\` - ${finalErrorMsg}`
+      } else {
+        formattedError = `Line ${actualUserLine} - ${finalErrorMsg}`
+      }
+    }
+  }
+
+  // For stdout, just return the clean error message without the full traceback
+  const cleanedOutput = finalErrorMsg
+
+  return { formattedError, cleanedOutput }
+}
+
 /**
  * Create a detailed error message for users
  */
@@ -429,7 +533,7 @@ function escapeRegExp(string: string): string {
 }
 
 export async function POST(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const startTime = Date.now()
   let stdout = ''
   let userCodeStartLine = 3 // Default value for error reporting
@@ -442,6 +546,8 @@ export async function POST(req: NextRequest) {
       code,
       params = {},
       timeout = 5000,
+      language = DEFAULT_CODE_LANGUAGE,
+      useLocalVM = false,
       envVars = {},
       blockData = {},
       blockNameMapping = {},
@@ -474,19 +580,164 @@ export async function POST(req: NextRequest) {
     resolvedCode = codeResolution.resolvedCode
     const contextVariables = codeResolution.contextVariables
 
-    const executionMethod = 'vm' // Default execution method
+    const e2bEnabled = isTruthy(env.E2B_ENABLED)
+    const lang = isValidCodeLanguage(language) ? language : DEFAULT_CODE_LANGUAGE
+    const useE2B =
+      e2bEnabled &&
+      !useLocalVM &&
+      !isCustomTool &&
+      (lang === CodeLanguage.JavaScript || lang === CodeLanguage.Python)
 
-    logger.info(`[${requestId}] Using VM for code execution`, {
-      hasEnvVars: Object.keys(envVars).length > 0,
-      hasWorkflowVariables: Object.keys(workflowVariables).length > 0,
-    })
+    if (useE2B) {
+      logger.info(`[${requestId}] E2B status`, {
+        enabled: e2bEnabled,
+        hasApiKey: Boolean(process.env.E2B_API_KEY),
+        language: lang,
+      })
+      let prologue = ''
+      const epilogue = ''
 
-    // Create a secure context with console logging
+      if (lang === CodeLanguage.JavaScript) {
+        // Track prologue lines for error adjustment
+        let prologueLineCount = 0
+        prologue += `const params = JSON.parse(${JSON.stringify(JSON.stringify(executionParams))});\n`
+        prologueLineCount++
+        prologue += `const environmentVariables = JSON.parse(${JSON.stringify(JSON.stringify(envVars))});\n`
+        prologueLineCount++
+        for (const [k, v] of Object.entries(contextVariables)) {
+          prologue += `const ${k} = JSON.parse(${JSON.stringify(JSON.stringify(v))});\n`
+          prologueLineCount++
+        }
+        const wrapped = [
+          ';(async () => {',
+          '  try {',
+          '    const __sim_result = await (async () => {',
+          `      ${resolvedCode.split('\n').join('\n      ')}`,
+          '    })();',
+          "    console.log('__SIM_RESULT__=' + JSON.stringify(__sim_result));",
+          '  } catch (error) {',
+          '    console.log(String((error && (error.stack || error.message)) || error));',
+          '    throw error;',
+          '  }',
+          '})();',
+        ].join('\n')
+        const codeForE2B = prologue + wrapped + epilogue
+
+        const execStart = Date.now()
+        const {
+          result: e2bResult,
+          stdout: e2bStdout,
+          sandboxId,
+          error: e2bError,
+        } = await executeInE2B({
+          code: codeForE2B,
+          language: CodeLanguage.JavaScript,
+          timeoutMs: timeout,
+        })
+        const executionTime = Date.now() - execStart
+        stdout += e2bStdout
+
+        logger.info(`[${requestId}] E2B JS sandbox`, {
+          sandboxId,
+          stdoutPreview: e2bStdout?.slice(0, 200),
+          error: e2bError,
+        })
+
+        // If there was an execution error, format it properly
+        if (e2bError) {
+          const { formattedError, cleanedOutput } = formatE2BError(
+            e2bError,
+            e2bStdout,
+            lang,
+            resolvedCode,
+            prologueLineCount
+          )
+          return NextResponse.json(
+            {
+              success: false,
+              error: formattedError,
+              output: { result: null, stdout: cleanedOutput, executionTime },
+            },
+            { status: 500 }
+          )
+        }
+
+        return NextResponse.json({
+          success: true,
+          output: { result: e2bResult ?? null, stdout, executionTime },
+        })
+      }
+      // Track prologue lines for error adjustment
+      let prologueLineCount = 0
+      prologue += 'import json\n'
+      prologueLineCount++
+      prologue += `params = json.loads(${JSON.stringify(JSON.stringify(executionParams))})\n`
+      prologueLineCount++
+      prologue += `environmentVariables = json.loads(${JSON.stringify(JSON.stringify(envVars))})\n`
+      prologueLineCount++
+      for (const [k, v] of Object.entries(contextVariables)) {
+        prologue += `${k} = json.loads(${JSON.stringify(JSON.stringify(v))})\n`
+        prologueLineCount++
+      }
+      const wrapped = [
+        'def __sim_main__():',
+        ...resolvedCode.split('\n').map((l) => `    ${l}`),
+        '__sim_result__ = __sim_main__()',
+        "print('__SIM_RESULT__=' + json.dumps(__sim_result__))",
+      ].join('\n')
+      const codeForE2B = prologue + wrapped + epilogue
+
+      const execStart = Date.now()
+      const {
+        result: e2bResult,
+        stdout: e2bStdout,
+        sandboxId,
+        error: e2bError,
+      } = await executeInE2B({
+        code: codeForE2B,
+        language: CodeLanguage.Python,
+        timeoutMs: timeout,
+      })
+      const executionTime = Date.now() - execStart
+      stdout += e2bStdout
+
+      logger.info(`[${requestId}] E2B Py sandbox`, {
+        sandboxId,
+        stdoutPreview: e2bStdout?.slice(0, 200),
+        error: e2bError,
+      })
+
+      // If there was an execution error, format it properly
+      if (e2bError) {
+        const { formattedError, cleanedOutput } = formatE2BError(
+          e2bError,
+          e2bStdout,
+          lang,
+          resolvedCode,
+          prologueLineCount
+        )
+        return NextResponse.json(
+          {
+            success: false,
+            error: formattedError,
+            output: { result: null, stdout: cleanedOutput, executionTime },
+          },
+          { status: 500 }
+        )
+      }
+
+      return NextResponse.json({
+        success: true,
+        output: { result: e2bResult ?? null, stdout, executionTime },
+      })
+    }
+
+    const executionMethod = 'vm'
     const context = createContext({
       params: executionParams,
       environmentVariables: envVars,
-      ...contextVariables, // Add resolved variables directly to context
-      fetch: globalThis.fetch || require('node-fetch').default,
+      ...contextVariables,
+      fetch: (globalThis as any).fetch || require('node-fetch').default,
       console: {
         log: (...args: any[]) => {
           const logMessage = `${args
@@ -504,23 +755,17 @@ export async function POST(req: NextRequest) {
       },
     })
 
-    // Calculate line offset for user code to provide accurate error reporting
     const wrapperLines = ['(async () => {', '  try {']
-
-    // Add custom tool parameter declarations if needed
     if (isCustomTool) {
       wrapperLines.push('    // For custom tools, make parameters directly accessible')
       Object.keys(executionParams).forEach((key) => {
         wrapperLines.push(`    const ${key} = params.${key};`)
       })
     }
-
-    userCodeStartLine = wrapperLines.length + 1 // +1 because user code starts on next line
-
-    // Build the complete script with proper formatting for line numbers
+    userCodeStartLine = wrapperLines.length + 1
     const fullScript = [
       ...wrapperLines,
-      `    ${resolvedCode.split('\n').join('\n    ')}`, // Indent user code
+      `    ${resolvedCode.split('\n').join('\n    ')}`,
       '  } catch (error) {',
       '    console.error(error);',
       '    throw error;',
@@ -529,33 +774,26 @@ export async function POST(req: NextRequest) {
     ].join('\n')
 
     const script = new Script(fullScript, {
-      filename: 'user-function.js', // This filename will appear in stack traces
-      lineOffset: 0, // Start line numbering from 0
-      columnOffset: 0, // Start column numbering from 0
+      filename: 'user-function.js',
+      lineOffset: 0,
+      columnOffset: 0,
     })
 
     const result = await script.runInContext(context, {
       timeout,
       displayErrors: true,
-      breakOnSigint: true, // Allow breaking on SIGINT for better debugging
+      breakOnSigint: true,
     })
-    // }
 
     const executionTime = Date.now() - startTime
     logger.info(`[${requestId}] Function executed successfully using ${executionMethod}`, {
       executionTime,
     })
 
-    const response = {
+    return NextResponse.json({
       success: true,
-      output: {
-        result,
-        stdout,
-        executionTime,
-      },
-    }
-
-    return NextResponse.json(response)
+      output: { result, stdout, executionTime },
+    })
   } catch (error: any) {
     const executionTime = Date.now() - startTime
     logger.error(`[${requestId}] Function execution failed`, {

apps/sim/app/api/help/route.ts

@@ -7,6 +7,7 @@ import { getFromEmailAddress } from '@/lib/email/utils'
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getEmailDomain } from '@/lib/urls/utils'
+import { generateRequestId } from '@/lib/utils'
 
 const logger = createLogger('HelpAPI')
 
@@ -17,7 +18,7 @@ const helpFormSchema = z.object({
 })
 
 export async function POST(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     // Get user session

apps/sim/app/api/jobs/[jobId]/route.ts

@@ -3,6 +3,7 @@ import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { createErrorResponse } from '@/app/api/workflows/utils'
 import { db } from '@/db'
 import { apiKey as apiKeyTable } from '@/db/schema'
@@ -14,7 +15,7 @@ export async function GET(
   { params }: { params: Promise<{ jobId: string }> }
 ) {
   const { jobId: taskId } = await params
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.debug(`[${requestId}] Getting status for task: ${taskId}`)

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts

@@ -1,12 +1,10 @@
-import { createHash, randomUUID } from 'crypto'
-import { eq, sql } from 'drizzle-orm'
+import { randomUUID } from 'crypto'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import { deleteChunk, updateChunk } from '@/lib/knowledge/chunks/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { checkChunkAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, embedding } from '@/db/schema'
 
 const logger = createLogger('ChunkByIdAPI')
 
@@ -102,33 +100,7 @@ export async function PUT(
     try {
       const validatedData = UpdateChunkSchema.parse(body)
 
-      const updateData: Partial<{
-        content: string
-        contentLength: number
-        tokenCount: number
-        chunkHash: string
-        enabled: boolean
-        updatedAt: Date
-      }> = {}
-
-      if (validatedData.content) {
-        updateData.content = validatedData.content
-        updateData.contentLength = validatedData.content.length
-        // Update token count estimation (rough approximation: 4 chars per token)
-        updateData.tokenCount = Math.ceil(validatedData.content.length / 4)
-        updateData.chunkHash = createHash('sha256').update(validatedData.content).digest('hex')
-      }
-
-      if (validatedData.enabled !== undefined) updateData.enabled = validatedData.enabled
-
-      await db.update(embedding).set(updateData).where(eq(embedding.id, chunkId))
-
-      // Fetch the updated chunk
-      const updatedChunk = await db
-        .select()
-        .from(embedding)
-        .where(eq(embedding.id, chunkId))
-        .limit(1)
+      const updatedChunk = await updateChunk(chunkId, validatedData, requestId)
 
       logger.info(
         `[${requestId}] Chunk updated: ${chunkId} in document ${documentId} in knowledge base ${knowledgeBaseId}`
@@ -136,7 +108,7 @@ export async function PUT(
 
       return NextResponse.json({
         success: true,
-        data: updatedChunk[0],
+        data: updatedChunk,
       })
     } catch (validationError) {
       if (validationError instanceof z.ZodError) {
@@ -190,37 +162,7 @@ export async function DELETE(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Use transaction to atomically delete chunk and update document statistics
-    await db.transaction(async (tx) => {
-      // Get chunk data before deletion for statistics update
-      const chunkToDelete = await tx
-        .select({
-          tokenCount: embedding.tokenCount,
-          contentLength: embedding.contentLength,
-        })
-        .from(embedding)
-        .where(eq(embedding.id, chunkId))
-        .limit(1)
-
-      if (chunkToDelete.length === 0) {
-        throw new Error('Chunk not found')
-      }
-
-      const chunk = chunkToDelete[0]
-
-      // Delete the chunk
-      await tx.delete(embedding).where(eq(embedding.id, chunkId))
-
-      // Update document statistics
-      await tx
-        .update(document)
-        .set({
-          chunkCount: sql`${document.chunkCount} - 1`,
-          tokenCount: sql`${document.tokenCount} - ${chunk.tokenCount}`,
-          characterCount: sql`${document.characterCount} - ${chunk.contentLength}`,
-        })
-        .where(eq(document.id, documentId))
-    })
+    await deleteChunk(chunkId, documentId, requestId)
 
     logger.info(
       `[${requestId}] Chunk deleted: ${chunkId} from document ${documentId} in knowledge base ${knowledgeBaseId}`

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts

@@ -1,378 +0,0 @@
-/**
- * Tests for knowledge document chunks API route
- *
- * @vitest-environment node
- */
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import {
-  createMockRequest,
-  mockAuth,
-  mockConsoleLogger,
-  mockDrizzleOrm,
-  mockKnowledgeSchemas,
-} from '@/app/api/__test-utils__/utils'
-
-mockKnowledgeSchemas()
-mockDrizzleOrm()
-mockConsoleLogger()
-
-vi.mock('@/lib/tokenization/estimators', () => ({
-  estimateTokenCount: vi.fn().mockReturnValue({ count: 452 }),
-}))
-
-vi.mock('@/providers/utils', () => ({
-  calculateCost: vi.fn().mockReturnValue({
-    input: 0.00000904,
-    output: 0,
-    total: 0.00000904,
-    pricing: {
-      input: 0.02,
-      output: 0,
-      updatedAt: '2025-07-10',
-    },
-  }),
-}))
-
-vi.mock('@/app/api/knowledge/utils', () => ({
-  checkKnowledgeBaseAccess: vi.fn(),
-  checkKnowledgeBaseWriteAccess: vi.fn(),
-  checkDocumentAccess: vi.fn(),
-  checkDocumentWriteAccess: vi.fn(),
-  checkChunkAccess: vi.fn(),
-  generateEmbeddings: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3, 0.4, 0.5]]),
-  processDocumentAsync: vi.fn(),
-}))
-
-describe('Knowledge Document Chunks API Route', () => {
-  const mockAuth$ = mockAuth()
-
-  const mockDbChain = {
-    select: vi.fn().mockReturnThis(),
-    from: vi.fn().mockReturnThis(),
-    where: vi.fn().mockReturnThis(),
-    orderBy: vi.fn().mockReturnThis(),
-    limit: vi.fn().mockReturnThis(),
-    offset: vi.fn().mockReturnThis(),
-    insert: vi.fn().mockReturnThis(),
-    values: vi.fn().mockResolvedValue(undefined),
-    update: vi.fn().mockReturnThis(),
-    set: vi.fn().mockReturnThis(),
-    returning: vi.fn().mockResolvedValue([]),
-    delete: vi.fn().mockReturnThis(),
-    transaction: vi.fn(),
-  }
-
-  const mockGetUserId = vi.fn()
-
-  beforeEach(async () => {
-    vi.clearAllMocks()
-
-    vi.doMock('@/db', () => ({
-      db: mockDbChain,
-    }))
-
-    vi.doMock('@/app/api/auth/oauth/utils', () => ({
-      getUserId: mockGetUserId,
-    }))
-
-    Object.values(mockDbChain).forEach((fn) => {
-      if (typeof fn === 'function' && fn !== mockDbChain.values && fn !== mockDbChain.returning) {
-        fn.mockClear().mockReturnThis()
-      }
-    })
-
-    vi.stubGlobal('crypto', {
-      randomUUID: vi.fn().mockReturnValue('mock-chunk-uuid-1234'),
-      createHash: vi.fn().mockReturnValue({
-        update: vi.fn().mockReturnThis(),
-        digest: vi.fn().mockReturnValue('mock-hash-123'),
-      }),
-    })
-  })
-
-  afterEach(() => {
-    vi.clearAllMocks()
-  })
-
-  describe('POST /api/knowledge/[id]/documents/[documentId]/chunks', () => {
-    const validChunkData = {
-      content: 'This is test chunk content for uploading to the knowledge base document.',
-      enabled: true,
-    }
-
-    const mockDocumentAccess = {
-      hasAccess: true,
-      notFound: false,
-      reason: '',
-      document: {
-        id: 'doc-123',
-        processingStatus: 'completed',
-        tag1: 'tag1-value',
-        tag2: 'tag2-value',
-        tag3: null,
-        tag4: null,
-        tag5: null,
-        tag6: null,
-        tag7: null,
-      },
-    }
-
-    const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' })
-
-    it('should create chunk successfully with cost tracking', async () => {
-      const { checkDocumentWriteAccess, generateEmbeddings } = await import(
-        '@/app/api/knowledge/utils'
-      )
-      const { estimateTokenCount } = await import('@/lib/tokenization/estimators')
-      const { calculateCost } = await import('@/providers/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        ...mockDocumentAccess,
-        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
-      } as any)
-
-      // Mock generateEmbeddings
-      vi.mocked(generateEmbeddings).mockResolvedValue([[0.1, 0.2, 0.3]])
-
-      // Mock transaction
-      const mockTx = {
-        select: vi.fn().mockReturnThis(),
-        from: vi.fn().mockReturnThis(),
-        where: vi.fn().mockReturnThis(),
-        orderBy: vi.fn().mockReturnThis(),
-        limit: vi.fn().mockResolvedValue([{ chunkIndex: 0 }]),
-        insert: vi.fn().mockReturnThis(),
-        values: vi.fn().mockResolvedValue(undefined),
-        update: vi.fn().mockReturnThis(),
-        set: vi.fn().mockReturnThis(),
-      }
-
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        return await callback(mockTx)
-      })
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(200)
-      expect(data.success).toBe(true)
-
-      // Verify cost tracking
-      expect(data.data.cost).toBeDefined()
-      expect(data.data.cost.input).toBe(0.00000904)
-      expect(data.data.cost.output).toBe(0)
-      expect(data.data.cost.total).toBe(0.00000904)
-      expect(data.data.cost.tokens).toEqual({
-        prompt: 452,
-        completion: 0,
-        total: 452,
-      })
-      expect(data.data.cost.model).toBe('text-embedding-3-small')
-      expect(data.data.cost.pricing).toEqual({
-        input: 0.02,
-        output: 0,
-        updatedAt: '2025-07-10',
-      })
-
-      // Verify function calls
-      expect(estimateTokenCount).toHaveBeenCalledWith(validChunkData.content, 'openai')
-      expect(calculateCost).toHaveBeenCalledWith('text-embedding-3-small', 452, 0, false)
-    })
-
-    it('should handle workflow-based authentication', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      const workflowData = {
-        ...validChunkData,
-        workflowId: 'workflow-123',
-      }
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        ...mockDocumentAccess,
-        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
-      } as any)
-
-      const mockTx = {
-        select: vi.fn().mockReturnThis(),
-        from: vi.fn().mockReturnThis(),
-        where: vi.fn().mockReturnThis(),
-        orderBy: vi.fn().mockReturnThis(),
-        limit: vi.fn().mockResolvedValue([]),
-        insert: vi.fn().mockReturnThis(),
-        values: vi.fn().mockResolvedValue(undefined),
-        update: vi.fn().mockReturnThis(),
-        set: vi.fn().mockReturnThis(),
-      }
-
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        return await callback(mockTx)
-      })
-
-      const req = createMockRequest('POST', workflowData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(200)
-      expect(data.success).toBe(true)
-      expect(mockGetUserId).toHaveBeenCalledWith(expect.any(String), 'workflow-123')
-    })
-
-    it.concurrent('should return unauthorized for unauthenticated request', async () => {
-      mockGetUserId.mockResolvedValue(null)
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(401)
-      expect(data.error).toBe('Unauthorized')
-    })
-
-    it('should return not found for workflow that does not exist', async () => {
-      const workflowData = {
-        ...validChunkData,
-        workflowId: 'nonexistent-workflow',
-      }
-
-      mockGetUserId.mockResolvedValue(null)
-
-      const req = createMockRequest('POST', workflowData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(404)
-      expect(data.error).toBe('Workflow not found')
-    })
-
-    it.concurrent('should return not found for document access denied', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        hasAccess: false,
-        notFound: true,
-        reason: 'Document not found',
-      })
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(404)
-      expect(data.error).toBe('Document not found')
-    })
-
-    it('should return unauthorized for unauthorized document access', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        hasAccess: false,
-        notFound: false,
-        reason: 'Unauthorized access',
-      })
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(401)
-      expect(data.error).toBe('Unauthorized')
-    })
-
-    it('should reject chunks for failed documents', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        ...mockDocumentAccess,
-        document: {
-          ...mockDocumentAccess.document!,
-          processingStatus: 'failed',
-        },
-        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
-      } as any)
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(400)
-      expect(data.error).toBe('Cannot add chunks to failed document')
-    })
-
-    it.concurrent('should validate chunk data', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        ...mockDocumentAccess,
-        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
-      } as any)
-
-      const invalidData = {
-        content: '', // Empty content
-        enabled: true,
-      }
-
-      const req = createMockRequest('POST', invalidData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      const response = await POST(req, { params: mockParams })
-      const data = await response.json()
-
-      expect(response.status).toBe(400)
-      expect(data.error).toBe('Invalid request data')
-      expect(data.details).toBeDefined()
-    })
-
-    it('should inherit tags from parent document', async () => {
-      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
-
-      mockGetUserId.mockResolvedValue('user-123')
-      vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
-        ...mockDocumentAccess,
-        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
-      } as any)
-
-      const mockTx = {
-        select: vi.fn().mockReturnThis(),
-        from: vi.fn().mockReturnThis(),
-        where: vi.fn().mockReturnThis(),
-        orderBy: vi.fn().mockReturnThis(),
-        limit: vi.fn().mockResolvedValue([]),
-        insert: vi.fn().mockReturnThis(),
-        values: vi.fn().mockImplementation((data) => {
-          // Verify that tags are inherited from document
-          expect(data.tag1).toBe('tag1-value')
-          expect(data.tag2).toBe('tag2-value')
-          expect(data.tag3).toBe(null)
-          return Promise.resolve(undefined)
-        }),
-        update: vi.fn().mockReturnThis(),
-        set: vi.fn().mockReturnThis(),
-      }
-
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        return await callback(mockTx)
-      })
-
-      const req = createMockRequest('POST', validChunkData)
-      const { POST } = await import('@/app/api/knowledge/[id]/documents/[documentId]/chunks/route')
-      await POST(req, { params: mockParams })
-
-      expect(mockTx.values).toHaveBeenCalled()
-    })
-
-    // REMOVED: "should handle cost calculation with different content lengths" test - it was failing
-  })
-})

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts

@@ -1,18 +1,11 @@
-import crypto from 'crypto'
-import { and, asc, eq, ilike, inArray, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import { batchChunkOperation, createChunk, queryChunks } from '@/lib/knowledge/chunks/service'
 import { createLogger } from '@/lib/logs/console/logger'
-import { estimateTokenCount } from '@/lib/tokenization/estimators'
+import { generateRequestId } from '@/lib/utils'
 import { getUserId } from '@/app/api/auth/oauth/utils'
-import {
-  checkDocumentAccess,
-  checkDocumentWriteAccess,
-  generateEmbeddings,
-} from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, embedding } from '@/db/schema'
+import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'
 import { calculateCost } from '@/providers/utils'
 
 const logger = createLogger('DocumentChunksAPI')
@@ -41,7 +34,7 @@ export async function GET(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -66,7 +59,6 @@ export async function GET(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if document processing is completed
     const doc = accessCheck.document
     if (!doc) {
       logger.warn(
@@ -89,7 +81,6 @@ export async function GET(
       )
     }
 
-    // Parse query parameters
     const { searchParams } = new URL(req.url)
     const queryParams = GetChunksQuerySchema.parse({
       search: searchParams.get('search') || undefined,
@@ -98,67 +89,12 @@ export async function GET(
       offset: searchParams.get('offset') || undefined,
     })
 
-    // Build query conditions
-    const conditions = [eq(embedding.documentId, documentId)]
-
-    // Add enabled filter
-    if (queryParams.enabled === 'true') {
-      conditions.push(eq(embedding.enabled, true))
-    } else if (queryParams.enabled === 'false') {
-      conditions.push(eq(embedding.enabled, false))
-    }
-
-    // Add search filter
-    if (queryParams.search) {
-      conditions.push(ilike(embedding.content, `%${queryParams.search}%`))
-    }
-
-    // Fetch chunks
-    const chunks = await db
-      .select({
-        id: embedding.id,
-        chunkIndex: embedding.chunkIndex,
-        content: embedding.content,
-        contentLength: embedding.contentLength,
-        tokenCount: embedding.tokenCount,
-        enabled: embedding.enabled,
-        startOffset: embedding.startOffset,
-        endOffset: embedding.endOffset,
-        tag1: embedding.tag1,
-        tag2: embedding.tag2,
-        tag3: embedding.tag3,
-        tag4: embedding.tag4,
-        tag5: embedding.tag5,
-        tag6: embedding.tag6,
-        tag7: embedding.tag7,
-        createdAt: embedding.createdAt,
-        updatedAt: embedding.updatedAt,
-      })
-      .from(embedding)
-      .where(and(...conditions))
-      .orderBy(asc(embedding.chunkIndex))
-      .limit(queryParams.limit)
-      .offset(queryParams.offset)
-
-    // Get total count for pagination
-    const totalCount = await db
-      .select({ count: sql`count(*)` })
-      .from(embedding)
-      .where(and(...conditions))
-
-    logger.info(
-      `[${requestId}] Retrieved ${chunks.length} chunks for document ${documentId} in knowledge base ${knowledgeBaseId}`
-    )
+    const result = await queryChunks(documentId, queryParams, requestId)
 
     return NextResponse.json({
       success: true,
-      data: chunks,
-      pagination: {
-        total: Number(totalCount[0]?.count || 0),
-        limit: queryParams.limit,
-        offset: queryParams.offset,
-        hasMore: chunks.length === queryParams.limit,
-      },
+      data: result.chunks,
+      pagination: result.pagination,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error fetching chunks`, error)
@@ -170,7 +106,7 @@ export async function POST(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -219,76 +155,27 @@ export async function POST(
     try {
       const validatedData = CreateChunkSchema.parse(searchParams)
 
-      // Generate embedding for the content first (outside transaction for performance)
-      logger.info(`[${requestId}] Generating embedding for manual chunk`)
-      const embeddings = await generateEmbeddings([validatedData.content])
+      const docTags = {
+        tag1: doc.tag1 ?? null,
+        tag2: doc.tag2 ?? null,
+        tag3: doc.tag3 ?? null,
+        tag4: doc.tag4 ?? null,
+        tag5: doc.tag5 ?? null,
+        tag6: doc.tag6 ?? null,
+        tag7: doc.tag7 ?? null,
+      }
 
-      // Calculate accurate token count for both database storage and cost calculation
-      const tokenCount = estimateTokenCount(validatedData.content, 'openai')
+      const newChunk = await createChunk(
+        knowledgeBaseId,
+        documentId,
+        docTags,
+        validatedData,
+        requestId
+      )
 
-      const chunkId = crypto.randomUUID()
-      const now = new Date()
-
-      // Use transaction to atomically get next index and insert chunk
-      const newChunk = await db.transaction(async (tx) => {
-        // Get the next chunk index atomically within the transaction
-        const lastChunk = await tx
-          .select({ chunkIndex: embedding.chunkIndex })
-          .from(embedding)
-          .where(eq(embedding.documentId, documentId))
-          .orderBy(sql`${embedding.chunkIndex} DESC`)
-          .limit(1)
-
-        const nextChunkIndex = lastChunk.length > 0 ? lastChunk[0].chunkIndex + 1 : 0
-
-        const chunkData = {
-          id: chunkId,
-          knowledgeBaseId,
-          documentId,
-          chunkIndex: nextChunkIndex,
-          chunkHash: crypto.createHash('sha256').update(validatedData.content).digest('hex'),
-          content: validatedData.content,
-          contentLength: validatedData.content.length,
-          tokenCount: tokenCount.count, // Use accurate token count
-          embedding: embeddings[0],
-          embeddingModel: 'text-embedding-3-small',
-          startOffset: 0, // Manual chunks don't have document offsets
-          endOffset: validatedData.content.length,
-          // Inherit tags from parent document
-          tag1: doc.tag1,
-          tag2: doc.tag2,
-          tag3: doc.tag3,
-          tag4: doc.tag4,
-          tag5: doc.tag5,
-          tag6: doc.tag6,
-          tag7: doc.tag7,
-          enabled: validatedData.enabled,
-          createdAt: now,
-          updatedAt: now,
-        }
-
-        // Insert the new chunk
-        await tx.insert(embedding).values(chunkData)
-
-        // Update document statistics
-        await tx
-          .update(document)
-          .set({
-            chunkCount: sql`${document.chunkCount} + 1`,
-            tokenCount: sql`${document.tokenCount} + ${chunkData.tokenCount}`,
-            characterCount: sql`${document.characterCount} + ${chunkData.contentLength}`,
-          })
-          .where(eq(document.id, documentId))
-
-        return chunkData
-      })
-
-      logger.info(`[${requestId}] Manual chunk created: ${chunkId} in document ${documentId}`)
-
-      // Calculate cost for the embedding (with fallback if calculation fails)
       let cost = null
       try {
-        cost = calculateCost('text-embedding-3-small', tokenCount.count, 0, false)
+        cost = calculateCost('text-embedding-3-small', newChunk.tokenCount, 0, false)
       } catch (error) {
         logger.warn(`[${requestId}] Failed to calculate cost for chunk upload`, {
           error: error instanceof Error ? error.message : 'Unknown error',
@@ -300,6 +187,8 @@ export async function POST(
         success: true,
         data: {
           ...newChunk,
+          documentId,
+          documentName: doc.filename,
           ...(cost
             ? {
                 cost: {
@@ -307,9 +196,9 @@ export async function POST(
                   output: cost.output,
                   total: cost.total,
                   tokens: {
-                    prompt: tokenCount.count,
+                    prompt: newChunk.tokenCount,
                     completion: 0,
-                    total: tokenCount.count,
+                    total: newChunk.tokenCount,
                   },
                   model: 'text-embedding-3-small',
                   pricing: cost.pricing,
@@ -340,7 +229,7 @@ export async function PATCH(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -371,92 +260,16 @@ export async function PATCH(
       const validatedData = BatchOperationSchema.parse(body)
       const { operation, chunkIds } = validatedData
 
-      logger.info(
-        `[${requestId}] Starting batch ${operation} operation on ${chunkIds.length} chunks for document ${documentId}`
-      )
-
-      const results = []
-      let successCount = 0
-      const errorCount = 0
-
-      if (operation === 'delete') {
-        // Handle batch delete with transaction for consistency
-        await db.transaction(async (tx) => {
-          // Get chunks to delete for statistics update
-          const chunksToDelete = await tx
-            .select({
-              id: embedding.id,
-              tokenCount: embedding.tokenCount,
-              contentLength: embedding.contentLength,
-            })
-            .from(embedding)
-            .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
-
-          if (chunksToDelete.length === 0) {
-            throw new Error('No valid chunks found to delete')
-          }
-
-          // Delete chunks
-          await tx
-            .delete(embedding)
-            .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
-
-          // Update document statistics
-          const totalTokens = chunksToDelete.reduce((sum, chunk) => sum + chunk.tokenCount, 0)
-          const totalCharacters = chunksToDelete.reduce(
-            (sum, chunk) => sum + chunk.contentLength,
-            0
-          )
-
-          await tx
-            .update(document)
-            .set({
-              chunkCount: sql`${document.chunkCount} - ${chunksToDelete.length}`,
-              tokenCount: sql`${document.tokenCount} - ${totalTokens}`,
-              characterCount: sql`${document.characterCount} - ${totalCharacters}`,
-            })
-            .where(eq(document.id, documentId))
-
-          successCount = chunksToDelete.length
-          results.push({
-            operation: 'delete',
-            deletedCount: chunksToDelete.length,
-            chunkIds: chunksToDelete.map((c) => c.id),
-          })
-        })
-      } else {
-        // Handle batch enable/disable
-        const enabled = operation === 'enable'
-
-        // Update chunks in a single query
-        const updateResult = await db
-          .update(embedding)
-          .set({
-            enabled,
-            updatedAt: new Date(),
-          })
-          .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
-          .returning({ id: embedding.id })
-
-        successCount = updateResult.length
-        results.push({
-          operation,
-          updatedCount: updateResult.length,
-          chunkIds: updateResult.map((r) => r.id),
-        })
-      }
-
-      logger.info(
-        `[${requestId}] Batch ${operation} operation completed: ${successCount} successful, ${errorCount} errors`
-      )
+      const result = await batchChunkOperation(documentId, operation, chunkIds, requestId)
 
       return NextResponse.json({
         success: true,
         data: {
           operation,
-          successCount,
-          errorCount,
-          results,
+          successCount: result.processed,
+          errorCount: result.errors.length,
+          processed: result.processed,
+          errors: result.errors,
         },
       })
     } catch (validationError) {

apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.test.ts

@@ -24,7 +24,14 @@ vi.mock('@/app/api/knowledge/utils', () => ({
   processDocumentAsync: vi.fn(),
 }))
 
-// Setup common mocks
+vi.mock('@/lib/knowledge/documents/service', () => ({
+  updateDocument: vi.fn(),
+  deleteDocument: vi.fn(),
+  markDocumentAsFailedTimeout: vi.fn(),
+  retryDocumentProcessing: vi.fn(),
+  processDocumentAsync: vi.fn(),
+}))
+
 mockDrizzleOrm()
 mockConsoleLogger()
 
@@ -42,8 +49,6 @@ describe('Document By ID API Route', () => {
     transaction: vi.fn(),
   }
 
-  // Mock functions will be imported dynamically in tests
-
   const mockDocument = {
     id: 'doc-123',
     knowledgeBaseId: 'kb-123',
@@ -73,7 +78,6 @@ describe('Document By ID API Route', () => {
         }
       }
     })
-    // Mock functions are cleared automatically by vitest
   }
 
   beforeEach(async () => {
@@ -83,8 +87,6 @@ describe('Document By ID API Route', () => {
       db: mockDbChain,
     }))
 
-    // Utils are mocked at the top level
-
     vi.stubGlobal('crypto', {
       randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'),
     })
@@ -195,6 +197,7 @@ describe('Document By ID API Route', () => {
 
     it('should update document successfully', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { updateDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
@@ -203,31 +206,12 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Create a sequence of mocks for the database operations
-      const updateChain = {
-        set: vi.fn().mockReturnValue({
-          where: vi.fn().mockResolvedValue(undefined), // Update operation completes
-        }),
+      const updatedDocument = {
+        ...mockDocument,
+        ...validUpdateData,
+        deletedAt: null,
       }
-
-      const selectChain = {
-        from: vi.fn().mockReturnValue({
-          where: vi.fn().mockReturnValue({
-            limit: vi.fn().mockResolvedValue([{ ...mockDocument, ...validUpdateData }]),
-          }),
-        }),
-      }
-
-      // Mock transaction
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        const mockTx = {
-          update: vi.fn().mockReturnValue(updateChain),
-        }
-        await callback(mockTx)
-      })
-
-      // Mock db operations in sequence
-      mockDbChain.select.mockReturnValue(selectChain)
+      vi.mocked(updateDocument).mockResolvedValue(updatedDocument)
 
       const req = createMockRequest('PUT', validUpdateData)
       const { PUT } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
@@ -238,8 +222,11 @@ describe('Document By ID API Route', () => {
       expect(data.success).toBe(true)
       expect(data.data.filename).toBe('updated-document.pdf')
       expect(data.data.enabled).toBe(false)
-      expect(mockDbChain.transaction).toHaveBeenCalled()
-      expect(mockDbChain.select).toHaveBeenCalled()
+      expect(vi.mocked(updateDocument)).toHaveBeenCalledWith(
+        'doc-123',
+        validUpdateData,
+        expect.any(String)
+      )
     })
 
     it('should validate update data', async () => {
@@ -274,6 +261,7 @@ describe('Document By ID API Route', () => {
 
     it('should mark document as failed due to timeout successfully', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { markDocumentAsFailedTimeout } = await import('@/lib/knowledge/documents/service')
 
       const processingDocument = {
         ...mockDocument,
@@ -288,34 +276,11 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Create a sequence of mocks for the database operations
-      const updateChain = {
-        set: vi.fn().mockReturnValue({
-          where: vi.fn().mockResolvedValue(undefined), // Update operation completes
-        }),
-      }
-
-      const selectChain = {
-        from: vi.fn().mockReturnValue({
-          where: vi.fn().mockReturnValue({
-            limit: vi
-              .fn()
-              .mockResolvedValue([{ ...processingDocument, processingStatus: 'failed' }]),
-          }),
-        }),
-      }
-
-      // Mock transaction
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        const mockTx = {
-          update: vi.fn().mockReturnValue(updateChain),
-        }
-        await callback(mockTx)
+      vi.mocked(markDocumentAsFailedTimeout).mockResolvedValue({
+        success: true,
+        processingDuration: 200000,
       })
 
-      // Mock db operations in sequence
-      mockDbChain.select.mockReturnValue(selectChain)
-
       const req = createMockRequest('PUT', { markFailedDueToTimeout: true })
       const { PUT } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
       const response = await PUT(req, { params: mockParams })
@@ -323,13 +288,13 @@ describe('Document By ID API Route', () => {
 
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
-      expect(mockDbChain.transaction).toHaveBeenCalled()
-      expect(updateChain.set).toHaveBeenCalledWith(
-        expect.objectContaining({
-          processingStatus: 'failed',
-          processingError: 'Processing timed out - background process may have been terminated',
-          processingCompletedAt: expect.any(Date),
-        })
+      expect(data.data.documentId).toBe('doc-123')
+      expect(data.data.status).toBe('failed')
+      expect(data.data.message).toBe('Document marked as failed due to timeout')
+      expect(vi.mocked(markDocumentAsFailedTimeout)).toHaveBeenCalledWith(
+        'doc-123',
+        processingDocument.processingStartedAt,
+        expect.any(String)
       )
     })
 
@@ -354,6 +319,7 @@ describe('Document By ID API Route', () => {
 
     it('should reject marking failed for recently started processing', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { markDocumentAsFailedTimeout } = await import('@/lib/knowledge/documents/service')
 
       const recentProcessingDocument = {
         ...mockDocument,
@@ -368,6 +334,10 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
+      vi.mocked(markDocumentAsFailedTimeout).mockRejectedValue(
+        new Error('Document has not been processing long enough to be considered dead')
+      )
+
       const req = createMockRequest('PUT', { markFailedDueToTimeout: true })
       const { PUT } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
       const response = await PUT(req, { params: mockParams })
@@ -382,9 +352,8 @@ describe('Document By ID API Route', () => {
     const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' })
 
     it('should retry processing successfully', async () => {
-      const { checkDocumentWriteAccess, processDocumentAsync } = await import(
-        '@/app/api/knowledge/utils'
-      )
+      const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { retryDocumentProcessing } = await import('@/lib/knowledge/documents/service')
 
       const failedDocument = {
         ...mockDocument,
@@ -399,23 +368,12 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock transaction
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        const mockTx = {
-          delete: vi.fn().mockReturnValue({
-            where: vi.fn().mockResolvedValue(undefined),
-          }),
-          update: vi.fn().mockReturnValue({
-            set: vi.fn().mockReturnValue({
-              where: vi.fn().mockResolvedValue(undefined),
-            }),
-          }),
-        }
-        return await callback(mockTx)
+      vi.mocked(retryDocumentProcessing).mockResolvedValue({
+        success: true,
+        status: 'pending',
+        message: 'Document retry processing started',
       })
 
-      vi.mocked(processDocumentAsync).mockResolvedValue(undefined)
-
       const req = createMockRequest('PUT', { retryProcessing: true })
       const { PUT } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
       const response = await PUT(req, { params: mockParams })
@@ -425,8 +383,17 @@ describe('Document By ID API Route', () => {
       expect(data.success).toBe(true)
       expect(data.data.status).toBe('pending')
       expect(data.data.message).toBe('Document retry processing started')
-      expect(mockDbChain.transaction).toHaveBeenCalled()
-      expect(vi.mocked(processDocumentAsync)).toHaveBeenCalled()
+      expect(vi.mocked(retryDocumentProcessing)).toHaveBeenCalledWith(
+        'kb-123',
+        'doc-123',
+        {
+          filename: failedDocument.filename,
+          fileUrl: failedDocument.fileUrl,
+          fileSize: failedDocument.fileSize,
+          mimeType: failedDocument.mimeType,
+        },
+        expect.any(String)
+      )
     })
 
     it('should reject retry for non-failed document', async () => {
@@ -486,6 +453,7 @@ describe('Document By ID API Route', () => {
 
     it('should handle database errors during update', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { updateDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
@@ -494,8 +462,7 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock transaction to throw an error
-      mockDbChain.transaction.mockRejectedValue(new Error('Database error'))
+      vi.mocked(updateDocument).mockRejectedValue(new Error('Database error'))
 
       const req = createMockRequest('PUT', validUpdateData)
       const { PUT } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
@@ -512,6 +479,7 @@ describe('Document By ID API Route', () => {
 
     it('should delete document successfully', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { deleteDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
@@ -520,10 +488,10 @@ describe('Document By ID API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Properly chain the mock database operations for soft delete
-      mockDbChain.update.mockReturnValue(mockDbChain)
-      mockDbChain.set.mockReturnValue(mockDbChain)
-      mockDbChain.where.mockResolvedValue(undefined) // Update operation resolves
+      vi.mocked(deleteDocument).mockResolvedValue({
+        success: true,
+        message: 'Document deleted successfully',
+      })
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')
@@ -533,12 +501,7 @@ describe('Document By ID API Route', () => {
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.message).toBe('Document deleted successfully')
-      expect(mockDbChain.update).toHaveBeenCalled()
-      expect(mockDbChain.set).toHaveBeenCalledWith(
-        expect.objectContaining({
-          deletedAt: expect.any(Date),
-        })
-      )
+      expect(vi.mocked(deleteDocument)).toHaveBeenCalledWith('doc-123', expect.any(String))
     })
 
     it('should return unauthorized for unauthenticated user', async () => {
@@ -592,6 +555,7 @@ describe('Document By ID API Route', () => {
 
     it('should handle database errors during deletion', async () => {
       const { checkDocumentWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { deleteDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkDocumentWriteAccess).mockResolvedValue({
@@ -599,7 +563,7 @@ describe('Document By ID API Route', () => {
         document: mockDocument,
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
-      mockDbChain.set.mockRejectedValue(new Error('Database error'))
+      vi.mocked(deleteDocument).mockRejectedValue(new Error('Database error'))
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/documents/[documentId]/route')

apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts

@@ -1,16 +1,15 @@
-import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
-import { TAG_SLOTS } from '@/lib/constants/knowledge'
-import { createLogger } from '@/lib/logs/console/logger'
 import {
-  checkDocumentAccess,
-  checkDocumentWriteAccess,
-  processDocumentAsync,
-} from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, embedding } from '@/db/schema'
+  deleteDocument,
+  markDocumentAsFailedTimeout,
+  retryDocumentProcessing,
+  updateDocument,
+} from '@/lib/knowledge/documents/service'
+import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
+import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'
 
 const logger = createLogger('DocumentByIdAPI')
 
@@ -38,7 +37,7 @@ export async function GET(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -81,7 +80,7 @@ export async function PUT(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -113,9 +112,7 @@ export async function PUT(
 
       const updateData: any = {}
 
-      // Handle special operations first
       if (validatedData.markFailedDueToTimeout) {
-        // Mark document as failed due to timeout (replaces mark-failed endpoint)
         const doc = accessCheck.document
 
         if (doc.processingStatus !== 'processing') {
@@ -132,58 +129,30 @@ export async function PUT(
           )
         }
 
-        const now = new Date()
-        const processingDuration = now.getTime() - new Date(doc.processingStartedAt).getTime()
-        const DEAD_PROCESS_THRESHOLD_MS = 150 * 1000
+        try {
+          await markDocumentAsFailedTimeout(documentId, doc.processingStartedAt, requestId)
 
-        if (processingDuration <= DEAD_PROCESS_THRESHOLD_MS) {
-          return NextResponse.json(
-            { error: 'Document has not been processing long enough to be considered dead' },
-            { status: 400 }
-          )
+          return NextResponse.json({
+            success: true,
+            data: {
+              documentId,
+              status: 'failed',
+              message: 'Document marked as failed due to timeout',
+            },
+          })
+        } catch (error) {
+          if (error instanceof Error) {
+            return NextResponse.json({ error: error.message }, { status: 400 })
+          }
+          throw error
         }
-
-        updateData.processingStatus = 'failed'
-        updateData.processingError =
-          'Processing timed out - background process may have been terminated'
-        updateData.processingCompletedAt = now
-
-        logger.info(
-          `[${requestId}] Marked document ${documentId} as failed due to dead process (processing time: ${Math.round(processingDuration / 1000)}s)`
-        )
       } else if (validatedData.retryProcessing) {
-        // Retry processing (replaces retry endpoint)
         const doc = accessCheck.document
 
         if (doc.processingStatus !== 'failed') {
           return NextResponse.json({ error: 'Document is not in failed state' }, { status: 400 })
         }
 
-        // Clear existing embeddings and reset document state
-        await db.transaction(async (tx) => {
-          await tx.delete(embedding).where(eq(embedding.documentId, documentId))
-
-          await tx
-            .update(document)
-            .set({
-              processingStatus: 'pending',
-              processingStartedAt: null,
-              processingCompletedAt: null,
-              processingError: null,
-              chunkCount: 0,
-              tokenCount: 0,
-              characterCount: 0,
-            })
-            .where(eq(document.id, documentId))
-        })
-
-        const processingOptions = {
-          chunkSize: 1024,
-          minCharactersPerChunk: 24,
-          recipe: 'default',
-          lang: 'en',
-        }
-
         const docData = {
           filename: doc.filename,
           fileUrl: doc.fileUrl,
@@ -191,80 +160,33 @@ export async function PUT(
           mimeType: doc.mimeType,
         }
 
-        processDocumentAsync(knowledgeBaseId, documentId, docData, processingOptions).catch(
-          (error: unknown) => {
-            logger.error(`[${requestId}] Background retry processing error:`, error)
-          }
+        const result = await retryDocumentProcessing(
+          knowledgeBaseId,
+          documentId,
+          docData,
+          requestId
         )
 
-        logger.info(`[${requestId}] Document retry initiated: ${documentId}`)
-
         return NextResponse.json({
           success: true,
           data: {
             documentId,
-            status: 'pending',
-            message: 'Document retry processing started',
+            status: result.status,
+            message: result.message,
           },
         })
       } else {
-        // Regular field updates
-        if (validatedData.filename !== undefined) updateData.filename = validatedData.filename
-        if (validatedData.enabled !== undefined) updateData.enabled = validatedData.enabled
-        if (validatedData.chunkCount !== undefined) updateData.chunkCount = validatedData.chunkCount
-        if (validatedData.tokenCount !== undefined) updateData.tokenCount = validatedData.tokenCount
-        if (validatedData.characterCount !== undefined)
-          updateData.characterCount = validatedData.characterCount
-        if (validatedData.processingStatus !== undefined)
-          updateData.processingStatus = validatedData.processingStatus
-        if (validatedData.processingError !== undefined)
-          updateData.processingError = validatedData.processingError
+        const updatedDocument = await updateDocument(documentId, validatedData, requestId)
 
-        // Tag field updates
-        TAG_SLOTS.forEach((slot) => {
-          if ((validatedData as any)[slot] !== undefined) {
-            ;(updateData as any)[slot] = (validatedData as any)[slot]
-          }
+        logger.info(
+          `[${requestId}] Document updated: ${documentId} in knowledge base ${knowledgeBaseId}`
+        )
+
+        return NextResponse.json({
+          success: true,
+          data: updatedDocument,
         })
       }
-
-      await db.transaction(async (tx) => {
-        // Update the document
-        await tx.update(document).set(updateData).where(eq(document.id, documentId))
-
-        // If any tag fields were updated, also update the embeddings
-        const hasTagUpdates = TAG_SLOTS.some((field) => (validatedData as any)[field] !== undefined)
-
-        if (hasTagUpdates) {
-          const embeddingUpdateData: Record<string, string | null> = {}
-          TAG_SLOTS.forEach((field) => {
-            if ((validatedData as any)[field] !== undefined) {
-              embeddingUpdateData[field] = (validatedData as any)[field] || null
-            }
-          })
-
-          await tx
-            .update(embedding)
-            .set(embeddingUpdateData)
-            .where(eq(embedding.documentId, documentId))
-        }
-      })
-
-      // Fetch the updated document
-      const updatedDocument = await db
-        .select()
-        .from(document)
-        .where(eq(document.id, documentId))
-        .limit(1)
-
-      logger.info(
-        `[${requestId}] Document updated: ${documentId} in knowledge base ${knowledgeBaseId}`
-      )
-
-      return NextResponse.json({
-        success: true,
-        data: updatedDocument[0],
-      })
     } catch (validationError) {
       if (validationError instanceof z.ZodError) {
         logger.warn(`[${requestId}] Invalid document update data`, {
@@ -288,7 +210,7 @@ export async function DELETE(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id: knowledgeBaseId, documentId } = await params
 
   try {
@@ -313,13 +235,7 @@ export async function DELETE(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Soft delete by setting deletedAt timestamp
-    await db
-      .update(document)
-      .set({
-        deletedAt: new Date(),
-      })
-      .where(eq(document.id, documentId))
+    const result = await deleteDocument(documentId, requestId)
 
     logger.info(
       `[${requestId}] Document deleted: ${documentId} from knowledge base ${knowledgeBaseId}`
@@ -327,7 +243,7 @@ export async function DELETE(
 
     return NextResponse.json({
       success: true,
-      data: { message: 'Document deleted successfully' },
+      data: result,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error deleting document`, error)

apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts

@@ -1,17 +1,17 @@
 import { randomUUID } from 'crypto'
-import { and, eq, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/consts'
 import {
-  getMaxSlotsForFieldType,
-  getSlotsForFieldType,
-  SUPPORTED_FIELD_TYPES,
-} from '@/lib/constants/knowledge'
+  cleanupUnusedTagDefinitions,
+  createOrUpdateTagDefinitionsBulk,
+  deleteAllTagDefinitions,
+  getDocumentTagDefinitions,
+} from '@/lib/knowledge/tags/service'
+import type { BulkTagDefinitionsData } from '@/lib/knowledge/tags/types'
 import { createLogger } from '@/lib/logs/console/logger'
-import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, knowledgeBaseTagDefinitions } from '@/db/schema'
+import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'
 
 export const dynamic = 'force-dynamic'
 
@@ -29,106 +29,6 @@ const BulkTagDefinitionsSchema = z.object({
   definitions: z.array(TagDefinitionSchema),
 })
 
-// Helper function to get the next available slot for a knowledge base and field type
-async function getNextAvailableSlot(
-  knowledgeBaseId: string,
-  fieldType: string,
-  existingBySlot?: Map<string, any>
-): Promise<string | null> {
-  // Get available slots for this field type
-  const availableSlots = getSlotsForFieldType(fieldType)
-  let usedSlots: Set<string>
-
-  if (existingBySlot) {
-    // Use provided map if available (for performance in batch operations)
-    // Filter by field type
-    usedSlots = new Set(
-      Array.from(existingBySlot.entries())
-        .filter(([_, def]) => def.fieldType === fieldType)
-        .map(([slot, _]) => slot)
-    )
-  } else {
-    // Query database for existing tag definitions of the same field type
-    const existingDefinitions = await db
-      .select({ tagSlot: knowledgeBaseTagDefinitions.tagSlot })
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
-          eq(knowledgeBaseTagDefinitions.fieldType, fieldType)
-        )
-      )
-
-    usedSlots = new Set(existingDefinitions.map((def) => def.tagSlot))
-  }
-
-  // Find the first available slot for this field type
-  for (const slot of availableSlots) {
-    if (!usedSlots.has(slot)) {
-      return slot
-    }
-  }
-
-  return null // No available slots for this field type
-}
-
-// Helper function to clean up unused tag definitions
-async function cleanupUnusedTagDefinitions(knowledgeBaseId: string, requestId: string) {
-  try {
-    logger.info(`[${requestId}] Starting cleanup for KB ${knowledgeBaseId}`)
-
-    // Get all tag definitions for this KB
-    const allDefinitions = await db
-      .select()
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
-
-    logger.info(`[${requestId}] Found ${allDefinitions.length} tag definitions to check`)
-
-    if (allDefinitions.length === 0) {
-      return 0
-    }
-
-    let cleanedCount = 0
-
-    // For each tag definition, check if any documents use that tag slot
-    for (const definition of allDefinitions) {
-      const slot = definition.tagSlot
-
-      // Use raw SQL with proper column name injection
-      const countResult = await db.execute(sql`
-        SELECT count(*) as count
-        FROM document
-        WHERE knowledge_base_id = ${knowledgeBaseId}
-        AND ${sql.raw(slot)} IS NOT NULL
-        AND trim(${sql.raw(slot)}) != ''
-      `)
-      const count = Number(countResult[0]?.count) || 0
-
-      logger.info(
-        `[${requestId}] Tag ${definition.displayName} (${slot}): ${count} documents using it`
-      )
-
-      // If count is 0, remove this tag definition
-      if (count === 0) {
-        await db
-          .delete(knowledgeBaseTagDefinitions)
-          .where(eq(knowledgeBaseTagDefinitions.id, definition.id))
-
-        cleanedCount++
-        logger.info(
-          `[${requestId}] Removed unused tag definition: ${definition.displayName} (${definition.tagSlot})`
-        )
-      }
-    }
-
-    return cleanedCount
-  } catch (error) {
-    logger.warn(`[${requestId}] Failed to cleanup unused tag definitions:`, error)
-    return 0 // Don't fail the main operation if cleanup fails
-  }
-}
-
 // GET /api/knowledge/[id]/documents/[documentId]/tag-definitions - Get tag definitions for a document
 export async function GET(
   req: NextRequest,
@@ -145,35 +45,22 @@ export async function GET(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has access to the knowledge base
-    const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
-    if (!accessCheck.hasAccess) {
-      return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
-    }
-
     // Verify document exists and belongs to the knowledge base
-    const documentExists = await db
-      .select({ id: document.id })
-      .from(document)
-      .where(and(eq(document.id, documentId), eq(document.knowledgeBaseId, knowledgeBaseId)))
-      .limit(1)
-
-    if (documentExists.length === 0) {
-      return NextResponse.json({ error: 'Document not found' }, { status: 404 })
+    const accessCheck = await checkDocumentAccess(knowledgeBaseId, documentId, session.user.id)
+    if (!accessCheck.hasAccess) {
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
+      logger.warn(
+        `[${requestId}] User ${session.user.id} attempted unauthorized document access: ${accessCheck.reason}`
+      )
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Get tag definitions for the knowledge base
-    const tagDefinitions = await db
-      .select({
-        id: knowledgeBaseTagDefinitions.id,
-        tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-        displayName: knowledgeBaseTagDefinitions.displayName,
-        fieldType: knowledgeBaseTagDefinitions.fieldType,
-        createdAt: knowledgeBaseTagDefinitions.createdAt,
-        updatedAt: knowledgeBaseTagDefinitions.updatedAt,
-      })
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+    const tagDefinitions = await getDocumentTagDefinitions(knowledgeBaseId)
 
     logger.info(`[${requestId}] Retrieved ${tagDefinitions.length} tag definitions`)
 
@@ -203,21 +90,19 @@ export async function POST(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has write access to the knowledge base
-    const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, session.user.id)
+    // Verify document exists and user has write access
+    const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, session.user.id)
     if (!accessCheck.hasAccess) {
-      return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
-    }
-
-    // Verify document exists and belongs to the knowledge base
-    const documentExists = await db
-      .select({ id: document.id })
-      .from(document)
-      .where(and(eq(document.id, documentId), eq(document.knowledgeBaseId, knowledgeBaseId)))
-      .limit(1)
-
-    if (documentExists.length === 0) {
-      return NextResponse.json({ error: 'Document not found' }, { status: 404 })
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
+      logger.warn(
+        `[${requestId}] User ${session.user.id} attempted unauthorized document write access: ${accessCheck.reason}`
+      )
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
     let body
@@ -238,197 +123,24 @@ export async function POST(
 
     const validatedData = BulkTagDefinitionsSchema.parse(body)
 
-    // Validate slots are valid for their field types
-    for (const definition of validatedData.definitions) {
-      const validSlots = getSlotsForFieldType(definition.fieldType)
-      if (validSlots.length === 0) {
-        return NextResponse.json(
-          { error: `Unsupported field type: ${definition.fieldType}` },
-          { status: 400 }
-        )
-      }
-
-      if (!validSlots.includes(definition.tagSlot)) {
-        return NextResponse.json(
-          {
-            error: `Invalid slot '${definition.tagSlot}' for field type '${definition.fieldType}'. Valid slots: ${validSlots.join(', ')}`,
-          },
-          { status: 400 }
-        )
-      }
+    const bulkData: BulkTagDefinitionsData = {
+      definitions: validatedData.definitions.map((def) => ({
+        tagSlot: def.tagSlot,
+        displayName: def.displayName,
+        fieldType: def.fieldType,
+        originalDisplayName: def._originalDisplayName,
+      })),
     }
 
-    // Validate no duplicate tag slots within the same field type
-    const slotsByFieldType = new Map<string, Set<string>>()
-    for (const definition of validatedData.definitions) {
-      if (!slotsByFieldType.has(definition.fieldType)) {
-        slotsByFieldType.set(definition.fieldType, new Set())
-      }
-      const slotsForType = slotsByFieldType.get(definition.fieldType)!
-      if (slotsForType.has(definition.tagSlot)) {
-        return NextResponse.json(
-          {
-            error: `Duplicate slot '${definition.tagSlot}' for field type '${definition.fieldType}'`,
-          },
-          { status: 400 }
-        )
-      }
-      slotsForType.add(definition.tagSlot)
-    }
-
-    const now = new Date()
-    const createdDefinitions: (typeof knowledgeBaseTagDefinitions.$inferSelect)[] = []
-
-    // Get existing definitions
-    const existingDefinitions = await db
-      .select()
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
-
-    // Group by field type for validation
-    const existingByFieldType = new Map<string, number>()
-    for (const def of existingDefinitions) {
-      existingByFieldType.set(def.fieldType, (existingByFieldType.get(def.fieldType) || 0) + 1)
-    }
-
-    // Validate we don't exceed limits per field type
-    const newByFieldType = new Map<string, number>()
-    for (const definition of validatedData.definitions) {
-      // Skip validation for edit operations - they don't create new slots
-      if (definition._originalDisplayName) {
-        continue
-      }
-
-      const existingTagNames = new Set(
-        existingDefinitions
-          .filter((def) => def.fieldType === definition.fieldType)
-          .map((def) => def.displayName)
-      )
-
-      if (!existingTagNames.has(definition.displayName)) {
-        newByFieldType.set(
-          definition.fieldType,
-          (newByFieldType.get(definition.fieldType) || 0) + 1
-        )
-      }
-    }
-
-    for (const [fieldType, newCount] of newByFieldType.entries()) {
-      const existingCount = existingByFieldType.get(fieldType) || 0
-      const maxSlots = getMaxSlotsForFieldType(fieldType)
-
-      if (existingCount + newCount > maxSlots) {
-        return NextResponse.json(
-          {
-            error: `Cannot create ${newCount} new '${fieldType}' tags. Knowledge base already has ${existingCount} '${fieldType}' tag definitions. Maximum is ${maxSlots} per field type.`,
-          },
-          { status: 400 }
-        )
-      }
-    }
-
-    // Use transaction to ensure consistency
-    await db.transaction(async (tx) => {
-      // Create maps for lookups
-      const existingByName = new Map(existingDefinitions.map((def) => [def.displayName, def]))
-      const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot, def]))
-
-      // Process each definition
-      for (const definition of validatedData.definitions) {
-        if (definition._originalDisplayName) {
-          // This is an EDIT operation - find by original name and update
-          const originalDefinition = existingByName.get(definition._originalDisplayName)
-
-          if (originalDefinition) {
-            logger.info(
-              `[${requestId}] Editing tag definition: ${definition._originalDisplayName} -> ${definition.displayName} (slot ${originalDefinition.tagSlot})`
-            )
-
-            await tx
-              .update(knowledgeBaseTagDefinitions)
-              .set({
-                displayName: definition.displayName,
-                fieldType: definition.fieldType,
-                updatedAt: now,
-              })
-              .where(eq(knowledgeBaseTagDefinitions.id, originalDefinition.id))
-
-            createdDefinitions.push({
-              ...originalDefinition,
-              displayName: definition.displayName,
-              fieldType: definition.fieldType,
-              updatedAt: now,
-            })
-            continue
-          }
-          logger.warn(
-            `[${requestId}] Could not find original definition for: ${definition._originalDisplayName}`
-          )
-        }
-
-        // Regular create/update logic
-        const existingByDisplayName = existingByName.get(definition.displayName)
-
-        if (existingByDisplayName) {
-          // Display name exists - UPDATE operation
-          logger.info(
-            `[${requestId}] Updating existing tag definition: ${definition.displayName} (slot ${existingByDisplayName.tagSlot})`
-          )
-
-          await tx
-            .update(knowledgeBaseTagDefinitions)
-            .set({
-              fieldType: definition.fieldType,
-              updatedAt: now,
-            })
-            .where(eq(knowledgeBaseTagDefinitions.id, existingByDisplayName.id))
-
-          createdDefinitions.push({
-            ...existingByDisplayName,
-            fieldType: definition.fieldType,
-            updatedAt: now,
-          })
-        } else {
-          // Display name doesn't exist - CREATE operation
-          const targetSlot = await getNextAvailableSlot(
-            knowledgeBaseId,
-            definition.fieldType,
-            existingBySlot
-          )
-
-          if (!targetSlot) {
-            logger.error(
-              `[${requestId}] No available slots for new tag definition: ${definition.displayName}`
-            )
-            continue
-          }
-
-          logger.info(
-            `[${requestId}] Creating new tag definition: ${definition.displayName} -> ${targetSlot}`
-          )
-
-          const newDefinition = {
-            id: randomUUID(),
-            knowledgeBaseId,
-            tagSlot: targetSlot as any,
-            displayName: definition.displayName,
-            fieldType: definition.fieldType,
-            createdAt: now,
-            updatedAt: now,
-          }
-
-          await tx.insert(knowledgeBaseTagDefinitions).values(newDefinition)
-          existingBySlot.set(targetSlot as any, newDefinition)
-          createdDefinitions.push(newDefinition as any)
-        }
-      }
-    })
-
-    logger.info(`[${requestId}] Created/updated ${createdDefinitions.length} tag definitions`)
+    const result = await createOrUpdateTagDefinitionsBulk(knowledgeBaseId, bulkData, requestId)
 
     return NextResponse.json({
       success: true,
-      data: createdDefinitions,
+      data: {
+        created: result.created,
+        updated: result.updated,
+        errors: result.errors,
+      },
     })
   } catch (error) {
     if (error instanceof z.ZodError) {
@@ -459,10 +171,19 @@ export async function DELETE(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has write access to the knowledge base
-    const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, session.user.id)
+    // Verify document exists and user has write access
+    const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, session.user.id)
     if (!accessCheck.hasAccess) {
-      return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
+      logger.warn(
+        `[${requestId}] User ${session.user.id} attempted unauthorized document write access: ${accessCheck.reason}`
+      )
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
     if (action === 'cleanup') {
@@ -478,13 +199,12 @@ export async function DELETE(
     // Delete all tag definitions (original behavior)
     logger.info(`[${requestId}] Deleting all tag definitions for KB ${knowledgeBaseId}`)
 
-    const result = await db
-      .delete(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+    const deletedCount = await deleteAllTagDefinitions(knowledgeBaseId, requestId)
 
     return NextResponse.json({
       success: true,
       message: 'Tag definitions deleted successfully',
+      data: { deleted: deletedCount },
     })
   } catch (error) {
     logger.error(`[${requestId}] Error with tag definitions operation`, error)

apps/sim/app/api/knowledge/[id]/documents/route.test.ts

@@ -24,6 +24,19 @@ vi.mock('@/app/api/knowledge/utils', () => ({
   processDocumentAsync: vi.fn(),
 }))
 
+vi.mock('@/lib/knowledge/documents/service', () => ({
+  getDocuments: vi.fn(),
+  createSingleDocument: vi.fn(),
+  createDocumentRecords: vi.fn(),
+  processDocumentsWithQueue: vi.fn(),
+  getProcessingConfig: vi.fn(),
+  bulkDocumentOperation: vi.fn(),
+  updateDocument: vi.fn(),
+  deleteDocument: vi.fn(),
+  markDocumentAsFailedTimeout: vi.fn(),
+  retryDocumentProcessing: vi.fn(),
+}))
+
 mockDrizzleOrm()
 mockConsoleLogger()
 
@@ -72,7 +85,6 @@ describe('Knowledge Base Documents API Route', () => {
         }
       }
     })
-    // Clear all mocks - they will be set up in individual tests
   }
 
   beforeEach(async () => {
@@ -96,6 +108,7 @@ describe('Knowledge Base Documents API Route', () => {
 
     it('should retrieve documents successfully for authenticated user', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
+      const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({
@@ -103,11 +116,15 @@ describe('Knowledge Base Documents API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock the count query (first query)
-      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
-
-      // Mock the documents query (second query)
-      mockDbChain.offset.mockResolvedValue([mockDocument])
+      vi.mocked(getDocuments).mockResolvedValue({
+        documents: [mockDocument],
+        pagination: {
+          total: 1,
+          limit: 50,
+          offset: 0,
+          hasMore: false,
+        },
+      })
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -118,12 +135,22 @@ describe('Knowledge Base Documents API Route', () => {
       expect(data.success).toBe(true)
       expect(data.data.documents).toHaveLength(1)
       expect(data.data.documents[0].id).toBe('doc-123')
-      expect(mockDbChain.select).toHaveBeenCalled()
       expect(vi.mocked(checkKnowledgeBaseAccess)).toHaveBeenCalledWith('kb-123', 'user-123')
+      expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
+        'kb-123',
+        {
+          includeDisabled: false,
+          search: undefined,
+          limit: 50,
+          offset: 0,
+        },
+        expect.any(String)
+      )
     })
 
     it('should filter disabled documents by default', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
+      const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({
@@ -131,22 +158,36 @@ describe('Knowledge Base Documents API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock the count query (first query)
-      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
-
-      // Mock the documents query (second query)
-      mockDbChain.offset.mockResolvedValue([mockDocument])
+      vi.mocked(getDocuments).mockResolvedValue({
+        documents: [mockDocument],
+        pagination: {
+          total: 1,
+          limit: 50,
+          offset: 0,
+          hasMore: false,
+        },
+      })
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/documents/route')
       const response = await GET(req, { params: mockParams })
 
       expect(response.status).toBe(200)
-      expect(mockDbChain.where).toHaveBeenCalled()
+      expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
+        'kb-123',
+        {
+          includeDisabled: false,
+          search: undefined,
+          limit: 50,
+          offset: 0,
+        },
+        expect.any(String)
+      )
     })
 
     it('should include disabled documents when requested', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
+      const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({
@@ -154,11 +195,15 @@ describe('Knowledge Base Documents API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock the count query (first query)
-      mockDbChain.where.mockResolvedValueOnce([{ count: 1 }])
-
-      // Mock the documents query (second query)
-      mockDbChain.offset.mockResolvedValue([mockDocument])
+      vi.mocked(getDocuments).mockResolvedValue({
+        documents: [mockDocument],
+        pagination: {
+          total: 1,
+          limit: 50,
+          offset: 0,
+          hasMore: false,
+        },
+      })
 
       const url = 'http://localhost:3000/api/knowledge/kb-123/documents?includeDisabled=true'
       const req = new Request(url, { method: 'GET' }) as any
@@ -167,6 +212,16 @@ describe('Knowledge Base Documents API Route', () => {
       const response = await GET(req, { params: mockParams })
 
       expect(response.status).toBe(200)
+      expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
+        'kb-123',
+        {
+          includeDisabled: true,
+          search: undefined,
+          limit: 50,
+          offset: 0,
+        },
+        expect.any(String)
+      )
     })
 
     it('should return unauthorized for unauthenticated user', async () => {
@@ -216,13 +271,14 @@ describe('Knowledge Base Documents API Route', () => {
 
     it('should handle database errors', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
+      const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({
         hasAccess: true,
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
-      mockDbChain.orderBy.mockRejectedValue(new Error('Database error'))
+      vi.mocked(getDocuments).mockRejectedValue(new Error('Database error'))
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -245,13 +301,35 @@ describe('Knowledge Base Documents API Route', () => {
 
     it('should create single document successfully', async () => {
       const { checkKnowledgeBaseWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { createSingleDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({
         hasAccess: true,
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
-      mockDbChain.values.mockResolvedValue(undefined)
+
+      const createdDocument = {
+        id: 'doc-123',
+        knowledgeBaseId: 'kb-123',
+        filename: validDocumentData.filename,
+        fileUrl: validDocumentData.fileUrl,
+        fileSize: validDocumentData.fileSize,
+        mimeType: validDocumentData.mimeType,
+        chunkCount: 0,
+        tokenCount: 0,
+        characterCount: 0,
+        enabled: true,
+        uploadedAt: new Date(),
+        tag1: null,
+        tag2: null,
+        tag3: null,
+        tag4: null,
+        tag5: null,
+        tag6: null,
+        tag7: null,
+      }
+      vi.mocked(createSingleDocument).mockResolvedValue(createdDocument)
 
       const req = createMockRequest('POST', validDocumentData)
       const { POST } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -262,7 +340,11 @@ describe('Knowledge Base Documents API Route', () => {
       expect(data.success).toBe(true)
       expect(data.data.filename).toBe(validDocumentData.filename)
       expect(data.data.fileUrl).toBe(validDocumentData.fileUrl)
-      expect(mockDbChain.insert).toHaveBeenCalled()
+      expect(vi.mocked(createSingleDocument)).toHaveBeenCalledWith(
+        validDocumentData,
+        'kb-123',
+        expect.any(String)
+      )
     })
 
     it('should validate single document data', async () => {
@@ -320,9 +402,9 @@ describe('Knowledge Base Documents API Route', () => {
     }
 
     it('should create bulk documents successfully', async () => {
-      const { checkKnowledgeBaseWriteAccess, processDocumentAsync } = await import(
-        '@/app/api/knowledge/utils'
-      )
+      const { checkKnowledgeBaseWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { createDocumentRecords, processDocumentsWithQueue, getProcessingConfig } =
+        await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({
@@ -330,17 +412,31 @@ describe('Knowledge Base Documents API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock transaction to return the created documents
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        const mockTx = {
-          insert: vi.fn().mockReturnValue({
-            values: vi.fn().mockResolvedValue(undefined),
-          }),
-        }
-        return await callback(mockTx)
-      })
+      const createdDocuments = [
+        {
+          documentId: 'doc-1',
+          filename: 'doc1.pdf',
+          fileUrl: 'https://example.com/doc1.pdf',
+          fileSize: 1024,
+          mimeType: 'application/pdf',
+        },
+        {
+          documentId: 'doc-2',
+          filename: 'doc2.pdf',
+          fileUrl: 'https://example.com/doc2.pdf',
+          fileSize: 2048,
+          mimeType: 'application/pdf',
+        },
+      ]
 
-      vi.mocked(processDocumentAsync).mockResolvedValue(undefined)
+      vi.mocked(createDocumentRecords).mockResolvedValue(createdDocuments)
+      vi.mocked(processDocumentsWithQueue).mockResolvedValue(undefined)
+      vi.mocked(getProcessingConfig).mockReturnValue({
+        maxConcurrentDocuments: 8,
+        batchSize: 20,
+        delayBetweenBatches: 100,
+        delayBetweenDocuments: 0,
+      })
 
       const req = createMockRequest('POST', validBulkData)
       const { POST } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -352,7 +448,12 @@ describe('Knowledge Base Documents API Route', () => {
       expect(data.data.total).toBe(2)
       expect(data.data.documentsCreated).toHaveLength(2)
       expect(data.data.processingMethod).toBe('background')
-      expect(mockDbChain.transaction).toHaveBeenCalled()
+      expect(vi.mocked(createDocumentRecords)).toHaveBeenCalledWith(
+        validBulkData.documents,
+        'kb-123',
+        expect.any(String)
+      )
+      expect(vi.mocked(processDocumentsWithQueue)).toHaveBeenCalled()
     })
 
     it('should validate bulk document data', async () => {
@@ -394,9 +495,9 @@ describe('Knowledge Base Documents API Route', () => {
     })
 
     it('should handle processing errors gracefully', async () => {
-      const { checkKnowledgeBaseWriteAccess, processDocumentAsync } = await import(
-        '@/app/api/knowledge/utils'
-      )
+      const { checkKnowledgeBaseWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { createDocumentRecords, processDocumentsWithQueue, getProcessingConfig } =
+        await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({
@@ -404,26 +505,30 @@ describe('Knowledge Base Documents API Route', () => {
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
 
-      // Mock transaction to succeed but processing to fail
-      mockDbChain.transaction.mockImplementation(async (callback) => {
-        const mockTx = {
-          insert: vi.fn().mockReturnValue({
-            values: vi.fn().mockResolvedValue(undefined),
-          }),
-        }
-        return await callback(mockTx)
-      })
+      const createdDocuments = [
+        {
+          documentId: 'doc-1',
+          filename: 'doc1.pdf',
+          fileUrl: 'https://example.com/doc1.pdf',
+          fileSize: 1024,
+          mimeType: 'application/pdf',
+        },
+      ]
 
-      // Don't reject the promise - the processing is async and catches errors internally
-      vi.mocked(processDocumentAsync).mockResolvedValue(undefined)
+      vi.mocked(createDocumentRecords).mockResolvedValue(createdDocuments)
+      vi.mocked(processDocumentsWithQueue).mockResolvedValue(undefined)
+      vi.mocked(getProcessingConfig).mockReturnValue({
+        maxConcurrentDocuments: 8,
+        batchSize: 20,
+        delayBetweenBatches: 100,
+        delayBetweenDocuments: 0,
+      })
 
       const req = createMockRequest('POST', validBulkData)
       const { POST } = await import('@/app/api/knowledge/[id]/documents/route')
       const response = await POST(req, { params: mockParams })
       const data = await response.json()
 
-      // The endpoint should still return success since documents are created
-      // and processing happens asynchronously
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
     })
@@ -485,13 +590,14 @@ describe('Knowledge Base Documents API Route', () => {
 
     it('should handle database errors during creation', async () => {
       const { checkKnowledgeBaseWriteAccess } = await import('@/app/api/knowledge/utils')
+      const { createSingleDocument } = await import('@/lib/knowledge/documents/service')
 
       mockAuth$.mockAuthenticatedUser()
       vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({
         hasAccess: true,
         knowledgeBase: { id: 'kb-123', userId: 'user-123' },
       })
-      mockDbChain.values.mockRejectedValue(new Error('Database error'))
+      vi.mocked(createSingleDocument).mockRejectedValue(new Error('Database error'))
 
       const req = createMockRequest('POST', validDocumentData)
       const { POST } = await import('@/app/api/knowledge/[id]/documents/route')

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -1,279 +1,22 @@
 import { randomUUID } from 'crypto'
-import { and, desc, eq, inArray, isNull, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
-import { getSlotsForFieldType } from '@/lib/constants/knowledge'
+import {
+  bulkDocumentOperation,
+  createDocumentRecords,
+  createSingleDocument,
+  getDocuments,
+  getProcessingConfig,
+  processDocumentsWithQueue,
+} from '@/lib/knowledge/documents/service'
+import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
 import { createLogger } from '@/lib/logs/console/logger'
 import { getUserId } from '@/app/api/auth/oauth/utils'
-import {
-  checkKnowledgeBaseAccess,
-  checkKnowledgeBaseWriteAccess,
-  processDocumentAsync,
-} from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, knowledgeBaseTagDefinitions } from '@/db/schema'
+import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
 
 const logger = createLogger('DocumentsAPI')
 
-const PROCESSING_CONFIG = {
-  maxConcurrentDocuments: 3,
-  batchSize: 5,
-  delayBetweenBatches: 1000,
-  delayBetweenDocuments: 500,
-}
-
-// Helper function to get the next available slot for a knowledge base and field type
-async function getNextAvailableSlot(
-  knowledgeBaseId: string,
-  fieldType: string,
-  existingBySlot?: Map<string, any>
-): Promise<string | null> {
-  let usedSlots: Set<string>
-
-  if (existingBySlot) {
-    // Use provided map if available (for performance in batch operations)
-    // Filter by field type
-    usedSlots = new Set(
-      Array.from(existingBySlot.entries())
-        .filter(([_, def]) => def.fieldType === fieldType)
-        .map(([slot, _]) => slot)
-    )
-  } else {
-    // Query database for existing tag definitions of the same field type
-    const existingDefinitions = await db
-      .select({ tagSlot: knowledgeBaseTagDefinitions.tagSlot })
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
-          eq(knowledgeBaseTagDefinitions.fieldType, fieldType)
-        )
-      )
-
-    usedSlots = new Set(existingDefinitions.map((def) => def.tagSlot))
-  }
-
-  // Find the first available slot for this field type
-  const availableSlots = getSlotsForFieldType(fieldType)
-  for (const slot of availableSlots) {
-    if (!usedSlots.has(slot)) {
-      return slot
-    }
-  }
-
-  return null // No available slots for this field type
-}
-
-// Helper function to process structured document tags
-async function processDocumentTags(
-  knowledgeBaseId: string,
-  tagData: Array<{ tagName: string; fieldType: string; value: string }>,
-  requestId: string
-): Promise<Record<string, string | null>> {
-  const result: Record<string, string | null> = {}
-
-  // Initialize all text tag slots to null (only text type is supported currently)
-  const textSlots = getSlotsForFieldType('text')
-  textSlots.forEach((slot) => {
-    result[slot] = null
-  })
-
-  if (!Array.isArray(tagData) || tagData.length === 0) {
-    return result
-  }
-
-  try {
-    // Get existing tag definitions
-    const existingDefinitions = await db
-      .select()
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
-
-    const existingByName = new Map(existingDefinitions.map((def) => [def.displayName, def]))
-    const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot, def]))
-
-    // Process each tag
-    for (const tag of tagData) {
-      if (!tag.tagName?.trim() || !tag.value?.trim()) continue
-
-      const tagName = tag.tagName.trim()
-      const fieldType = tag.fieldType
-      const value = tag.value.trim()
-
-      let targetSlot: string | null = null
-
-      // Check if tag definition already exists
-      const existingDef = existingByName.get(tagName)
-      if (existingDef) {
-        targetSlot = existingDef.tagSlot
-      } else {
-        // Find next available slot using the helper function
-        targetSlot = await getNextAvailableSlot(knowledgeBaseId, fieldType, existingBySlot)
-
-        // Create new tag definition if we have a slot
-        if (targetSlot) {
-          const newDefinition = {
-            id: randomUUID(),
-            knowledgeBaseId,
-            tagSlot: targetSlot as any,
-            displayName: tagName,
-            fieldType,
-            createdAt: new Date(),
-            updatedAt: new Date(),
-          }
-
-          await db.insert(knowledgeBaseTagDefinitions).values(newDefinition)
-          existingBySlot.set(targetSlot as any, newDefinition)
-
-          logger.info(`[${requestId}] Created tag definition: ${tagName} -> ${targetSlot}`)
-        }
-      }
-
-      // Assign value to the slot
-      if (targetSlot) {
-        result[targetSlot] = value
-      }
-    }
-
-    return result
-  } catch (error) {
-    logger.error(`[${requestId}] Error processing document tags:`, error)
-    return result
-  }
-}
-
-async function processDocumentsWithConcurrencyControl(
-  createdDocuments: Array<{
-    documentId: string
-    filename: string
-    fileUrl: string
-    fileSize: number
-    mimeType: string
-  }>,
-  knowledgeBaseId: string,
-  processingOptions: {
-    chunkSize: number
-    minCharactersPerChunk: number
-    recipe: string
-    lang: string
-    chunkOverlap: number
-  },
-  requestId: string
-): Promise<void> {
-  const totalDocuments = createdDocuments.length
-  const batches = []
-
-  for (let i = 0; i < totalDocuments; i += PROCESSING_CONFIG.batchSize) {
-    batches.push(createdDocuments.slice(i, i + PROCESSING_CONFIG.batchSize))
-  }
-
-  logger.info(`[${requestId}] Processing ${totalDocuments} documents in ${batches.length} batches`)
-
-  for (const [batchIndex, batch] of batches.entries()) {
-    logger.info(
-      `[${requestId}] Starting batch ${batchIndex + 1}/${batches.length} with ${batch.length} documents`
-    )
-
-    await processBatchWithConcurrency(batch, knowledgeBaseId, processingOptions, requestId)
-
-    if (batchIndex < batches.length - 1) {
-      await new Promise((resolve) => setTimeout(resolve, PROCESSING_CONFIG.delayBetweenBatches))
-    }
-  }
-
-  logger.info(`[${requestId}] Completed processing initiation for all ${totalDocuments} documents`)
-}
-
-async function processBatchWithConcurrency(
-  batch: Array<{
-    documentId: string
-    filename: string
-    fileUrl: string
-    fileSize: number
-    mimeType: string
-  }>,
-  knowledgeBaseId: string,
-  processingOptions: {
-    chunkSize: number
-    minCharactersPerChunk: number
-    recipe: string
-    lang: string
-    chunkOverlap: number
-  },
-  requestId: string
-): Promise<void> {
-  const semaphore = new Array(PROCESSING_CONFIG.maxConcurrentDocuments).fill(0)
-  const processingPromises = batch.map(async (doc, index) => {
-    if (index > 0) {
-      await new Promise((resolve) =>
-        setTimeout(resolve, index * PROCESSING_CONFIG.delayBetweenDocuments)
-      )
-    }
-
-    await new Promise<void>((resolve) => {
-      const checkSlot = () => {
-        const availableIndex = semaphore.findIndex((slot) => slot === 0)
-        if (availableIndex !== -1) {
-          semaphore[availableIndex] = 1
-          resolve()
-        } else {
-          setTimeout(checkSlot, 100)
-        }
-      }
-      checkSlot()
-    })
-
-    try {
-      logger.info(`[${requestId}] Starting processing for document: ${doc.filename}`)
-
-      await processDocumentAsync(
-        knowledgeBaseId,
-        doc.documentId,
-        {
-          filename: doc.filename,
-          fileUrl: doc.fileUrl,
-          fileSize: doc.fileSize,
-          mimeType: doc.mimeType,
-        },
-        processingOptions
-      )
-
-      logger.info(`[${requestId}] Successfully initiated processing for document: ${doc.filename}`)
-    } catch (error: unknown) {
-      logger.error(`[${requestId}] Failed to process document: ${doc.filename}`, {
-        documentId: doc.documentId,
-        filename: doc.filename,
-        error: error instanceof Error ? error.message : 'Unknown error',
-      })
-
-      try {
-        await db
-          .update(document)
-          .set({
-            processingStatus: 'failed',
-            processingError:
-              error instanceof Error ? error.message : 'Failed to initiate processing',
-            processingCompletedAt: new Date(),
-          })
-          .where(eq(document.id, doc.documentId))
-      } catch (dbError: unknown) {
-        logger.error(
-          `[${requestId}] Failed to update document status for failed document: ${doc.documentId}`,
-          dbError
-        )
-      }
-    } finally {
-      const slotIndex = semaphore.findIndex((slot) => slot === 1)
-      if (slotIndex !== -1) {
-        semaphore[slotIndex] = 0
-      }
-    }
-  })
-
-  await Promise.allSettled(processingPromises)
-}
-
 const CreateDocumentSchema = z.object({
   filename: z.string().min(1, 'Filename is required'),
   fileUrl: z.string().url('File URL must be valid'),
@@ -337,83 +80,50 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
 
     const url = new URL(req.url)
     const includeDisabled = url.searchParams.get('includeDisabled') === 'true'
-    const search = url.searchParams.get('search')
+    const search = url.searchParams.get('search') || undefined
     const limit = Number.parseInt(url.searchParams.get('limit') || '50')
     const offset = Number.parseInt(url.searchParams.get('offset') || '0')
+    const sortByParam = url.searchParams.get('sortBy')
+    const sortOrderParam = url.searchParams.get('sortOrder')
 
-    // Build where conditions
-    const whereConditions = [
-      eq(document.knowledgeBaseId, knowledgeBaseId),
-      isNull(document.deletedAt),
+    // Validate sort parameters
+    const validSortFields: DocumentSortField[] = [
+      'filename',
+      'fileSize',
+      'tokenCount',
+      'chunkCount',
+      'uploadedAt',
+      'processingStatus',
     ]
+    const validSortOrders: SortOrder[] = ['asc', 'desc']
 
-    // Filter out disabled documents unless specifically requested
-    if (!includeDisabled) {
-      whereConditions.push(eq(document.enabled, true))
-    }
+    const sortBy =
+      sortByParam && validSortFields.includes(sortByParam as DocumentSortField)
+        ? (sortByParam as DocumentSortField)
+        : undefined
+    const sortOrder =
+      sortOrderParam && validSortOrders.includes(sortOrderParam as SortOrder)
+        ? (sortOrderParam as SortOrder)
+        : undefined
 
-    // Add search condition if provided
-    if (search) {
-      whereConditions.push(
-        // Search in filename
-        sql`LOWER(${document.filename}) LIKE LOWER(${`%${search}%`})`
-      )
-    }
-
-    // Get total count for pagination
-    const totalResult = await db
-      .select({ count: sql<number>`COUNT(*)` })
-      .from(document)
-      .where(and(...whereConditions))
-
-    const total = totalResult[0]?.count || 0
-    const hasMore = offset + limit < total
-
-    const documents = await db
-      .select({
-        id: document.id,
-        filename: document.filename,
-        fileUrl: document.fileUrl,
-        fileSize: document.fileSize,
-        mimeType: document.mimeType,
-        chunkCount: document.chunkCount,
-        tokenCount: document.tokenCount,
-        characterCount: document.characterCount,
-        processingStatus: document.processingStatus,
-        processingStartedAt: document.processingStartedAt,
-        processingCompletedAt: document.processingCompletedAt,
-        processingError: document.processingError,
-        enabled: document.enabled,
-        uploadedAt: document.uploadedAt,
-        // Include tags in response
-        tag1: document.tag1,
-        tag2: document.tag2,
-        tag3: document.tag3,
-        tag4: document.tag4,
-        tag5: document.tag5,
-        tag6: document.tag6,
-        tag7: document.tag7,
-      })
-      .from(document)
-      .where(and(...whereConditions))
-      .orderBy(desc(document.uploadedAt))
-      .limit(limit)
-      .offset(offset)
-
-    logger.info(
-      `[${requestId}] Retrieved ${documents.length} documents (${offset}-${offset + documents.length} of ${total}) for knowledge base ${knowledgeBaseId}`
+    const result = await getDocuments(
+      knowledgeBaseId,
+      {
+        includeDisabled,
+        search,
+        limit,
+        offset,
+        ...(sortBy && { sortBy }),
+        ...(sortOrder && { sortOrder }),
+      },
+      requestId
     )
 
     return NextResponse.json({
       success: true,
       data: {
-        documents,
-        pagination: {
-          total,
-          limit,
-          offset,
-          hasMore,
-        },
+        documents: result.documents,
+        pagination: result.pagination,
       },
     })
   } catch (error) {
@@ -462,80 +172,21 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if this is a bulk operation
     if (body.bulk === true) {
-      // Handle bulk processing (replaces process-documents endpoint)
       try {
         const validatedData = BulkCreateDocumentsSchema.parse(body)
 
-        const createdDocuments = await db.transaction(async (tx) => {
-          const documentPromises = validatedData.documents.map(async (docData) => {
-            const documentId = randomUUID()
-            const now = new Date()
-
-            // Process documentTagsData if provided (for knowledge base block)
-            let processedTags: Record<string, string | null> = {
-              tag1: null,
-              tag2: null,
-              tag3: null,
-              tag4: null,
-              tag5: null,
-              tag6: null,
-              tag7: null,
-            }
-
-            if (docData.documentTagsData) {
-              try {
-                const tagData = JSON.parse(docData.documentTagsData)
-                if (Array.isArray(tagData)) {
-                  processedTags = await processDocumentTags(knowledgeBaseId, tagData, requestId)
-                }
-              } catch (error) {
-                logger.warn(
-                  `[${requestId}] Failed to parse documentTagsData for bulk document:`,
-                  error
-                )
-              }
-            }
-
-            const newDocument = {
-              id: documentId,
-              knowledgeBaseId,
-              filename: docData.filename,
-              fileUrl: docData.fileUrl,
-              fileSize: docData.fileSize,
-              mimeType: docData.mimeType,
-              chunkCount: 0,
-              tokenCount: 0,
-              characterCount: 0,
-              processingStatus: 'pending' as const,
-              enabled: true,
-              uploadedAt: now,
-              // Use processed tags if available, otherwise fall back to individual tag fields
-              tag1: processedTags.tag1 || docData.tag1 || null,
-              tag2: processedTags.tag2 || docData.tag2 || null,
-              tag3: processedTags.tag3 || docData.tag3 || null,
-              tag4: processedTags.tag4 || docData.tag4 || null,
-              tag5: processedTags.tag5 || docData.tag5 || null,
-              tag6: processedTags.tag6 || docData.tag6 || null,
-              tag7: processedTags.tag7 || docData.tag7 || null,
-            }
-
-            await tx.insert(document).values(newDocument)
-            logger.info(
-              `[${requestId}] Document record created: ${documentId} for file: ${docData.filename}`
-            )
-            return { documentId, ...docData }
-          })
-
-          return await Promise.all(documentPromises)
-        })
+        const createdDocuments = await createDocumentRecords(
+          validatedData.documents,
+          knowledgeBaseId,
+          requestId
+        )
 
         logger.info(
           `[${requestId}] Starting controlled async processing of ${createdDocuments.length} documents`
         )
 
-        processDocumentsWithConcurrencyControl(
+        processDocumentsWithQueue(
           createdDocuments,
           knowledgeBaseId,
           validatedData.processingOptions,
@@ -555,9 +206,9 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
             })),
             processingMethod: 'background',
             processingConfig: {
-              maxConcurrentDocuments: PROCESSING_CONFIG.maxConcurrentDocuments,
-              batchSize: PROCESSING_CONFIG.batchSize,
-              totalBatches: Math.ceil(createdDocuments.length / PROCESSING_CONFIG.batchSize),
+              maxConcurrentDocuments: getProcessingConfig().maxConcurrentDocuments,
+              batchSize: getProcessingConfig().batchSize,
+              totalBatches: Math.ceil(createdDocuments.length / getProcessingConfig().batchSize),
             },
           },
         })
@@ -578,52 +229,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       try {
         const validatedData = CreateDocumentSchema.parse(body)
 
-        const documentId = randomUUID()
-        const now = new Date()
-
-        // Process structured tag data if provided
-        let processedTags: Record<string, string | null> = {
-          tag1: validatedData.tag1 || null,
-          tag2: validatedData.tag2 || null,
-          tag3: validatedData.tag3 || null,
-          tag4: validatedData.tag4 || null,
-          tag5: validatedData.tag5 || null,
-          tag6: validatedData.tag6 || null,
-          tag7: validatedData.tag7 || null,
-        }
-
-        if (validatedData.documentTagsData) {
-          try {
-            const tagData = JSON.parse(validatedData.documentTagsData)
-            if (Array.isArray(tagData)) {
-              // Process structured tag data and create tag definitions
-              processedTags = await processDocumentTags(knowledgeBaseId, tagData, requestId)
-            }
-          } catch (error) {
-            logger.warn(`[${requestId}] Failed to parse documentTagsData:`, error)
-          }
-        }
-
-        const newDocument = {
-          id: documentId,
-          knowledgeBaseId,
-          filename: validatedData.filename,
-          fileUrl: validatedData.fileUrl,
-          fileSize: validatedData.fileSize,
-          mimeType: validatedData.mimeType,
-          chunkCount: 0,
-          tokenCount: 0,
-          characterCount: 0,
-          enabled: true,
-          uploadedAt: now,
-          ...processedTags,
-        }
-
-        await db.insert(document).values(newDocument)
-
-        logger.info(
-          `[${requestId}] Document created: ${documentId} in knowledge base ${knowledgeBaseId}`
-        )
+        const newDocument = await createSingleDocument(validatedData, knowledgeBaseId, requestId)
 
         return NextResponse.json({
           success: true,
@@ -649,7 +255,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
 }
 
 export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = randomUUID().slice(0, 8)
   const { id: knowledgeBaseId } = await params
 
   try {
@@ -678,89 +284,28 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id
       const validatedData = BulkUpdateDocumentsSchema.parse(body)
       const { operation, documentIds } = validatedData
 
-      logger.info(
-        `[${requestId}] Starting bulk ${operation} operation on ${documentIds.length} documents in knowledge base ${knowledgeBaseId}`
-      )
-
-      // Verify all documents belong to this knowledge base and user has access
-      const documentsToUpdate = await db
-        .select({
-          id: document.id,
-          enabled: document.enabled,
-        })
-        .from(document)
-        .where(
-          and(
-            eq(document.knowledgeBaseId, knowledgeBaseId),
-            inArray(document.id, documentIds),
-            isNull(document.deletedAt)
-          )
-        )
-
-      if (documentsToUpdate.length === 0) {
-        return NextResponse.json({ error: 'No valid documents found to update' }, { status: 404 })
-      }
-
-      if (documentsToUpdate.length !== documentIds.length) {
-        logger.warn(
-          `[${requestId}] Some documents not found or don't belong to knowledge base. Requested: ${documentIds.length}, Found: ${documentsToUpdate.length}`
-        )
-      }
-
-      // Perform the bulk operation
-      let updateResult: Array<{ id: string; enabled?: boolean; deletedAt?: Date | null }>
-      let successCount: number
-
-      if (operation === 'delete') {
-        // Handle bulk soft delete
-        updateResult = await db
-          .update(document)
-          .set({
-            deletedAt: new Date(),
-          })
-          .where(
-            and(
-              eq(document.knowledgeBaseId, knowledgeBaseId),
-              inArray(document.id, documentIds),
-              isNull(document.deletedAt)
-            )
-          )
-          .returning({ id: document.id, deletedAt: document.deletedAt })
-
-        successCount = updateResult.length
-      } else {
-        // Handle bulk enable/disable
-        const enabled = operation === 'enable'
-
-        updateResult = await db
-          .update(document)
-          .set({
-            enabled,
-          })
-          .where(
-            and(
-              eq(document.knowledgeBaseId, knowledgeBaseId),
-              inArray(document.id, documentIds),
-              isNull(document.deletedAt)
-            )
-          )
-          .returning({ id: document.id, enabled: document.enabled })
-
-        successCount = updateResult.length
-      }
-
-      logger.info(
-        `[${requestId}] Bulk ${operation} operation completed: ${successCount} documents updated in knowledge base ${knowledgeBaseId}`
-      )
-
-      return NextResponse.json({
-        success: true,
-        data: {
+      try {
+        const result = await bulkDocumentOperation(
+          knowledgeBaseId,
           operation,
-          successCount,
-          updatedDocuments: updateResult,
-        },
-      })
+          documentIds,
+          requestId
+        )
+
+        return NextResponse.json({
+          success: true,
+          data: {
+            operation,
+            successCount: result.successCount,
+            updatedDocuments: result.updatedDocuments,
+          },
+        })
+      } catch (error) {
+        if (error instanceof Error && error.message === 'No valid documents found to update') {
+          return NextResponse.json({ error: 'No valid documents found to update' }, { status: 404 })
+        }
+        throw error
+      }
     } catch (validationError) {
       if (validationError instanceof z.ZodError) {
         logger.warn(`[${requestId}] Invalid bulk operation data`, {

apps/sim/app/api/knowledge/[id]/next-available-slot/route.ts

@@ -1,12 +1,9 @@
 import { randomUUID } from 'crypto'
-import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
-import { getMaxSlotsForFieldType, getSlotsForFieldType } from '@/lib/constants/knowledge'
+import { getNextAvailableSlot, getTagDefinitions } from '@/lib/knowledge/tags/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { knowledgeBaseTagDefinitions } from '@/db/schema'
 
 const logger = createLogger('NextAvailableSlotAPI')
 
@@ -31,51 +28,36 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has read access to the knowledge base
     const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
     if (!accessCheck.hasAccess) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
     }
 
-    // Get available slots for this field type
-    const availableSlots = getSlotsForFieldType(fieldType)
-    const maxSlots = getMaxSlotsForFieldType(fieldType)
+    // Get existing definitions once and reuse
+    const existingDefinitions = await getTagDefinitions(knowledgeBaseId)
+    const usedSlots = existingDefinitions
+      .filter((def) => def.fieldType === fieldType)
+      .map((def) => def.tagSlot)
 
-    // Get existing tag definitions to find used slots for this field type
-    const existingDefinitions = await db
-      .select({ tagSlot: knowledgeBaseTagDefinitions.tagSlot })
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
-          eq(knowledgeBaseTagDefinitions.fieldType, fieldType)
-        )
-      )
-
-    const usedSlots = new Set(existingDefinitions.map((def) => def.tagSlot as string))
-
-    // Find the first available slot for this field type
-    let nextAvailableSlot: string | null = null
-    for (const slot of availableSlots) {
-      if (!usedSlots.has(slot)) {
-        nextAvailableSlot = slot
-        break
-      }
-    }
+    // Create a map for efficient lookup and pass to avoid redundant query
+    const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot as string, def]))
+    const nextAvailableSlot = await getNextAvailableSlot(knowledgeBaseId, fieldType, existingBySlot)
 
     logger.info(
       `[${requestId}] Next available slot for fieldType ${fieldType}: ${nextAvailableSlot}`
     )
 
+    const result = {
+      nextAvailableSlot,
+      fieldType,
+      usedSlots,
+      totalSlots: 7,
+      availableSlots: nextAvailableSlot ? 7 - usedSlots.length : 0,
+    }
+
     return NextResponse.json({
       success: true,
-      data: {
-        nextAvailableSlot,
-        fieldType,
-        usedSlots: Array.from(usedSlots),
-        totalSlots: maxSlots,
-        availableSlots: maxSlots - usedSlots.size,
-      },
+      data: result,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error getting next available slot`, error)

apps/sim/app/api/knowledge/[id]/route.test.ts

@@ -16,9 +16,26 @@ mockKnowledgeSchemas()
 mockDrizzleOrm()
 mockConsoleLogger()
 
+vi.mock('@/lib/knowledge/service', () => ({
+  getKnowledgeBaseById: vi.fn(),
+  updateKnowledgeBase: vi.fn(),
+  deleteKnowledgeBase: vi.fn(),
+}))
+
+vi.mock('@/app/api/knowledge/utils', () => ({
+  checkKnowledgeBaseAccess: vi.fn(),
+  checkKnowledgeBaseWriteAccess: vi.fn(),
+}))
+
 describe('Knowledge Base By ID API Route', () => {
   const mockAuth$ = mockAuth()
 
+  let mockGetKnowledgeBaseById: any
+  let mockUpdateKnowledgeBase: any
+  let mockDeleteKnowledgeBase: any
+  let mockCheckKnowledgeBaseAccess: any
+  let mockCheckKnowledgeBaseWriteAccess: any
+
   const mockDbChain = {
     select: vi.fn().mockReturnThis(),
     from: vi.fn().mockReturnThis(),
@@ -62,6 +79,15 @@ describe('Knowledge Base By ID API Route', () => {
     vi.stubGlobal('crypto', {
       randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'),
     })
+
+    const knowledgeService = await import('@/lib/knowledge/service')
+    const knowledgeUtils = await import('@/app/api/knowledge/utils')
+
+    mockGetKnowledgeBaseById = knowledgeService.getKnowledgeBaseById as any
+    mockUpdateKnowledgeBase = knowledgeService.updateKnowledgeBase as any
+    mockDeleteKnowledgeBase = knowledgeService.deleteKnowledgeBase as any
+    mockCheckKnowledgeBaseAccess = knowledgeUtils.checkKnowledgeBaseAccess as any
+    mockCheckKnowledgeBaseWriteAccess = knowledgeUtils.checkKnowledgeBaseWriteAccess as any
   })
 
   afterEach(() => {
@@ -74,9 +100,12 @@ describe('Knowledge Base By ID API Route', () => {
     it('should retrieve knowledge base successfully for authenticated user', async () => {
       mockAuth$.mockAuthenticatedUser()
 
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
-      mockDbChain.limit.mockResolvedValueOnce([mockKnowledgeBase])
+      mockGetKnowledgeBaseById.mockResolvedValueOnce(mockKnowledgeBase)
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/route')
@@ -87,7 +116,8 @@ describe('Knowledge Base By ID API Route', () => {
       expect(data.success).toBe(true)
       expect(data.data.id).toBe('kb-123')
       expect(data.data.name).toBe('Test Knowledge Base')
-      expect(mockDbChain.select).toHaveBeenCalled()
+      expect(mockCheckKnowledgeBaseAccess).toHaveBeenCalledWith('kb-123', 'user-123')
+      expect(mockGetKnowledgeBaseById).toHaveBeenCalledWith('kb-123')
     })
 
     it('should return unauthorized for unauthenticated user', async () => {
@@ -105,7 +135,10 @@ describe('Knowledge Base By ID API Route', () => {
     it('should return not found for non-existent knowledge base', async () => {
       mockAuth$.mockAuthenticatedUser()
 
-      mockDbChain.limit.mockResolvedValueOnce([])
+      mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+        hasAccess: false,
+        notFound: true,
+      })
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/route')
@@ -119,7 +152,10 @@ describe('Knowledge Base By ID API Route', () => {
     it('should return unauthorized for knowledge base owned by different user', async () => {
       mockAuth$.mockAuthenticatedUser()
 
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'different-user' }])
+      mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+        hasAccess: false,
+        notFound: false,
+      })
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/route')
@@ -130,9 +166,29 @@ describe('Knowledge Base By ID API Route', () => {
       expect(data.error).toBe('Unauthorized')
     })
 
+    it('should return not found when service returns null', async () => {
+      mockAuth$.mockAuthenticatedUser()
+
+      mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
+
+      mockGetKnowledgeBaseById.mockResolvedValueOnce(null)
+
+      const req = createMockRequest('GET')
+      const { GET } = await import('@/app/api/knowledge/[id]/route')
+      const response = await GET(req, { params: mockParams })
+      const data = await response.json()
+
+      expect(response.status).toBe(404)
+      expect(data.error).toBe('Knowledge base not found')
+    })
+
     it('should handle database errors', async () => {
       mockAuth$.mockAuthenticatedUser()
-      mockDbChain.limit.mockRejectedValueOnce(new Error('Database error'))
+
+      mockCheckKnowledgeBaseAccess.mockRejectedValueOnce(new Error('Database error'))
 
       const req = createMockRequest('GET')
       const { GET } = await import('@/app/api/knowledge/[id]/route')
@@ -156,13 +212,13 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
-      mockDbChain.where.mockResolvedValueOnce(undefined)
-
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([{ ...mockKnowledgeBase, ...validUpdateData }])
+      const updatedKnowledgeBase = { ...mockKnowledgeBase, ...validUpdateData }
+      mockUpdateKnowledgeBase.mockResolvedValueOnce(updatedKnowledgeBase)
 
       const req = createMockRequest('PUT', validUpdateData)
       const { PUT } = await import('@/app/api/knowledge/[id]/route')
@@ -172,7 +228,16 @@ describe('Knowledge Base By ID API Route', () => {
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.name).toBe('Updated Knowledge Base')
-      expect(mockDbChain.update).toHaveBeenCalled()
+      expect(mockCheckKnowledgeBaseWriteAccess).toHaveBeenCalledWith('kb-123', 'user-123')
+      expect(mockUpdateKnowledgeBase).toHaveBeenCalledWith(
+        'kb-123',
+        {
+          name: validUpdateData.name,
+          description: validUpdateData.description,
+          chunkingConfig: undefined,
+        },
+        expect.any(String)
+      )
     })
 
     it('should return unauthorized for unauthenticated user', async () => {
@@ -192,8 +257,10 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: false,
+        notFound: true,
+      })
 
       const req = createMockRequest('PUT', validUpdateData)
       const { PUT } = await import('@/app/api/knowledge/[id]/route')
@@ -209,8 +276,10 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
       const invalidData = {
         name: '',
@@ -229,9 +298,13 @@ describe('Knowledge Base By ID API Route', () => {
     it('should handle database errors during update', async () => {
       mockAuth$.mockAuthenticatedUser()
 
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      // Mock successful write access check
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
-      mockDbChain.where.mockRejectedValueOnce(new Error('Database error'))
+      mockUpdateKnowledgeBase.mockRejectedValueOnce(new Error('Database error'))
 
       const req = createMockRequest('PUT', validUpdateData)
       const { PUT } = await import('@/app/api/knowledge/[id]/route')
@@ -251,10 +324,12 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
-      mockDbChain.where.mockResolvedValueOnce(undefined)
+      mockDeleteKnowledgeBase.mockResolvedValueOnce(undefined)
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/route')
@@ -264,7 +339,8 @@ describe('Knowledge Base By ID API Route', () => {
       expect(response.status).toBe(200)
       expect(data.success).toBe(true)
       expect(data.data.message).toBe('Knowledge base deleted successfully')
-      expect(mockDbChain.update).toHaveBeenCalled()
+      expect(mockCheckKnowledgeBaseWriteAccess).toHaveBeenCalledWith('kb-123', 'user-123')
+      expect(mockDeleteKnowledgeBase).toHaveBeenCalledWith('kb-123', expect.any(String))
     })
 
     it('should return unauthorized for unauthenticated user', async () => {
@@ -284,8 +360,10 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: false,
+        notFound: true,
+      })
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/route')
@@ -301,8 +379,10 @@ describe('Knowledge Base By ID API Route', () => {
 
       resetMocks()
 
-      mockDbChain.where.mockReturnValueOnce(mockDbChain) // Return this to continue chain
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'different-user' }])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: false,
+        notFound: false,
+      })
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/route')
@@ -316,9 +396,12 @@ describe('Knowledge Base By ID API Route', () => {
     it('should handle database errors during delete', async () => {
       mockAuth$.mockAuthenticatedUser()
 
-      mockDbChain.limit.mockResolvedValueOnce([{ id: 'kb-123', userId: 'user-123' }])
+      mockCheckKnowledgeBaseWriteAccess.mockResolvedValueOnce({
+        hasAccess: true,
+        knowledgeBase: { id: 'kb-123', userId: 'user-123' },
+      })
 
-      mockDbChain.where.mockRejectedValueOnce(new Error('Database error'))
+      mockDeleteKnowledgeBase.mockRejectedValueOnce(new Error('Database error'))
 
       const req = createMockRequest('DELETE')
       const { DELETE } = await import('@/app/api/knowledge/[id]/route')

apps/sim/app/api/knowledge/[id]/route.ts

@@ -1,11 +1,14 @@
-import { and, eq, isNull } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import {
+  deleteKnowledgeBase,
+  getKnowledgeBaseById,
+  updateKnowledgeBase,
+} from '@/lib/knowledge/service'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { knowledgeBase } from '@/db/schema'
 
 const logger = createLogger('KnowledgeBaseByIdAPI')
 
@@ -25,7 +28,7 @@ const UpdateKnowledgeBaseSchema = z.object({
 })
 
 export async function GET(_req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {
@@ -48,13 +51,9 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    const knowledgeBases = await db
-      .select()
-      .from(knowledgeBase)
-      .where(and(eq(knowledgeBase.id, id), isNull(knowledgeBase.deletedAt)))
-      .limit(1)
+    const knowledgeBaseData = await getKnowledgeBaseById(id)
 
-    if (knowledgeBases.length === 0) {
+    if (!knowledgeBaseData) {
       return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 })
     }
 
@@ -62,7 +61,7 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ id:
 
     return NextResponse.json({
       success: true,
-      data: knowledgeBases[0],
+      data: knowledgeBaseData,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error fetching knowledge base`, error)
@@ -71,7 +70,7 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ id:
 }
 
 export async function PUT(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {
@@ -99,42 +98,21 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id:
     try {
       const validatedData = UpdateKnowledgeBaseSchema.parse(body)
 
-      const updateData: any = {
-        updatedAt: new Date(),
-      }
-
-      if (validatedData.name !== undefined) updateData.name = validatedData.name
-      if (validatedData.description !== undefined)
-        updateData.description = validatedData.description
-      if (validatedData.workspaceId !== undefined)
-        updateData.workspaceId = validatedData.workspaceId
-
-      // Handle embedding model and dimension together to ensure consistency
-      if (
-        validatedData.embeddingModel !== undefined ||
-        validatedData.embeddingDimension !== undefined
-      ) {
-        updateData.embeddingModel = 'text-embedding-3-small'
-        updateData.embeddingDimension = 1536
-      }
-
-      if (validatedData.chunkingConfig !== undefined)
-        updateData.chunkingConfig = validatedData.chunkingConfig
-
-      await db.update(knowledgeBase).set(updateData).where(eq(knowledgeBase.id, id))
-
-      // Fetch the updated knowledge base
-      const updatedKnowledgeBase = await db
-        .select()
-        .from(knowledgeBase)
-        .where(eq(knowledgeBase.id, id))
-        .limit(1)
+      const updatedKnowledgeBase = await updateKnowledgeBase(
+        id,
+        {
+          name: validatedData.name,
+          description: validatedData.description,
+          chunkingConfig: validatedData.chunkingConfig,
+        },
+        requestId
+      )
 
       logger.info(`[${requestId}] Knowledge base updated: ${id} for user ${session.user.id}`)
 
       return NextResponse.json({
         success: true,
-        data: updatedKnowledgeBase[0],
+        data: updatedKnowledgeBase,
       })
     } catch (validationError) {
       if (validationError instanceof z.ZodError) {
@@ -155,7 +133,7 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ id:
 }
 
 export async function DELETE(_req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {
@@ -178,14 +156,7 @@ export async function DELETE(_req: NextRequest, { params }: { params: Promise<{
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Soft delete by setting deletedAt timestamp
-    await db
-      .update(knowledgeBase)
-      .set({
-        deletedAt: new Date(),
-        updatedAt: new Date(),
-      })
-      .where(eq(knowledgeBase.id, id))
+    await deleteKnowledgeBase(id, requestId)
 
     logger.info(`[${requestId}] Knowledge base deleted: ${id} for user ${session.user.id}`)
 

apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts

@@ -1,11 +1,9 @@
 import { randomUUID } from 'crypto'
-import { and, eq, isNotNull } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
+import { deleteTagDefinition } from '@/lib/knowledge/tags/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, embedding, knowledgeBaseTagDefinitions } from '@/db/schema'
 
 export const dynamic = 'force-dynamic'
 
@@ -29,87 +27,16 @@ export async function DELETE(
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has access to the knowledge base
     const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
     if (!accessCheck.hasAccess) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
     }
 
-    // Get the tag definition to find which slot it uses
-    const tagDefinition = await db
-      .select({
-        id: knowledgeBaseTagDefinitions.id,
-        tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-        displayName: knowledgeBaseTagDefinitions.displayName,
-      })
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.id, tagId),
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)
-        )
-      )
-      .limit(1)
-
-    if (tagDefinition.length === 0) {
-      return NextResponse.json({ error: 'Tag definition not found' }, { status: 404 })
-    }
-
-    const tagDef = tagDefinition[0]
-
-    // Delete the tag definition and clear all document tags in a transaction
-    await db.transaction(async (tx) => {
-      logger.info(`[${requestId}] Starting transaction to delete ${tagDef.tagSlot}`)
-
-      try {
-        // Clear the tag from documents that actually have this tag set
-        logger.info(`[${requestId}] Clearing tag from documents...`)
-        await tx
-          .update(document)
-          .set({ [tagDef.tagSlot]: null })
-          .where(
-            and(
-              eq(document.knowledgeBaseId, knowledgeBaseId),
-              isNotNull(document[tagDef.tagSlot as keyof typeof document.$inferSelect])
-            )
-          )
-
-        logger.info(`[${requestId}] Documents updated successfully`)
-
-        // Clear the tag from embeddings that actually have this tag set
-        logger.info(`[${requestId}] Clearing tag from embeddings...`)
-        await tx
-          .update(embedding)
-          .set({ [tagDef.tagSlot]: null })
-          .where(
-            and(
-              eq(embedding.knowledgeBaseId, knowledgeBaseId),
-              isNotNull(embedding[tagDef.tagSlot as keyof typeof embedding.$inferSelect])
-            )
-          )
-
-        logger.info(`[${requestId}] Embeddings updated successfully`)
-
-        // Delete the tag definition
-        logger.info(`[${requestId}] Deleting tag definition...`)
-        await tx
-          .delete(knowledgeBaseTagDefinitions)
-          .where(eq(knowledgeBaseTagDefinitions.id, tagId))
-
-        logger.info(`[${requestId}] Tag definition deleted successfully`)
-      } catch (error) {
-        logger.error(`[${requestId}] Error in transaction:`, error)
-        throw error
-      }
-    })
-
-    logger.info(
-      `[${requestId}] Successfully deleted tag definition ${tagDef.displayName} (${tagDef.tagSlot})`
-    )
+    const deletedTag = await deleteTagDefinition(tagId, requestId)
 
     return NextResponse.json({
       success: true,
-      message: `Tag definition "${tagDef.displayName}" deleted successfully`,
+      message: `Tag definition "${deletedTag.displayName}" deleted successfully`,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error deleting tag definition`, error)

apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts

@@ -1,11 +1,11 @@
 import { randomUUID } from 'crypto'
-import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
+import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/consts'
+import { createTagDefinition, getTagDefinitions } from '@/lib/knowledge/tags/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { knowledgeBaseTagDefinitions } from '@/db/schema'
 
 export const dynamic = 'force-dynamic'
 
@@ -24,25 +24,12 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has access to the knowledge base
     const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
     if (!accessCheck.hasAccess) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
     }
 
-    // Get tag definitions for the knowledge base
-    const tagDefinitions = await db
-      .select({
-        id: knowledgeBaseTagDefinitions.id,
-        tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-        displayName: knowledgeBaseTagDefinitions.displayName,
-        fieldType: knowledgeBaseTagDefinitions.fieldType,
-        createdAt: knowledgeBaseTagDefinitions.createdAt,
-        updatedAt: knowledgeBaseTagDefinitions.updatedAt,
-      })
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
-      .orderBy(knowledgeBaseTagDefinitions.tagSlot)
+    const tagDefinitions = await getTagDefinitions(knowledgeBaseId)
 
     logger.info(`[${requestId}] Retrieved ${tagDefinitions.length} tag definitions`)
 
@@ -69,68 +56,43 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has access to the knowledge base
     const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
     if (!accessCheck.hasAccess) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
     }
 
     const body = await req.json()
-    const { tagSlot, displayName, fieldType } = body
 
-    if (!tagSlot || !displayName || !fieldType) {
-      return NextResponse.json(
-        { error: 'tagSlot, displayName, and fieldType are required' },
-        { status: 400 }
-      )
-    }
+    const CreateTagDefinitionSchema = z.object({
+      tagSlot: z.string().min(1, 'Tag slot is required'),
+      displayName: z.string().min(1, 'Display name is required'),
+      fieldType: z.enum(SUPPORTED_FIELD_TYPES as [string, ...string[]], {
+        errorMap: () => ({ message: 'Invalid field type' }),
+      }),
+    })
 
-    // Check if tag slot is already used
-    const existingTag = await db
-      .select()
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
-          eq(knowledgeBaseTagDefinitions.tagSlot, tagSlot)
+    let validatedData
+    try {
+      validatedData = CreateTagDefinitionSchema.parse(body)
+    } catch (error) {
+      if (error instanceof z.ZodError) {
+        return NextResponse.json(
+          { error: 'Invalid request data', details: error.errors },
+          { status: 400 }
         )
-      )
-      .limit(1)
-
-    if (existingTag.length > 0) {
-      return NextResponse.json({ error: 'Tag slot is already in use' }, { status: 409 })
+      }
+      throw error
     }
 
-    // Check if display name is already used
-    const existingName = await db
-      .select()
-      .from(knowledgeBaseTagDefinitions)
-      .where(
-        and(
-          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
-          eq(knowledgeBaseTagDefinitions.displayName, displayName)
-        )
-      )
-      .limit(1)
-
-    if (existingName.length > 0) {
-      return NextResponse.json({ error: 'Tag name is already in use' }, { status: 409 })
-    }
-
-    // Create the new tag definition
-    const newTagDefinition = {
-      id: randomUUID(),
-      knowledgeBaseId,
-      tagSlot,
-      displayName,
-      fieldType,
-      createdAt: new Date(),
-      updatedAt: new Date(),
-    }
-
-    await db.insert(knowledgeBaseTagDefinitions).values(newTagDefinition)
-
-    logger.info(`[${requestId}] Successfully created tag definition ${displayName} (${tagSlot})`)
+    const newTagDefinition = await createTagDefinition(
+      {
+        knowledgeBaseId,
+        tagSlot: validatedData.tagSlot,
+        displayName: validatedData.displayName,
+        fieldType: validatedData.fieldType,
+      },
+      requestId
+    )
 
     return NextResponse.json({
       success: true,

apps/sim/app/api/knowledge/[id]/tag-usage/route.ts

@@ -1,11 +1,9 @@
 import { randomUUID } from 'crypto'
-import { and, eq, isNotNull } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
+import { getTagUsage } from '@/lib/knowledge/tags/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { document, knowledgeBaseTagDefinitions } from '@/db/schema'
 
 export const dynamic = 'force-dynamic'
 
@@ -24,57 +22,15 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user has access to the knowledge base
     const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
     if (!accessCheck.hasAccess) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
     }
 
-    // Get all tag definitions for the knowledge base
-    const tagDefinitions = await db
-      .select({
-        id: knowledgeBaseTagDefinitions.id,
-        tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-        displayName: knowledgeBaseTagDefinitions.displayName,
-      })
-      .from(knowledgeBaseTagDefinitions)
-      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
-
-    // Get usage statistics for each tag definition
-    const usageStats = await Promise.all(
-      tagDefinitions.map(async (tagDef) => {
-        // Count documents using this tag slot
-        const tagSlotColumn = tagDef.tagSlot as keyof typeof document.$inferSelect
-
-        const documentsWithTag = await db
-          .select({
-            id: document.id,
-            filename: document.filename,
-            [tagDef.tagSlot]: document[tagSlotColumn as keyof typeof document.$inferSelect] as any,
-          })
-          .from(document)
-          .where(
-            and(
-              eq(document.knowledgeBaseId, knowledgeBaseId),
-              isNotNull(document[tagSlotColumn as keyof typeof document.$inferSelect])
-            )
-          )
-
-        return {
-          tagName: tagDef.displayName,
-          tagSlot: tagDef.tagSlot,
-          documentCount: documentsWithTag.length,
-          documents: documentsWithTag.map((doc) => ({
-            id: doc.id,
-            name: doc.filename,
-            tagValue: doc[tagDef.tagSlot],
-          })),
-        }
-      })
-    )
+    const usageStats = await getTagUsage(knowledgeBaseId, requestId)
 
     logger.info(
-      `[${requestId}] Retrieved usage statistics for ${tagDefinitions.length} tag definitions`
+      `[${requestId}] Retrieved usage statistics for ${usageStats.length} tag definitions`
     )
 
     return NextResponse.json({

apps/sim/app/api/knowledge/route.ts

@@ -1,11 +1,9 @@
-import { and, count, eq, isNotNull, isNull, or } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
+import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service'
 import { createLogger } from '@/lib/logs/console/logger'
-import { getUserEntityPermissions } from '@/lib/permissions/utils'
-import { db } from '@/db'
-import { document, knowledgeBase, permissions } from '@/db/schema'
+import { generateRequestId } from '@/lib/utils'
 
 const logger = createLogger('KnowledgeBaseAPI')
 
@@ -32,7 +30,7 @@ const CreateKnowledgeBaseSchema = z.object({
 })
 
 export async function GET(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const session = await getSession()
@@ -41,60 +39,10 @@ export async function GET(req: NextRequest) {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check for workspace filtering
     const { searchParams } = new URL(req.url)
     const workspaceId = searchParams.get('workspaceId')
 
-    // Get knowledge bases that user can access through direct ownership OR workspace permissions
-    const knowledgeBasesWithCounts = await db
-      .select({
-        id: knowledgeBase.id,
-        name: knowledgeBase.name,
-        description: knowledgeBase.description,
-        tokenCount: knowledgeBase.tokenCount,
-        embeddingModel: knowledgeBase.embeddingModel,
-        embeddingDimension: knowledgeBase.embeddingDimension,
-        chunkingConfig: knowledgeBase.chunkingConfig,
-        createdAt: knowledgeBase.createdAt,
-        updatedAt: knowledgeBase.updatedAt,
-        workspaceId: knowledgeBase.workspaceId,
-        docCount: count(document.id),
-      })
-      .from(knowledgeBase)
-      .leftJoin(
-        document,
-        and(eq(document.knowledgeBaseId, knowledgeBase.id), isNull(document.deletedAt))
-      )
-      .leftJoin(
-        permissions,
-        and(
-          eq(permissions.entityType, 'workspace'),
-          eq(permissions.entityId, knowledgeBase.workspaceId),
-          eq(permissions.userId, session.user.id)
-        )
-      )
-      .where(
-        and(
-          isNull(knowledgeBase.deletedAt),
-          workspaceId
-            ? // When filtering by workspace
-              or(
-                // Knowledge bases belonging to the specified workspace (user must have workspace permissions)
-                and(eq(knowledgeBase.workspaceId, workspaceId), isNotNull(permissions.userId)),
-                // Fallback: User-owned knowledge bases without workspace (legacy)
-                and(eq(knowledgeBase.userId, session.user.id), isNull(knowledgeBase.workspaceId))
-              )
-            : // When not filtering by workspace, use original logic
-              or(
-                // User owns the knowledge base directly
-                eq(knowledgeBase.userId, session.user.id),
-                // User has permissions on the knowledge base's workspace
-                isNotNull(permissions.userId)
-              )
-        )
-      )
-      .groupBy(knowledgeBase.id)
-      .orderBy(knowledgeBase.createdAt)
+    const knowledgeBasesWithCounts = await getKnowledgeBases(session.user.id, workspaceId)
 
     return NextResponse.json({
       success: true,
@@ -107,7 +55,7 @@ export async function GET(req: NextRequest) {
 }
 
 export async function POST(req: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const session = await getSession()
@@ -121,49 +69,16 @@ export async function POST(req: NextRequest) {
     try {
       const validatedData = CreateKnowledgeBaseSchema.parse(body)
 
-      // If creating in a workspace, check if user has write/admin permissions
-      if (validatedData.workspaceId) {
-        const userPermission = await getUserEntityPermissions(
-          session.user.id,
-          'workspace',
-          validatedData.workspaceId
-        )
-        if (userPermission !== 'write' && userPermission !== 'admin') {
-          logger.warn(
-            `[${requestId}] User ${session.user.id} denied permission to create knowledge base in workspace ${validatedData.workspaceId}`
-          )
-          return NextResponse.json(
-            { error: 'Insufficient permissions to create knowledge base in this workspace' },
-            { status: 403 }
-          )
-        }
-      }
-
-      const id = crypto.randomUUID()
-      const now = new Date()
-
-      const newKnowledgeBase = {
-        id,
+      const createData = {
+        ...validatedData,
         userId: session.user.id,
-        workspaceId: validatedData.workspaceId || null,
-        name: validatedData.name,
-        description: validatedData.description || null,
-        tokenCount: 0,
-        embeddingModel: validatedData.embeddingModel,
-        embeddingDimension: validatedData.embeddingDimension,
-        chunkingConfig: validatedData.chunkingConfig || {
-          maxSize: 1024,
-          minSize: 100,
-          overlap: 200,
-        },
-        docCount: 0,
-        createdAt: now,
-        updatedAt: now,
       }
 
-      await db.insert(knowledgeBase).values(newKnowledgeBase)
+      const newKnowledgeBase = await createKnowledgeBase(createData, requestId)
 
-      logger.info(`[${requestId}] Knowledge base created: ${id} for user ${session.user.id}`)
+      logger.info(
+        `[${requestId}] Knowledge base created: ${newKnowledgeBase.id} for user ${session.user.id}`
+      )
 
       return NextResponse.json({
         success: true,

apps/sim/app/api/knowledge/search/route.test.ts

@@ -34,6 +34,10 @@ vi.mock('@/lib/env', () => ({
     typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value),
 }))
 
+vi.mock('@/lib/utils', () => ({
+  generateRequestId: vi.fn(() => 'test-request-id'),
+}))
+
 vi.mock('@/lib/documents/utils', () => ({
   retryWithExponentialBackoff: vi.fn().mockImplementation((fn) => fn()),
 }))
@@ -65,12 +69,14 @@ const mockHandleVectorOnlySearch = vi.fn()
 const mockHandleTagAndVectorSearch = vi.fn()
 const mockGetQueryStrategy = vi.fn()
 const mockGenerateSearchEmbedding = vi.fn()
+const mockGetDocumentNamesByIds = vi.fn()
 vi.mock('./utils', () => ({
   handleTagOnlySearch: mockHandleTagOnlySearch,
   handleVectorOnlySearch: mockHandleVectorOnlySearch,
   handleTagAndVectorSearch: mockHandleTagAndVectorSearch,
   getQueryStrategy: mockGetQueryStrategy,
   generateSearchEmbedding: mockGenerateSearchEmbedding,
+  getDocumentNamesByIds: mockGetDocumentNamesByIds,
   APIError: class APIError extends Error {
     public status: number
     constructor(message: string, status: number) {
@@ -146,6 +152,10 @@ describe('Knowledge Search API Route', () => {
       singleQueryOptimized: true,
     })
     mockGenerateSearchEmbedding.mockClear().mockResolvedValue([0.1, 0.2, 0.3, 0.4, 0.5])
+    mockGetDocumentNamesByIds.mockClear().mockResolvedValue({
+      doc1: 'Document 1',
+      doc2: 'Document 2',
+    })
 
     vi.stubGlobal('crypto', {
       randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'),

apps/sim/app/api/knowledge/search/route.ts

@@ -1,16 +1,16 @@
-import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
-import { TAG_SLOTS } from '@/lib/constants/knowledge'
+import { TAG_SLOTS } from '@/lib/knowledge/consts'
+import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
 import { createLogger } from '@/lib/logs/console/logger'
 import { estimateTokenCount } from '@/lib/tokenization/estimators'
+import { generateRequestId } from '@/lib/utils'
 import { getUserId } from '@/app/api/auth/oauth/utils'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
-import { db } from '@/db'
-import { knowledgeBaseTagDefinitions } from '@/db/schema'
 import { calculateCost } from '@/providers/utils'
 import {
   generateSearchEmbedding,
+  getDocumentNamesByIds,
   getQueryStrategy,
   handleTagAndVectorSearch,
   handleTagOnlySearch,
@@ -58,7 +58,7 @@ const VectorSearchSchema = z
   )
 
 export async function POST(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const body = await request.json()
@@ -79,14 +79,13 @@ export async function POST(request: NextRequest) {
         ? validatedData.knowledgeBaseIds
         : [validatedData.knowledgeBaseIds]
 
-      // Check access permissions for each knowledge base using proper workspace-based permissions
-      const accessibleKbIds: string[] = []
-      for (const kbId of knowledgeBaseIds) {
-        const accessCheck = await checkKnowledgeBaseAccess(kbId, userId)
-        if (accessCheck.hasAccess) {
-          accessibleKbIds.push(kbId)
-        }
-      }
+      // Check access permissions in parallel for performance
+      const accessChecks = await Promise.all(
+        knowledgeBaseIds.map((kbId) => checkKnowledgeBaseAccess(kbId, userId))
+      )
+      const accessibleKbIds: string[] = knowledgeBaseIds.filter(
+        (_, idx) => accessChecks[idx]?.hasAccess
+      )
 
       // Map display names to tag slots for filtering
       let mappedFilters: Record<string, string> = {}
@@ -94,13 +93,7 @@ export async function POST(request: NextRequest) {
         try {
           // Fetch tag definitions for the first accessible KB (since we're using single KB now)
           const kbId = accessibleKbIds[0]
-          const tagDefs = await db
-            .select({
-              tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-              displayName: knowledgeBaseTagDefinitions.displayName,
-            })
-            .from(knowledgeBaseTagDefinitions)
-            .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, kbId))
+          const tagDefs = await getDocumentTagDefinitions(kbId)
 
           logger.debug(`[${requestId}] Found tag definitions:`, tagDefs)
           logger.debug(`[${requestId}] Original filters:`, validatedData.filters)
@@ -145,7 +138,10 @@ export async function POST(request: NextRequest) {
 
       // Generate query embedding only if query is provided
       const hasQuery = validatedData.query && validatedData.query.trim().length > 0
-      const queryEmbedding = hasQuery ? await generateSearchEmbedding(validatedData.query!) : null
+      // Start embedding generation early and await when needed
+      const queryEmbeddingPromise = hasQuery
+        ? generateSearchEmbedding(validatedData.query!)
+        : Promise.resolve(null)
 
       // Check if any requested knowledge bases were not accessible
       const inaccessibleKbIds = knowledgeBaseIds.filter((id) => !accessibleKbIds.includes(id))
@@ -173,7 +169,7 @@ export async function POST(request: NextRequest) {
         // Tag + Vector search
         logger.debug(`[${requestId}] Executing tag + vector search with filters:`, mappedFilters)
         const strategy = getQueryStrategy(accessibleKbIds.length, validatedData.topK)
-        const queryVector = JSON.stringify(queryEmbedding)
+        const queryVector = JSON.stringify(await queryEmbeddingPromise)
 
         results = await handleTagAndVectorSearch({
           knowledgeBaseIds: accessibleKbIds,
@@ -186,7 +182,7 @@ export async function POST(request: NextRequest) {
         // Vector-only search
         logger.debug(`[${requestId}] Executing vector-only search`)
         const strategy = getQueryStrategy(accessibleKbIds.length, validatedData.topK)
-        const queryVector = JSON.stringify(queryEmbedding)
+        const queryVector = JSON.stringify(await queryEmbeddingPromise)
 
         results = await handleVectorOnlySearch({
           knowledgeBaseIds: accessibleKbIds,
@@ -221,30 +217,32 @@ export async function POST(request: NextRequest) {
       }
 
       // Fetch tag definitions for display name mapping (reuse the same fetch from filtering)
-      const tagDefinitionsMap: Record<string, Record<string, string>> = {}
-      for (const kbId of accessibleKbIds) {
-        try {
-          const tagDefs = await db
-            .select({
-              tagSlot: knowledgeBaseTagDefinitions.tagSlot,
-              displayName: knowledgeBaseTagDefinitions.displayName,
+      const tagDefsResults = await Promise.all(
+        accessibleKbIds.map(async (kbId) => {
+          try {
+            const tagDefs = await getDocumentTagDefinitions(kbId)
+            const map: Record<string, string> = {}
+            tagDefs.forEach((def) => {
+              map[def.tagSlot] = def.displayName
             })
-            .from(knowledgeBaseTagDefinitions)
-            .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, kbId))
+            return { kbId, map }
+          } catch (error) {
+            logger.warn(
+              `[${requestId}] Failed to fetch tag definitions for display mapping:`,
+              error
+            )
+            return { kbId, map: {} as Record<string, string> }
+          }
+        })
+      )
+      const tagDefinitionsMap: Record<string, Record<string, string>> = {}
+      tagDefsResults.forEach(({ kbId, map }) => {
+        tagDefinitionsMap[kbId] = map
+      })
 
-          tagDefinitionsMap[kbId] = {}
-          tagDefs.forEach((def) => {
-            tagDefinitionsMap[kbId][def.tagSlot] = def.displayName
-          })
-          logger.debug(
-            `[${requestId}] Display mapping - KB ${kbId} tag definitions:`,
-            tagDefinitionsMap[kbId]
-          )
-        } catch (error) {
-          logger.warn(`[${requestId}] Failed to fetch tag definitions for display mapping:`, error)
-          tagDefinitionsMap[kbId] = {}
-        }
-      }
+      // Fetch document names for the results
+      const documentIds = results.map((result) => result.documentId)
+      const documentNameMap = await getDocumentNamesByIds(documentIds)
 
       return NextResponse.json({
         success: true,
@@ -271,11 +269,11 @@ export async function POST(request: NextRequest) {
             })
 
             return {
-              id: result.id,
-              content: result.content,
               documentId: result.documentId,
+              documentName: documentNameMap[result.documentId] || undefined,
+              content: result.content,
               chunkIndex: result.chunkIndex,
-              tags, // Clean display name mapped tags
+              metadata: tags, // Clean display name mapped tags
               similarity: hasQuery ? 1 - result.distance : 1, // Perfect similarity for tag-only searches
             }
           }),

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -16,7 +16,7 @@ vi.mock('@/lib/logs/console/logger', () => ({
   })),
 }))
 vi.mock('@/db')
-vi.mock('@/lib/documents/utils', () => ({
+vi.mock('@/lib/knowledge/documents/utils', () => ({
   retryWithExponentialBackoff: (fn: any) => fn(),
 }))
 

apps/sim/app/api/knowledge/search/utils.ts

@@ -1,10 +1,34 @@
 import { and, eq, inArray, sql } from 'drizzle-orm'
 import { createLogger } from '@/lib/logs/console/logger'
 import { db } from '@/db'
-import { embedding } from '@/db/schema'
+import { document, embedding } from '@/db/schema'
 
 const logger = createLogger('KnowledgeSearchUtils')
 
+export async function getDocumentNamesByIds(
+  documentIds: string[]
+): Promise<Record<string, string>> {
+  if (documentIds.length === 0) {
+    return {}
+  }
+
+  const uniqueIds = [...new Set(documentIds)]
+  const documents = await db
+    .select({
+      id: document.id,
+      filename: document.filename,
+    })
+    .from(document)
+    .where(inArray(document.id, uniqueIds))
+
+  const documentNameMap: Record<string, string> = {}
+  documents.forEach((doc) => {
+    documentNameMap[doc.id] = doc.filename
+  })
+
+  return documentNameMap
+}
+
 export interface SearchResult {
   id: string
   content: string

apps/sim/app/api/knowledge/utils.test.ts

@@ -21,11 +21,11 @@ vi.mock('@/lib/env', () => ({
     typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value),
 }))
 
-vi.mock('@/lib/documents/utils', () => ({
+vi.mock('@/lib/knowledge/documents/utils', () => ({
   retryWithExponentialBackoff: (fn: any) => fn(),
 }))
 
-vi.mock('@/lib/documents/document-processor', () => ({
+vi.mock('@/lib/knowledge/documents/document-processor', () => ({
   processDocument: vi.fn().mockResolvedValue({
     chunks: [
       {
@@ -149,12 +149,12 @@ vi.mock('@/db', () => {
   }
 })
 
+import { generateEmbeddings } from '@/lib/embeddings/utils'
+import { processDocumentAsync } from '@/lib/knowledge/documents/service'
 import {
   checkChunkAccess,
   checkDocumentAccess,
   checkKnowledgeBaseAccess,
-  generateEmbeddings,
-  processDocumentAsync,
 } from '@/app/api/knowledge/utils'
 
 describe('Knowledge Utils', () => {

apps/sim/app/api/knowledge/utils.ts

@@ -1,35 +1,8 @@
-import crypto from 'crypto'
 import { and, eq, isNull } from 'drizzle-orm'
-import { processDocument } from '@/lib/documents/document-processor'
-import { generateEmbeddings } from '@/lib/embeddings/utils'
-import { createLogger } from '@/lib/logs/console/logger'
 import { getUserEntityPermissions } from '@/lib/permissions/utils'
 import { db } from '@/db'
 import { document, embedding, knowledgeBase } from '@/db/schema'
 
-const logger = createLogger('KnowledgeUtils')
-
-const TIMEOUTS = {
-  OVERALL_PROCESSING: 150000, // 150 seconds (2.5 minutes)
-  EMBEDDINGS_API: 60000, // 60 seconds per batch
-} as const
-
-/**
- * Create a timeout wrapper for async operations
- */
-function withTimeout<T>(
-  promise: Promise<T>,
-  timeoutMs: number,
-  operation = 'Operation'
-): Promise<T> {
-  return Promise.race([
-    promise,
-    new Promise<never>((_, reject) =>
-      setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs)
-    ),
-  ])
-}
-
 export interface KnowledgeBaseData {
   id: string
   userId: string
@@ -380,154 +353,3 @@ export async function checkChunkAccess(
     knowledgeBase: kbAccess.knowledgeBase!,
   }
 }
-
-// Export for external use
-export { generateEmbeddings }
-
-/**
- * Process a document asynchronously with full error handling
- */
-export async function processDocumentAsync(
-  knowledgeBaseId: string,
-  documentId: string,
-  docData: {
-    filename: string
-    fileUrl: string
-    fileSize: number
-    mimeType: string
-  },
-  processingOptions: {
-    chunkSize?: number
-    minCharactersPerChunk?: number
-    recipe?: string
-    lang?: string
-    chunkOverlap?: number
-  }
-): Promise<void> {
-  const startTime = Date.now()
-  try {
-    logger.info(`[${documentId}] Starting document processing: ${docData.filename}`)
-
-    // Set status to processing
-    await db
-      .update(document)
-      .set({
-        processingStatus: 'processing',
-        processingStartedAt: new Date(),
-        processingError: null, // Clear any previous error
-      })
-      .where(eq(document.id, documentId))
-
-    logger.info(`[${documentId}] Status updated to 'processing', starting document processor`)
-
-    // Wrap the entire processing operation with a 5-minute timeout
-    await withTimeout(
-      (async () => {
-        const processed = await processDocument(
-          docData.fileUrl,
-          docData.filename,
-          docData.mimeType,
-          processingOptions.chunkSize || 1000,
-          processingOptions.chunkOverlap || 200,
-          processingOptions.minCharactersPerChunk || 1
-        )
-
-        const now = new Date()
-
-        logger.info(
-          `[${documentId}] Document parsed successfully, generating embeddings for ${processed.chunks.length} chunks`
-        )
-
-        const chunkTexts = processed.chunks.map((chunk) => chunk.text)
-        const embeddings = chunkTexts.length > 0 ? await generateEmbeddings(chunkTexts) : []
-
-        logger.info(`[${documentId}] Embeddings generated, fetching document tags`)
-
-        // Fetch document to get tags
-        const documentRecord = await db
-          .select({
-            tag1: document.tag1,
-            tag2: document.tag2,
-            tag3: document.tag3,
-            tag4: document.tag4,
-            tag5: document.tag5,
-            tag6: document.tag6,
-            tag7: document.tag7,
-          })
-          .from(document)
-          .where(eq(document.id, documentId))
-          .limit(1)
-
-        const documentTags = documentRecord[0] || {}
-
-        logger.info(`[${documentId}] Creating embedding records with tags`)
-
-        const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({
-          id: crypto.randomUUID(),
-          knowledgeBaseId,
-          documentId,
-          chunkIndex,
-          chunkHash: crypto.createHash('sha256').update(chunk.text).digest('hex'),
-          content: chunk.text,
-          contentLength: chunk.text.length,
-          tokenCount: Math.ceil(chunk.text.length / 4),
-          embedding: embeddings[chunkIndex] || null,
-          embeddingModel: 'text-embedding-3-small',
-          startOffset: chunk.metadata.startIndex,
-          endOffset: chunk.metadata.endIndex,
-          // Copy tags from document
-          tag1: documentTags.tag1,
-          tag2: documentTags.tag2,
-          tag3: documentTags.tag3,
-          tag4: documentTags.tag4,
-          tag5: documentTags.tag5,
-          tag6: documentTags.tag6,
-          tag7: documentTags.tag7,
-          createdAt: now,
-          updatedAt: now,
-        }))
-
-        await db.transaction(async (tx) => {
-          if (embeddingRecords.length > 0) {
-            await tx.insert(embedding).values(embeddingRecords)
-          }
-
-          await tx
-            .update(document)
-            .set({
-              chunkCount: processed.metadata.chunkCount,
-              tokenCount: processed.metadata.tokenCount,
-              characterCount: processed.metadata.characterCount,
-              processingStatus: 'completed',
-              processingCompletedAt: now,
-              processingError: null,
-            })
-            .where(eq(document.id, documentId))
-        })
-      })(),
-      TIMEOUTS.OVERALL_PROCESSING,
-      'Document processing'
-    )
-
-    const processingTime = Date.now() - startTime
-    logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`)
-  } catch (error) {
-    const processingTime = Date.now() - startTime
-    logger.error(`[${documentId}] Failed to process document after ${processingTime}ms:`, {
-      error: error instanceof Error ? error.message : 'Unknown error',
-      stack: error instanceof Error ? error.stack : undefined,
-      filename: docData.filename,
-      fileUrl: docData.fileUrl,
-      mimeType: docData.mimeType,
-    })
-
-    await db
-      .update(document)
-      .set({
-        processingStatus: 'failed',
-        processingError: error instanceof Error ? error.message : 'Unknown error',
-        processingCompletedAt: new Date(),
-      })
-      .where(eq(document.id, documentId))
-  }
-}

apps/sim/app/api/logs/[id]/route.ts

@@ -2,6 +2,7 @@ import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { permissions, workflow, workflowExecutionLogs } from '@/db/schema'
 
@@ -10,7 +11,7 @@ const logger = createLogger('LogDetailsByIdAPI')
 export const revalidate = 0
 
 export async function GET(_request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const session = await getSession()

apps/sim/app/api/logs/execution/[executionId]/route.ts

@@ -4,7 +4,7 @@ import { createLogger } from '@/lib/logs/console/logger'
 import { db } from '@/db'
 import { workflowExecutionLogs, workflowExecutionSnapshots } from '@/db/schema'
 
-const logger = createLogger('FrozenCanvasAPI')
+const logger = createLogger('LogsByExecutionIdAPI')
 
 export async function GET(
   _request: NextRequest,
@@ -13,7 +13,7 @@ export async function GET(
   try {
     const { executionId } = await params
 
-    logger.debug(`Fetching frozen canvas data for execution: ${executionId}`)
+    logger.debug(`Fetching execution data for: ${executionId}`)
 
     // Get the workflow execution log to find the snapshot
     const [workflowLog] = await db
@@ -50,14 +50,14 @@ export async function GET(
       },
     }
 
-    logger.debug(`Successfully fetched frozen canvas data for execution: ${executionId}`)
+    logger.debug(`Successfully fetched execution data for: ${executionId}`)
     logger.debug(
       `Workflow state contains ${Object.keys((snapshot.stateData as any)?.blocks || {}).length} blocks`
     )
 
     return NextResponse.json(response)
   } catch (error) {
-    logger.error('Error fetching frozen canvas data:', error)
-    return NextResponse.json({ error: 'Failed to fetch frozen canvas data' }, { status: 500 })
+    logger.error('Error fetching execution data:', error)
+    return NextResponse.json({ error: 'Failed to fetch execution data' }, { status: 500 })
   }
 }

apps/sim/app/api/logs/route.ts

@@ -3,44 +3,12 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { permissions, workflow, workflowExecutionLogs } from '@/db/schema'
 
 const logger = createLogger('LogsAPI')
 
-// Helper function to extract block executions from trace spans
-function extractBlockExecutionsFromTraceSpans(traceSpans: any[]): any[] {
-  const blockExecutions: any[] = []
-
-  function processSpan(span: any) {
-    if (span.blockId) {
-      blockExecutions.push({
-        id: span.id,
-        blockId: span.blockId,
-        blockName: span.name || '',
-        blockType: span.type,
-        startedAt: span.startTime,
-        endedAt: span.endTime,
-        durationMs: span.duration || 0,
-        status: span.status || 'success',
-        errorMessage: span.output?.error || undefined,
-        inputData: span.input || {},
-        outputData: span.output || {},
-        cost: span.cost || undefined,
-        metadata: {},
-      })
-    }
-
-    // Process children recursively
-    if (span.children && Array.isArray(span.children)) {
-      span.children.forEach(processSpan)
-    }
-  }
-
-  traceSpans.forEach(processSpan)
-  return blockExecutions
-}
-
 export const revalidate = 0
 
 const QueryParamsSchema = z.object({
@@ -58,7 +26,7 @@ const QueryParamsSchema = z.object({
 })
 
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     const session = await getSession()

apps/sim/app/api/mcp/servers/[id]/refresh/route.ts

@@ -0,0 +1,99 @@
+import { and, eq, isNull } from 'drizzle-orm'
+import type { NextRequest } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { withMcpAuth } from '@/lib/mcp/middleware'
+import { mcpService } from '@/lib/mcp/service'
+import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
+import { db } from '@/db'
+import { mcpServers } from '@/db/schema'
+
+const logger = createLogger('McpServerRefreshAPI')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * POST - Refresh an MCP server connection (requires any workspace permission)
+ */
+export const POST = withMcpAuth('read')(
+  async (
+    request: NextRequest,
+    { userId, workspaceId, requestId },
+    { params }: { params: { id: string } }
+  ) => {
+    const serverId = params.id
+
+    try {
+      logger.info(
+        `[${requestId}] Refreshing MCP server: ${serverId} in workspace: ${workspaceId}`,
+        {
+          userId,
+        }
+      )
+
+      const [server] = await db
+        .select()
+        .from(mcpServers)
+        .where(
+          and(
+            eq(mcpServers.id, serverId),
+            eq(mcpServers.workspaceId, workspaceId),
+            isNull(mcpServers.deletedAt)
+          )
+        )
+        .limit(1)
+
+      if (!server) {
+        return createMcpErrorResponse(
+          new Error('Server not found or access denied'),
+          'Server not found',
+          404
+        )
+      }
+
+      let connectionStatus: 'connected' | 'disconnected' | 'error' = 'error'
+      let toolCount = 0
+      let lastError: string | null = null
+
+      try {
+        const tools = await mcpService.discoverServerTools(userId, serverId, workspaceId)
+        connectionStatus = 'connected'
+        toolCount = tools.length
+        logger.info(
+          `[${requestId}] Successfully connected to server ${serverId}, discovered ${toolCount} tools`
+        )
+      } catch (error) {
+        connectionStatus = 'error'
+        lastError = error instanceof Error ? error.message : 'Connection test failed'
+        logger.warn(`[${requestId}] Failed to connect to server ${serverId}:`, error)
+      }
+
+      const [refreshedServer] = await db
+        .update(mcpServers)
+        .set({
+          lastToolsRefresh: new Date(),
+          connectionStatus,
+          lastError,
+          lastConnected: connectionStatus === 'connected' ? new Date() : server.lastConnected,
+          toolCount,
+          updatedAt: new Date(),
+        })
+        .where(eq(mcpServers.id, serverId))
+        .returning()
+
+      logger.info(`[${requestId}] Successfully refreshed MCP server: ${serverId}`)
+      return createMcpSuccessResponse({
+        status: connectionStatus,
+        toolCount,
+        lastConnected: refreshedServer?.lastConnected?.toISOString() || null,
+        error: lastError,
+      })
+    } catch (error) {
+      logger.error(`[${requestId}] Error refreshing MCP server:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to refresh MCP server'),
+        'Failed to refresh MCP server',
+        500
+      )
+    }
+  }
+)

apps/sim/app/api/mcp/servers/[id]/route.ts

@@ -0,0 +1,92 @@
+import { and, eq, isNull } from 'drizzle-orm'
+import type { NextRequest } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { mcpService } from '@/lib/mcp/service'
+import { validateMcpServerUrl } from '@/lib/mcp/url-validator'
+import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
+import { db } from '@/db'
+import { mcpServers } from '@/db/schema'
+
+const logger = createLogger('McpServerAPI')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * PATCH - Update an MCP server in the workspace (requires write or admin permission)
+ */
+export const PATCH = withMcpAuth('write')(
+  async (
+    request: NextRequest,
+    { userId, workspaceId, requestId },
+    { params }: { params: { id: string } }
+  ) => {
+    const serverId = params.id
+
+    try {
+      const body = getParsedBody(request) || (await request.json())
+
+      logger.info(`[${requestId}] Updating MCP server: ${serverId} in workspace: ${workspaceId}`, {
+        userId,
+        updates: Object.keys(body).filter((k) => k !== 'workspaceId'),
+      })
+
+      // Validate URL if being updated
+      if (
+        body.url &&
+        (body.transport === 'http' ||
+          body.transport === 'sse' ||
+          body.transport === 'streamable-http')
+      ) {
+        const urlValidation = validateMcpServerUrl(body.url)
+        if (!urlValidation.isValid) {
+          return createMcpErrorResponse(
+            new Error(`Invalid MCP server URL: ${urlValidation.error}`),
+            'Invalid server URL',
+            400
+          )
+        }
+        body.url = urlValidation.normalizedUrl
+      }
+
+      // Remove workspaceId from body to prevent it from being updated
+      const { workspaceId: _, ...updateData } = body
+
+      const [updatedServer] = await db
+        .update(mcpServers)
+        .set({
+          ...updateData,
+          updatedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(mcpServers.id, serverId),
+            eq(mcpServers.workspaceId, workspaceId),
+            isNull(mcpServers.deletedAt)
+          )
+        )
+        .returning()
+
+      if (!updatedServer) {
+        return createMcpErrorResponse(
+          new Error('Server not found or access denied'),
+          'Server not found',
+          404
+        )
+      }
+
+      // Clear MCP service cache after update
+      mcpService.clearCache(workspaceId)
+
+      logger.info(`[${requestId}] Successfully updated MCP server: ${serverId}`)
+      return createMcpSuccessResponse({ server: updatedServer })
+    } catch (error) {
+      logger.error(`[${requestId}] Error updating MCP server:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to update MCP server'),
+        'Failed to update MCP server',
+        500
+      )
+    }
+  }
+)

apps/sim/app/api/mcp/servers/route.ts

@@ -0,0 +1,166 @@
+import { and, eq, isNull } from 'drizzle-orm'
+import type { NextRequest } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { mcpService } from '@/lib/mcp/service'
+import type { McpTransport } from '@/lib/mcp/types'
+import { validateMcpServerUrl } from '@/lib/mcp/url-validator'
+import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
+import { db } from '@/db'
+import { mcpServers } from '@/db/schema'
+
+const logger = createLogger('McpServersAPI')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * Check if transport type requires a URL
+ */
+function isUrlBasedTransport(transport: McpTransport): boolean {
+  return transport === 'http' || transport === 'sse' || transport === 'streamable-http'
+}
+
+/**
+ * GET - List all registered MCP servers for the workspace
+ */
+export const GET = withMcpAuth('read')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      logger.info(`[${requestId}] Listing MCP servers for workspace ${workspaceId}`)
+
+      const servers = await db
+        .select()
+        .from(mcpServers)
+        .where(and(eq(mcpServers.workspaceId, workspaceId), isNull(mcpServers.deletedAt)))
+
+      logger.info(
+        `[${requestId}] Listed ${servers.length} MCP servers for workspace ${workspaceId}`
+      )
+      return createMcpSuccessResponse({ servers })
+    } catch (error) {
+      logger.error(`[${requestId}] Error listing MCP servers:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to list MCP servers'),
+        'Failed to list MCP servers',
+        500
+      )
+    }
+  }
+)
+
+/**
+ * POST - Register a new MCP server for the workspace (requires write permission)
+ */
+export const POST = withMcpAuth('write')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const body = getParsedBody(request) || (await request.json())
+
+      logger.info(`[${requestId}] Registering new MCP server:`, {
+        name: body.name,
+        transport: body.transport,
+        workspaceId,
+      })
+
+      if (!body.name || !body.transport) {
+        return createMcpErrorResponse(
+          new Error('Missing required fields: name or transport'),
+          'Missing required fields',
+          400
+        )
+      }
+
+      if (isUrlBasedTransport(body.transport) && body.url) {
+        const urlValidation = validateMcpServerUrl(body.url)
+        if (!urlValidation.isValid) {
+          return createMcpErrorResponse(
+            new Error(`Invalid MCP server URL: ${urlValidation.error}`),
+            'Invalid server URL',
+            400
+          )
+        }
+        body.url = urlValidation.normalizedUrl
+      }
+
+      const serverId = body.id || crypto.randomUUID()
+
+      await db
+        .insert(mcpServers)
+        .values({
+          id: serverId,
+          workspaceId,
+          createdBy: userId,
+          name: body.name,
+          description: body.description,
+          transport: body.transport,
+          url: body.url,
+          headers: body.headers || {},
+          timeout: body.timeout || 30000,
+          retries: body.retries || 3,
+          enabled: body.enabled !== false,
+          createdAt: new Date(),
+          updatedAt: new Date(),
+        })
+        .returning()
+
+      mcpService.clearCache(workspaceId)
+
+      logger.info(`[${requestId}] Successfully registered MCP server: ${body.name}`)
+      return createMcpSuccessResponse({ serverId }, 201)
+    } catch (error) {
+      logger.error(`[${requestId}] Error registering MCP server:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to register MCP server'),
+        'Failed to register MCP server',
+        500
+      )
+    }
+  }
+)
+
+/**
+ * DELETE - Delete an MCP server from the workspace (requires admin permission)
+ */
+export const DELETE = withMcpAuth('admin')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const { searchParams } = new URL(request.url)
+      const serverId = searchParams.get('serverId')
+
+      if (!serverId) {
+        return createMcpErrorResponse(
+          new Error('serverId parameter is required'),
+          'Missing required parameter',
+          400
+        )
+      }
+
+      logger.info(`[${requestId}] Deleting MCP server: ${serverId} from workspace: ${workspaceId}`)
+
+      const [deletedServer] = await db
+        .delete(mcpServers)
+        .where(and(eq(mcpServers.id, serverId), eq(mcpServers.workspaceId, workspaceId)))
+        .returning()
+
+      if (!deletedServer) {
+        return createMcpErrorResponse(
+          new Error('Server not found or access denied'),
+          'Server not found',
+          404
+        )
+      }
+
+      mcpService.clearCache(workspaceId)
+
+      logger.info(`[${requestId}] Successfully deleted MCP server: ${serverId}`)
+      return createMcpSuccessResponse({ message: `Server ${serverId} deleted successfully` })
+    } catch (error) {
+      logger.error(`[${requestId}] Error deleting MCP server:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to delete MCP server'),
+        'Failed to delete MCP server',
+        500
+      )
+    }
+  }
+)

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -0,0 +1,209 @@
+import type { NextRequest } from 'next/server'
+import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
+import { createLogger } from '@/lib/logs/console/logger'
+import { McpClient } from '@/lib/mcp/client'
+import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import type { McpServerConfig, McpTransport } from '@/lib/mcp/types'
+import { validateMcpServerUrl } from '@/lib/mcp/url-validator'
+import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
+
+const logger = createLogger('McpServerTestAPI')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * Check if transport type requires a URL
+ */
+function isUrlBasedTransport(transport: McpTransport): boolean {
+  return transport === 'http' || transport === 'sse' || transport === 'streamable-http'
+}
+
+/**
+ * Resolve environment variables in strings
+ */
+function resolveEnvVars(value: string, envVars: Record<string, string>): string {
+  const envMatches = value.match(/\{\{([^}]+)\}\}/g)
+  if (!envMatches) return value
+
+  let resolvedValue = value
+  for (const match of envMatches) {
+    const envKey = match.slice(2, -2).trim()
+    const envValue = envVars[envKey]
+
+    if (envValue === undefined) {
+      logger.warn(`Environment variable "${envKey}" not found in MCP server test`)
+      continue
+    }
+
+    resolvedValue = resolvedValue.replace(match, envValue)
+  }
+  return resolvedValue
+}
+
+interface TestConnectionRequest {
+  name: string
+  transport: McpTransport
+  url?: string
+  headers?: Record<string, string>
+  timeout?: number
+  workspaceId: string
+}
+
+interface TestConnectionResult {
+  success: boolean
+  error?: string
+  serverInfo?: {
+    name: string
+    version: string
+  }
+  negotiatedVersion?: string
+  supportedCapabilities?: string[]
+  toolCount?: number
+  warnings?: string[]
+}
+
+/**
+ * POST - Test connection to an MCP server before registering it
+ */
+export const POST = withMcpAuth('write')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const body: TestConnectionRequest = getParsedBody(request) || (await request.json())
+
+      logger.info(`[${requestId}] Testing MCP server connection:`, {
+        name: body.name,
+        transport: body.transport,
+        url: body.url ? `${body.url.substring(0, 50)}...` : undefined, // Partial URL for security
+        workspaceId,
+      })
+
+      if (!body.name || !body.transport) {
+        return createMcpErrorResponse(
+          new Error('Missing required fields: name and transport are required'),
+          'Missing required fields',
+          400
+        )
+      }
+
+      if (isUrlBasedTransport(body.transport)) {
+        if (!body.url) {
+          return createMcpErrorResponse(
+            new Error('URL is required for HTTP-based transports'),
+            'Missing required URL',
+            400
+          )
+        }
+
+        const urlValidation = validateMcpServerUrl(body.url)
+        if (!urlValidation.isValid) {
+          return createMcpErrorResponse(
+            new Error(`Invalid MCP server URL: ${urlValidation.error}`),
+            'Invalid server URL',
+            400
+          )
+        }
+        body.url = urlValidation.normalizedUrl
+      }
+
+      let resolvedUrl = body.url
+      let resolvedHeaders = body.headers || {}
+
+      try {
+        const envVars = await getEffectiveDecryptedEnv(userId, workspaceId)
+
+        if (resolvedUrl) {
+          resolvedUrl = resolveEnvVars(resolvedUrl, envVars)
+        }
+
+        const resolvedHeadersObj: Record<string, string> = {}
+        for (const [key, value] of Object.entries(resolvedHeaders)) {
+          resolvedHeadersObj[key] = resolveEnvVars(value, envVars)
+        }
+        resolvedHeaders = resolvedHeadersObj
+      } catch (envError) {
+        logger.warn(
+          `[${requestId}] Failed to resolve environment variables, using raw values:`,
+          envError
+        )
+      }
+
+      const testConfig: McpServerConfig = {
+        id: `test-${requestId}`,
+        name: body.name,
+        transport: body.transport,
+        url: resolvedUrl,
+        headers: resolvedHeaders,
+        timeout: body.timeout || 10000,
+        retries: 1, // Only one retry for tests
+        enabled: true,
+      }
+
+      const testSecurityPolicy = {
+        requireConsent: false,
+        auditLevel: 'none' as const,
+        maxToolExecutionsPerHour: 0,
+      }
+
+      const result: TestConnectionResult = { success: false }
+      let client: McpClient | null = null
+
+      try {
+        client = new McpClient(testConfig, testSecurityPolicy)
+        await client.connect()
+
+        result.success = true
+        result.negotiatedVersion = client.getNegotiatedVersion()
+
+        try {
+          const tools = await client.listTools()
+          result.toolCount = tools.length
+        } catch (toolError) {
+          logger.warn(`[${requestId}] Could not list tools from test server:`, toolError)
+          result.warnings = result.warnings || []
+          result.warnings.push('Could not list tools from server')
+        }
+
+        const clientVersionInfo = McpClient.getVersionInfo()
+        if (result.negotiatedVersion !== clientVersionInfo.preferred) {
+          result.warnings = result.warnings || []
+          result.warnings.push(
+            `Server uses protocol version '${result.negotiatedVersion}' instead of preferred '${clientVersionInfo.preferred}'`
+          )
+        }
+
+        logger.info(`[${requestId}] MCP server test successful:`, {
+          name: body.name,
+          negotiatedVersion: result.negotiatedVersion,
+          toolCount: result.toolCount,
+          capabilities: result.supportedCapabilities,
+        })
+      } catch (error) {
+        logger.warn(`[${requestId}] MCP server test failed:`, error)
+
+        result.success = false
+        if (error instanceof Error) {
+          result.error = error.message
+        } else {
+          result.error = 'Unknown connection error'
+        }
+      } finally {
+        if (client) {
+          try {
+            await client.disconnect()
+          } catch (disconnectError) {
+            logger.debug(`[${requestId}] Test client disconnect error (expected):`, disconnectError)
+          }
+        }
+      }
+
+      return createMcpSuccessResponse(result, result.success ? 200 : 400)
+    } catch (error) {
+      logger.error(`[${requestId}] Error testing MCP server connection:`, error)
+      return createMcpErrorResponse(
+        error instanceof Error ? error : new Error('Failed to test server connection'),
+        'Failed to test server connection',
+        500
+      )
+    }
+  }
+)

apps/sim/app/api/mcp/tools/discover/route.ts

@@ -0,0 +1,122 @@
+import type { NextRequest } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { mcpService } from '@/lib/mcp/service'
+import type { McpToolDiscoveryResponse } from '@/lib/mcp/types'
+import { categorizeError, createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
+
+const logger = createLogger('McpToolDiscoveryAPI')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * GET - Discover all tools from user's MCP servers
+ */
+export const GET = withMcpAuth('read')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const { searchParams } = new URL(request.url)
+      const serverId = searchParams.get('serverId')
+      const forceRefresh = searchParams.get('refresh') === 'true'
+
+      logger.info(`[${requestId}] Discovering MCP tools for user ${userId}`, {
+        serverId,
+        workspaceId,
+        forceRefresh,
+      })
+
+      let tools
+      if (serverId) {
+        tools = await mcpService.discoverServerTools(userId, serverId, workspaceId)
+      } else {
+        tools = await mcpService.discoverTools(userId, workspaceId, forceRefresh)
+      }
+
+      const byServer: Record<string, number> = {}
+      for (const tool of tools) {
+        byServer[tool.serverId] = (byServer[tool.serverId] || 0) + 1
+      }
+
+      const responseData: McpToolDiscoveryResponse = {
+        tools,
+        totalCount: tools.length,
+        byServer,
+      }
+
+      logger.info(
+        `[${requestId}] Discovered ${tools.length} tools from ${Object.keys(byServer).length} servers`
+      )
+      return createMcpSuccessResponse(responseData)
+    } catch (error) {
+      logger.error(`[${requestId}] Error discovering MCP tools:`, error)
+      const { message, status } = categorizeError(error)
+      return createMcpErrorResponse(new Error(message), 'Failed to discover MCP tools', status)
+    }
+  }
+)
+
+/**
+ * POST - Refresh tool discovery for specific servers
+ */
+export const POST = withMcpAuth('read')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const body = getParsedBody(request) || (await request.json())
+      const { serverIds } = body
+
+      if (!Array.isArray(serverIds)) {
+        return createMcpErrorResponse(
+          new Error('serverIds must be an array'),
+          'Invalid request format',
+          400
+        )
+      }
+
+      logger.info(
+        `[${requestId}] Refreshing tool discovery for user ${userId}, servers:`,
+        serverIds
+      )
+
+      const results = await Promise.allSettled(
+        serverIds.map(async (serverId: string) => {
+          const tools = await mcpService.discoverServerTools(userId, serverId, workspaceId)
+          return { serverId, toolCount: tools.length }
+        })
+      )
+
+      const successes: Array<{ serverId: string; toolCount: number }> = []
+      const failures: Array<{ serverId: string; error: string }> = []
+
+      results.forEach((result, index) => {
+        const serverId = serverIds[index]
+        if (result.status === 'fulfilled') {
+          successes.push(result.value)
+        } else {
+          failures.push({
+            serverId,
+            error: result.reason instanceof Error ? result.reason.message : 'Unknown error',
+          })
+        }
+      })
+
+      const responseData = {
+        refreshed: successes,
+        failed: failures,
+        summary: {
+          total: serverIds.length,
+          successful: successes.length,
+          failed: failures.length,
+        },
+      }
+
+      logger.info(
+        `[${requestId}] Tool discovery refresh completed: ${successes.length}/${serverIds.length} successful`
+      )
+      return createMcpSuccessResponse(responseData)
+    } catch (error) {
+      logger.error(`[${requestId}] Error refreshing tool discovery:`, error)
+      const { message, status } = categorizeError(error)
+      return createMcpErrorResponse(new Error(message), 'Failed to refresh tool discovery', status)
+    }
+  }
+)

apps/sim/app/api/mcp/tools/execute/route.ts

@@ -0,0 +1,252 @@
+import type { NextRequest } from 'next/server'
+import { createLogger } from '@/lib/logs/console/logger'
+import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { mcpService } from '@/lib/mcp/service'
+import type { McpTool, McpToolCall, McpToolResult } from '@/lib/mcp/types'
+import {
+  categorizeError,
+  createMcpErrorResponse,
+  createMcpSuccessResponse,
+  MCP_CONSTANTS,
+  validateStringParam,
+} from '@/lib/mcp/utils'
+
+const logger = createLogger('McpToolExecutionAPI')
+
+export const dynamic = 'force-dynamic'
+
+// Type definitions for improved type safety
+interface SchemaProperty {
+  type: 'string' | 'number' | 'boolean' | 'object' | 'array'
+  description?: string
+  enum?: unknown[]
+  format?: string
+  items?: SchemaProperty
+  properties?: Record<string, SchemaProperty>
+}
+
+interface ToolExecutionResult {
+  success: boolean
+  output?: McpToolResult
+  error?: string
+}
+
+/**
+ * Type guard to safely check if a schema property has a type field
+ */
+function hasType(prop: unknown): prop is SchemaProperty {
+  return typeof prop === 'object' && prop !== null && 'type' in prop
+}
+
+/**
+ * POST - Execute a tool on an MCP server
+ */
+export const POST = withMcpAuth('read')(
+  async (request: NextRequest, { userId, workspaceId, requestId }) => {
+    try {
+      const body = getParsedBody(request) || (await request.json())
+
+      logger.info(`[${requestId}] MCP tool execution request received`, {
+        hasAuthHeader: !!request.headers.get('authorization'),
+        authHeaderType: request.headers.get('authorization')?.substring(0, 10),
+        bodyKeys: Object.keys(body),
+        serverId: body.serverId,
+        toolName: body.toolName,
+        hasWorkflowId: !!body.workflowId,
+        workflowId: body.workflowId,
+        userId: userId,
+      })
+
+      const { serverId, toolName, arguments: args } = body
+
+      const serverIdValidation = validateStringParam(serverId, 'serverId')
+      if (!serverIdValidation.isValid) {
+        logger.warn(`[${requestId}] Invalid serverId: ${serverId}`)
+        return createMcpErrorResponse(new Error(serverIdValidation.error), 'Invalid serverId', 400)
+      }
+
+      const toolNameValidation = validateStringParam(toolName, 'toolName')
+      if (!toolNameValidation.isValid) {
+        logger.warn(`[${requestId}] Invalid toolName: ${toolName}`)
+        return createMcpErrorResponse(new Error(toolNameValidation.error), 'Invalid toolName', 400)
+      }
+
+      logger.info(
+        `[${requestId}] Executing tool ${toolName} on server ${serverId} for user ${userId} in workspace ${workspaceId}`
+      )
+
+      let tool = null
+      try {
+        const tools = await mcpService.discoverServerTools(userId, serverId, workspaceId)
+        tool = tools.find((t) => t.name === toolName)
+
+        if (!tool) {
+          return createMcpErrorResponse(
+            new Error(
+              `Tool ${toolName} not found on server ${serverId}. Available tools: ${tools.map((t) => t.name).join(', ')}`
+            ),
+            'Tool not found',
+            404
+          )
+        }
+
+        // Parse array arguments based on tool schema
+        if (tool.inputSchema?.properties) {
+          for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) {
+            const schema = paramSchema as any
+            if (
+              schema.type === 'array' &&
+              args[paramName] !== undefined &&
+              typeof args[paramName] === 'string'
+            ) {
+              const stringValue = args[paramName].trim()
+              if (stringValue) {
+                try {
+                  // Try to parse as JSON first (handles ["item1", "item2"])
+                  const parsed = JSON.parse(stringValue)
+                  if (Array.isArray(parsed)) {
+                    args[paramName] = parsed
+                  } else {
+                    // JSON parsed but not an array, wrap in array
+                    args[paramName] = [parsed]
+                  }
+                } catch (error) {
+                  // JSON parsing failed - treat as comma-separated if contains commas, otherwise single item
+                  if (stringValue.includes(',')) {
+                    args[paramName] = stringValue
+                      .split(',')
+                      .map((item) => item.trim())
+                      .filter((item) => item)
+                  } else {
+                    // Single item - wrap in array since schema expects array
+                    args[paramName] = [stringValue]
+                  }
+                }
+              } else {
+                // Empty string becomes empty array
+                args[paramName] = []
+              }
+            }
+          }
+        }
+      } catch (error) {
+        logger.warn(
+          `[${requestId}] Failed to discover tools for validation, proceeding anyway:`,
+          error
+        )
+      }
+
+      if (tool) {
+        const validationError = validateToolArguments(tool, args)
+        if (validationError) {
+          logger.warn(`[${requestId}] Tool validation failed: ${validationError}`)
+          return createMcpErrorResponse(
+            new Error(`Invalid arguments for tool ${toolName}: ${validationError}`),
+            'Invalid tool arguments',
+            400
+          )
+        }
+      }
+
+      const toolCall: McpToolCall = {
+        name: toolName,
+        arguments: args || {},
+      }
+
+      const result = await Promise.race([
+        mcpService.executeTool(userId, serverId, toolCall, workspaceId),
+        new Promise<never>((_, reject) =>
+          setTimeout(
+            () => reject(new Error('Tool execution timeout')),
+            MCP_CONSTANTS.EXECUTION_TIMEOUT
+          )
+        ),
+      ])
+
+      const transformedResult = transformToolResult(result)
+
+      if (result.isError) {
+        logger.warn(`[${requestId}] Tool execution returned error for ${toolName} on ${serverId}`)
+        return createMcpErrorResponse(
+          transformedResult,
+          transformedResult.error || 'Tool execution failed',
+          400
+        )
+      }
+      logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`)
+      return createMcpSuccessResponse(transformedResult)
+    } catch (error) {
+      logger.error(`[${requestId}] Error executing MCP tool:`, error)
+
+      const { message, status } = categorizeError(error)
+      return createMcpErrorResponse(new Error(message), message, status)
+    }
+  }
+)
+
+/**
+ * Validate tool arguments against schema
+ */
+function validateToolArguments(tool: McpTool, args: Record<string, unknown>): string | null {
+  if (!tool.inputSchema) {
+    return null // No schema to validate against
+  }
+
+  const schema = tool.inputSchema
+
+  if (schema.required && Array.isArray(schema.required)) {
+    for (const requiredProp of schema.required) {
+      if (!(requiredProp in (args || {}))) {
+        return `Missing required property: ${requiredProp}`
+      }
+    }
+  }
+
+  if (schema.properties && args) {
+    for (const [propName, propSchema] of Object.entries(schema.properties)) {
+      const propValue = args[propName]
+      if (propValue !== undefined && hasType(propSchema)) {
+        const expectedType = propSchema.type
+        const actualType = typeof propValue
+
+        if (expectedType === 'string' && actualType !== 'string') {
+          return `Property ${propName} must be a string`
+        }
+        if (expectedType === 'number' && actualType !== 'number') {
+          return `Property ${propName} must be a number`
+        }
+        if (expectedType === 'boolean' && actualType !== 'boolean') {
+          return `Property ${propName} must be a boolean`
+        }
+        if (
+          expectedType === 'object' &&
+          (actualType !== 'object' || propValue === null || Array.isArray(propValue))
+        ) {
+          return `Property ${propName} must be an object`
+        }
+        if (expectedType === 'array' && !Array.isArray(propValue)) {
+          return `Property ${propName} must be an array`
+        }
+      }
+    }
+  }
+
+  return null
+}
+
+/**
+ * Transform MCP tool result to platform format
+ */
+function transformToolResult(result: McpToolResult): ToolExecutionResult {
+  if (result.isError) {
+    return {
+      success: false,
+      error: result.content?.[0]?.text || 'Tool execution failed',
+    }
+  }
+
+  return {
+    success: true,
+    output: result,
+  }
+}

apps/sim/app/api/memory/[id]/route.ts

@@ -1,6 +1,7 @@
 import { and, eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { memory } from '@/db/schema'
 
@@ -13,7 +14,7 @@ export const runtime = 'nodejs'
  * GET handler for retrieving a specific memory by ID
  */
 export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {
@@ -85,7 +86,7 @@ export async function DELETE(
   request: NextRequest,
   { params }: { params: Promise<{ id: string }> }
 ) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {
@@ -156,7 +157,7 @@ export async function DELETE(
  * PUT handler for updating a specific memory
  */
 export async function PUT(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
   const { id } = await params
 
   try {

apps/sim/app/api/memory/route.ts

@@ -1,6 +1,7 @@
 import { and, eq, isNull, like } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console/logger'
+import { generateRequestId } from '@/lib/utils'
 import { db } from '@/db'
 import { memory } from '@/db/schema'
 
@@ -18,7 +19,7 @@ export const runtime = 'nodejs'
  * - workflowId: Filter by workflow ID (required)
  */
 export async function GET(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.info(`[${requestId}] Processing memory search request`)
@@ -101,7 +102,7 @@ export async function GET(request: NextRequest) {
  * - workflowId: ID of the workflow this memory belongs to
  */
 export async function POST(request: NextRequest) {
-  const requestId = crypto.randomUUID().slice(0, 8)
+  const requestId = generateRequestId()
 
   try {
     logger.info(`[${requestId}] Processing memory creation request`)

apps/sim/app/api/organizations/[id]/invitations/[invitationId]/route.ts

@@ -0,0 +1,198 @@
+import { randomUUID } from 'crypto'
+import { and, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { getSession } from '@/lib/auth'
+import { createLogger } from '@/lib/logs/console/logger'
+import { db } from '@/db'
+import {
+  invitation,
+  member,
+  organization,
+  permissions,
+  user,
+  type WorkspaceInvitationStatus,
+  workspaceInvitation,
+} from '@/db/schema'
+
+const logger = createLogger('OrganizationInvitation')
+
+// Get invitation details
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: Promise<{ id: string; invitationId: string }> }
+) {
+  const { id: organizationId, invitationId } = await params
+  const session = await getSession()
+
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  try {
+    const orgInvitation = await db
+      .select()
+      .from(invitation)
+      .where(and(eq(invitation.id, invitationId), eq(invitation.organizationId, organizationId)))
+      .then((rows) => rows[0])
+
+    if (!orgInvitation) {
+      return NextResponse.json({ error: 'Invitation not found' }, { status: 404 })
+    }
+
+    const org = await db
+      .select()
+      .from(organization)
+      .where(eq(organization.id, organizationId))
+      .then((rows) => rows[0])
+
+    if (!org) {
+      return NextResponse.json({ error: 'Organization not found' }, { status: 404 })
+    }
+
+    return NextResponse.json({
+      invitation: orgInvitation,
+      organization: org,
+    })
+  } catch (error) {
+    logger.error('Error fetching organization invitation:', error)
+    return NextResponse.json({ error: 'Failed to fetch invitation' }, { status: 500 })
+  }
+}
+
+export async function PUT(
+  req: NextRequest,
+  { params }: { params: Promise<{ id: string; invitationId: string }> }
+) {
+  const { id: organizationId, invitationId } = await params
+  const session = await getSession()
+
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  try {
+    const { status } = await req.json()
+
+    if (!status || !['accepted', 'rejected', 'cancelled'].includes(status)) {
+      return NextResponse.json(
+        { error: 'Invalid status. Must be "accepted", "rejected", or "cancelled"' },
+        { status: 400 }
+      )
+    }
+
+    const orgInvitation = await db
+      .select()
+      .from(invitation)
+      .where(and(eq(invitation.id, invitationId), eq(invitation.organizationId, organizationId)))
+      .then((rows) => rows[0])
+
+    if (!orgInvitation) {
+      return NextResponse.json({ error: 'Invitation not found' }, { status: 404 })
+    }
+
+    if (orgInvitation.status !== 'pending') {
+      return NextResponse.json({ error: 'Invitation already processed' }, { status: 400 })
+    }
+
+    if (status === 'accepted') {
+      const userData = await db
+        .select()
+        .from(user)
+        .where(eq(user.id, session.user.id))
+        .then((rows) => rows[0])
+
+      if (!userData || userData.email.toLowerCase() !== orgInvitation.email.toLowerCase()) {
+        return NextResponse.json(
+          { error: 'Email mismatch. You can only accept invitations sent to your email address.' },
+          { status: 403 }
+        )
+      }
+    }
+
+    if (status === 'cancelled') {
+      const isAdmin = await db
+        .select()
+        .from(member)
+        .where(
+          and(
+            eq(member.organizationId, organizationId),
+            eq(member.userId, session.user.id),
+            eq(member.role, 'admin')
+          )
+        )
+        .then((rows) => rows.length > 0)
+
+      if (!isAdmin) {
+        return NextResponse.json(
+          { error: 'Only organization admins can cancel invitations' },
+          { status: 403 }
+        )
+      }
+    }
+
+    await db.transaction(async (tx) => {
+      await tx.update(invitation).set({ status }).where(eq(invitation.id, invitationId))
+
+      if (status === 'accepted') {
+        await tx.insert(member).values({
+          id: randomUUID(),
+          userId: session.user.id,
+          organizationId,
+          role: orgInvitation.role,
+          createdAt: new Date(),
+        })
+
+        const linkedWorkspaceInvitations = await tx
+          .select()
+          .from(workspaceInvitation)
+          .where(
+            and(
+              eq(workspaceInvitation.orgInvitationId, invitationId),
+              eq(workspaceInvitation.status, 'pending' as WorkspaceInvitationStatus)
+            )
+          )
+
+        for (const wsInvitation of linkedWorkspaceInvitations) {
+          await tx
+            .update(workspaceInvitation)
+            .set({
+              status: 'accepted' as WorkspaceInvitationStatus,
+              updatedAt: new Date(),
+            })
+            .where(eq(workspaceInvitation.id, wsInvitation.id))
+
+          await tx.insert(permissions).values({
+            id: randomUUID(),
+            entityType: 'workspace',
+            entityId: wsInvitation.workspaceId,
+            userId: session.user.id,
+            permissionType: wsInvitation.permissions || 'read',
+            createdAt: new Date(),
+            updatedAt: new Date(),
+          })
+        }
+      } else if (status === 'cancelled') {
+        await tx
+          .update(workspaceInvitation)
+          .set({ status: 'cancelled' as WorkspaceInvitationStatus })
+          .where(eq(workspaceInvitation.orgInvitationId, invitationId))
+      }
+    })
+
+    logger.info(`Organization invitation ${status}`, {
+      organizationId,
+      invitationId,
+      userId: session.user.id,
+      email: orgInvitation.email,
+    })
+
+    return NextResponse.json({
+      success: true,
+      message: `Invitation ${status} successfully`,
+      invitation: { ...orgInvitation, status },
+    })
+  } catch (error) {
+    logger.error(`Error updating organization invitation:`, error)
+    return NextResponse.json({ error: 'Failed to update invitation' }, { status: 500 })
+  }
+}