mirror of
https://github.com/simstudioai/sim.git
synced 2026-04-28 03:00:29 -04:00
v0.6.58: queue abort state machine improvement, contributing guide
This commit is contained in:
Vikhyath Mondreti
259
.github/CONTRIBUTING.md
vendored
259
.github/CONTRIBUTING.md
vendored
@@ -2,8 +2,15 @@
|
||||
|
||||
Thank you for your interest in contributing to Sim! Our goal is to provide developers with a powerful, user-friendly platform for building, testing, and optimizing agentic workflows. We welcome contributions in all forms—from bug fixes and design improvements to brand-new features.
|
||||
|
||||
> **Project Overview:**
|
||||
> Sim is a monorepo using Turborepo, containing the main application (`apps/sim/`), documentation (`apps/docs/`), and shared packages (`packages/`). The main application is built with Next.js (app router), ReactFlow, Zustand, Shadcn, and Tailwind CSS. Please ensure your contributions follow our best practices for clarity, maintainability, and consistency.
|
||||
> **Project Overview:**
|
||||
> Sim is a Turborepo monorepo with two deployable apps and a set of shared packages:
|
||||
>
|
||||
> - `apps/sim/` — the main Next.js application (App Router, ReactFlow, Zustand, Shadcn, Tailwind CSS).
|
||||
> - `apps/realtime/` — a small Bun + Socket.IO server that powers the collaborative canvas. Shares DB and Better Auth secrets with `apps/sim` via `@sim/*` packages.
|
||||
> - `apps/docs/` — Fumadocs-based documentation site.
|
||||
> - `packages/` — shared workspace packages (`@sim/db`, `@sim/auth`, `@sim/audit`, `@sim/workflow-types`, `@sim/workflow-persistence`, `@sim/workflow-authz`, `@sim/realtime-protocol`, `@sim/security`, `@sim/logger`, `@sim/utils`, `@sim/testing`, `@sim/tsconfig`).
|
||||
>
|
||||
> Strict one-way dependency flow: `apps/* → packages/*`. Packages never import from apps. Please ensure your contributions follow this and our best practices for clarity, maintainability, and consistency.
|
||||
|
||||
---
|
||||
|
||||
@@ -24,14 +31,17 @@ Thank you for your interest in contributing to Sim! Our goal is to provide devel
|
||||
|
||||
We strive to keep our workflow as simple as possible. To contribute:
|
||||
|
||||
1. **Fork the Repository**
|
||||
1. **Fork the Repository**
|
||||
Click the **Fork** button on GitHub to create your own copy of the project.
|
||||
|
||||
2. **Clone Your Fork**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/sim.git
|
||||
cd sim
|
||||
```
|
||||
3. **Create a Feature Branch**
|
||||
|
||||
3. **Create a Feature Branch**
|
||||
Create a new branch with a descriptive name:
|
||||
|
||||
```bash
|
||||
@@ -40,21 +50,23 @@ We strive to keep our workflow as simple as possible. To contribute:
|
||||
|
||||
Use a clear naming convention to indicate the type of work (e.g., `feat/`, `fix/`, `docs/`).
|
||||
|
||||
4. **Make Your Changes**
|
||||
4. **Make Your Changes**
|
||||
Ensure your changes are small, focused, and adhere to our coding guidelines.
|
||||
|
||||
5. **Commit Your Changes**
|
||||
5. **Commit Your Changes**
|
||||
Write clear, descriptive commit messages that follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/#specification) specification. This allows us to maintain a coherent project history and generate changelogs automatically. For example:
|
||||
|
||||
- `feat(api): add new endpoint for user authentication`
|
||||
- `fix(ui): resolve button alignment issue`
|
||||
- `docs: update contribution guidelines`
|
||||
|
||||
6. **Push Your Branch**
|
||||
|
||||
```bash
|
||||
git push origin feat/your-feature-name
|
||||
```
|
||||
|
||||
7. **Create a Pull Request**
|
||||
7. **Create a Pull Request**
|
||||
Open a pull request against the `staging` branch on GitHub. Please provide a clear description of the changes and reference any relevant issues (e.g., `fixes #123`).
|
||||
|
||||
---
|
||||
@@ -65,7 +77,7 @@ If you discover a bug or have a feature request, please open an issue in our Git
|
||||
|
||||
- Provide a clear, descriptive title.
|
||||
- Include as many details as possible (steps to reproduce, screenshots, etc.).
|
||||
- **Tag Your Issue Appropriately:**
|
||||
- **Tag Your Issue Appropriately:**
|
||||
Use the following labels to help us categorize your issue:
|
||||
- **active:** Actively working on it right now.
|
||||
- **bug:** Something isn't working.
|
||||
@@ -82,12 +94,11 @@ If you discover a bug or have a feature request, please open an issue in our Git
|
||||
|
||||
Before creating a pull request:
|
||||
|
||||
- **Ensure Your Branch Is Up-to-Date:**
|
||||
- **Ensure Your Branch Is Up-to-Date:**
|
||||
Rebase your branch onto the latest `staging` branch to prevent merge conflicts.
|
||||
- **Follow the Guidelines:**
|
||||
- **Follow the Guidelines:**
|
||||
Make sure your changes are well-tested, follow our coding standards, and include relevant documentation if necessary.
|
||||
|
||||
- **Reference Issues:**
|
||||
- **Reference Issues:**
|
||||
If your PR addresses an existing issue, include `refs #<issue-number>` or `fixes #<issue-number>` in your PR description.
|
||||
|
||||
Our maintainers will review your pull request and provide feedback. We aim to make the review process as smooth and timely as possible.
|
||||
@@ -166,27 +177,27 @@ To use local models with Sim:
|
||||
|
||||
1. Install Ollama and pull models:
|
||||
|
||||
```bash
|
||||
# Install Ollama (if not already installed)
|
||||
curl -fsSL https://ollama.ai/install.sh | sh
|
||||
```bash
|
||||
# Install Ollama (if not already installed)
|
||||
curl -fsSL https://ollama.ai/install.sh | sh
|
||||
|
||||
# Pull a model (e.g., gemma3:4b)
|
||||
ollama pull gemma3:4b
|
||||
```
|
||||
# Pull a model (e.g., gemma3:4b)
|
||||
ollama pull gemma3:4b
|
||||
```
|
||||
|
||||
2. Start Sim with local model support:
|
||||
|
||||
```bash
|
||||
# With NVIDIA GPU support
|
||||
docker compose --profile local-gpu -f docker-compose.ollama.yml up -d
|
||||
```bash
|
||||
# With NVIDIA GPU support
|
||||
docker compose --profile local-gpu -f docker-compose.ollama.yml up -d
|
||||
|
||||
# Without GPU (CPU only)
|
||||
docker compose --profile local-cpu -f docker-compose.ollama.yml up -d
|
||||
# Without GPU (CPU only)
|
||||
docker compose --profile local-cpu -f docker-compose.ollama.yml up -d
|
||||
|
||||
# If hosting on a server, update the environment variables in the docker-compose.prod.yml file
|
||||
# to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434)
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
# If hosting on a server, update the environment variables in the docker-compose.prod.yml file
|
||||
# to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434)
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
### Option 3: Using VS Code / Cursor Dev Containers
|
||||
|
||||
@@ -201,61 +212,104 @@ Dev Containers provide a consistent and easy-to-use development environment:
|
||||
2. **Setup Steps:**
|
||||
|
||||
- Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/sim.git
|
||||
cd sim
|
||||
```
|
||||
- Open the project in VS Code/Cursor
|
||||
- When prompted, click "Reopen in Container" (or press F1 and select "Remote-Containers: Reopen in Container")
|
||||
- Wait for the container to build and initialize
|
||||
|
||||
- Open the project in VS Code/Cursor.
|
||||
- When prompted, click "Reopen in Container" (or press F1 and select "Remote-Containers: Reopen in Container").
|
||||
- Wait for the container to build and initialize.
|
||||
|
||||
3. **Start Developing:**
|
||||
|
||||
- Run `bun run dev:full` in the terminal or use the `sim-start` alias
|
||||
- This starts both the main application and the realtime socket server
|
||||
- All dependencies and configurations are automatically set up
|
||||
- Your changes will be automatically hot-reloaded
|
||||
- Run `bun run dev:full` in the terminal or use the `sim-start` alias.
|
||||
- This starts both the main application and the realtime socket server.
|
||||
- All dependencies and configurations are automatically set up.
|
||||
- Your changes will be automatically hot-reloaded.
|
||||
|
||||
4. **GitHub Codespaces:**
|
||||
- This setup also works with GitHub Codespaces if you prefer development in the browser
|
||||
- Just click "Code" → "Codespaces" → "Create codespace on staging"
|
||||
|
||||
- This setup also works with GitHub Codespaces if you prefer development in the browser.
|
||||
- Just click "Code" → "Codespaces" → "Create codespace on staging".
|
||||
|
||||
### Option 4: Manual Setup
|
||||
|
||||
If you prefer not to use Docker or Dev Containers:
|
||||
If you prefer not to use Docker or Dev Containers. **All commands run from the repository root unless explicitly noted.**
|
||||
|
||||
1. **Clone and Install:**
|
||||
|
||||
1. **Clone the Repository:**
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/sim.git
|
||||
cd sim
|
||||
bun install
|
||||
```
|
||||
|
||||
2. **Set Up Environment:**
|
||||
Bun workspaces handle dependency resolution for all apps and packages from the root `bun install`.
|
||||
|
||||
- Navigate to the app directory:
|
||||
```bash
|
||||
cd apps/sim
|
||||
```
|
||||
- Copy `.env.example` to `.env`
|
||||
- Configure required variables (DATABASE_URL, BETTER_AUTH_SECRET, BETTER_AUTH_URL)
|
||||
2. **Set Up Environment Files:**
|
||||
|
||||
3. **Set Up Database:**
|
||||
We use **per-app `.env` files** (the Turborepo-canonical pattern), not a single root `.env`. Three files are needed for local dev:
|
||||
|
||||
```bash
|
||||
bunx drizzle-kit push
|
||||
# Main app — large, app-specific (OAuth secrets, LLM keys, Stripe, etc.)
|
||||
cp apps/sim/.env.example apps/sim/.env
|
||||
|
||||
# Realtime server — small, only the values shared with the main app
|
||||
cp apps/realtime/.env.example apps/realtime/.env
|
||||
|
||||
# DB tooling (drizzle-kit, db:migrate)
|
||||
cp packages/db/.env.example packages/db/.env
|
||||
```
|
||||
|
||||
4. **Run the Development Server:**
|
||||
At minimum, each `.env` needs `DATABASE_URL`. `apps/sim/.env` and `apps/realtime/.env` additionally need matching values for `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `NEXT_PUBLIC_APP_URL`. `apps/sim/.env` also needs `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY`. Generate any 32-char secrets with `openssl rand -hex 32`.
|
||||
|
||||
The same `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `DATABASE_URL` must appear in both `apps/sim/.env` and `apps/realtime/.env` so the two services share auth and DB. After editing `apps/sim/.env`, you can mirror the shared subset into the realtime env in one shot:
|
||||
|
||||
```bash
|
||||
grep -E '^(DATABASE_URL|BETTER_AUTH_URL|BETTER_AUTH_SECRET|INTERNAL_API_SECRET|NEXT_PUBLIC_APP_URL|REDIS_URL)=' apps/sim/.env > apps/realtime/.env
|
||||
grep -E '^DATABASE_URL=' apps/sim/.env > packages/db/.env
|
||||
```
|
||||
|
||||
3. **Run Database Migrations:**
|
||||
|
||||
Migrations live in `packages/db/migrations/`. Run them via the dedicated workspace script:
|
||||
|
||||
```bash
|
||||
cd packages/db && bun run db:migrate && cd ../..
|
||||
```
|
||||
|
||||
For ad-hoc schema iteration during development you can also use `bun run db:push` from `packages/db`, but `db:migrate` is the canonical command for both local and CI/CD setups.
|
||||
|
||||
4. **Run the Development Servers:**
|
||||
|
||||
```bash
|
||||
bun run dev:full
|
||||
```
|
||||
|
||||
This command starts both the main application and the realtime socket server required for full functionality.
|
||||
This launches both apps with coloured prefixes:
|
||||
|
||||
- `[App]` — Next.js on `http://localhost:3000`
|
||||
- `[Realtime]` — Socket.IO on `http://localhost:3002`
|
||||
|
||||
Or run them separately:
|
||||
|
||||
```bash
|
||||
bun run dev # Next.js app only
|
||||
bun run dev:sockets # realtime server only
|
||||
```
|
||||
|
||||
5. **Make Your Changes and Test Locally.**
|
||||
|
||||
Before opening a PR, run the same checks CI runs:
|
||||
|
||||
```bash
|
||||
bun run type-check # TypeScript across every workspace
|
||||
bun run lint:check # Biome lint across every workspace
|
||||
bun run test # Vitest across every workspace
|
||||
```
|
||||
|
||||
### Email Template Development
|
||||
|
||||
When working on email templates, you can preview them using a local email preview server:
|
||||
@@ -263,18 +317,19 @@ When working on email templates, you can preview them using a local email previe
|
||||
1. **Run the Email Preview Server:**
|
||||
|
||||
```bash
|
||||
bun run email:dev
|
||||
cd apps/sim && bun run email:dev
|
||||
```
|
||||
|
||||
2. **Access the Preview:**
|
||||
|
||||
- Open `http://localhost:3000` in your browser
|
||||
- You'll see a list of all email templates
|
||||
- Click on any template to view and test it with various parameters
|
||||
- Open `http://localhost:3000` in your browser.
|
||||
- You'll see a list of all email templates.
|
||||
- Click on any template to view and test it with various parameters.
|
||||
|
||||
3. **Templates Location:**
|
||||
- Email templates are located in `sim/app/emails/`
|
||||
- After making changes to templates, they will automatically update in the preview
|
||||
|
||||
- Email templates live in `apps/sim/components/emails/`.
|
||||
- Changes hot-reload automatically in the preview.
|
||||
|
||||
---
|
||||
|
||||
@@ -282,28 +337,41 @@ When working on email templates, you can preview them using a local email previe
|
||||
|
||||
Sim is built in a modular fashion where blocks and tools extend the platform's functionality. To maintain consistency and quality, please follow the guidelines below when adding a new block or tool.
|
||||
|
||||
> **Use the skill guides for step-by-step recipes.** The repository ships opinionated, end-to-end guides under `.agents/skills/` that cover the exact file layout, conventions, registry wiring, and gotchas for each kind of contribution. Read the relevant SKILL.md before you start writing code:
|
||||
>
|
||||
> | Adding… | Read |
|
||||
> | ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
|
||||
> | A new integration end-to-end (tools + block + icon + optional triggers + all registrations) | [`.agents/skills/add-integration/SKILL.md`](../.agents/skills/add-integration/SKILL.md) |
|
||||
> | Just a block (or aligning an existing block with its tools) | [`.agents/skills/add-block/SKILL.md`](../.agents/skills/add-block/SKILL.md) |
|
||||
> | Just tool configs for a service | [`.agents/skills/add-tools/SKILL.md`](../.agents/skills/add-tools/SKILL.md) |
|
||||
> | A webhook trigger for a service | [`.agents/skills/add-trigger/SKILL.md`](../.agents/skills/add-trigger/SKILL.md) |
|
||||
> | A knowledge-base connector (sync docs from an external source) | [`.agents/skills/add-connector/SKILL.md`](../.agents/skills/add-connector/SKILL.md) |
|
||||
>
|
||||
> The shorter overview below is a high-level reference; the SKILL.md files are the authoritative source of truth and stay in sync with the codebase.
|
||||
|
||||
### Where to Add Your Code
|
||||
|
||||
- **Blocks:** Create your new block file under the `/apps/sim/blocks/blocks` directory. The name of the file should match the provider name (e.g., `pinecone.ts`).
|
||||
- **Tools:** Create a new directory under `/apps/sim/tools` with the same name as the provider (e.g., `/apps/sim/tools/pinecone`).
|
||||
- **Blocks:** Create your new block file under the `apps/sim/blocks/blocks/` directory. The name of the file should match the provider name (e.g., `pinecone.ts`).
|
||||
- **Tools:** Create a new directory under `apps/sim/tools/` with the same name as the provider (e.g., `apps/sim/tools/pinecone`).
|
||||
|
||||
In addition, you will need to update the registries:
|
||||
|
||||
- **Block Registry:** Update the blocks index (`/apps/sim/blocks/index.ts`) to include your new block.
|
||||
- **Tool Registry:** Update the tools registry (`/apps/sim/tools/index.ts`) to add your new tool.
|
||||
- **Block Registry:** Add your block to `apps/sim/blocks/registry.ts`. (`apps/sim/blocks/index.ts` re-exports lookups from the registry; you do not need to edit it.)
|
||||
- **Tool Registry:** Add your tool to `apps/sim/tools/index.ts`.
|
||||
|
||||
### How to Create a New Block
|
||||
|
||||
1. **Create a New File:**
|
||||
Create a file for your block named after the provider (e.g., `pinecone.ts`) in the `/apps/sim/blocks/blocks` directory.
|
||||
1. **Create a New File:**
|
||||
Create a file for your block named after the provider (e.g., `pinecone.ts`) in the `apps/sim/blocks/blocks/` directory.
|
||||
|
||||
2. **Create a New Icon:**
|
||||
Create a new icon for your block in the `/apps/sim/components/icons.tsx` file. The icon should follow the same naming convention as the block (e.g., `PineconeIcon`).
|
||||
Create a new icon for your block in `apps/sim/components/icons.tsx`. The icon should follow the same naming convention as the block (e.g., `PineconeIcon`).
|
||||
|
||||
3. **Define the Block Configuration:**
|
||||
3. **Define the Block Configuration:**
|
||||
Your block should export a constant of type `BlockConfig`. For example:
|
||||
|
||||
```typescript:/apps/sim/blocks/blocks/pinecone.ts
|
||||
```typescript
|
||||
// apps/sim/blocks/blocks/pinecone.ts
|
||||
import { PineconeIcon } from '@/components/icons'
|
||||
import type { BlockConfig } from '@/blocks/types'
|
||||
import type { PineconeResponse } from '@/tools/pinecone/types'
|
||||
@@ -321,7 +389,7 @@ In addition, you will need to update the registries:
|
||||
{
|
||||
id: 'operation',
|
||||
title: 'Operation',
|
||||
type: 'dropdown'
|
||||
type: 'dropdown',
|
||||
required: true,
|
||||
options: [
|
||||
{ label: 'Generate Embeddings', id: 'generate' },
|
||||
@@ -332,7 +400,7 @@ In addition, you will need to update the registries:
|
||||
{
|
||||
id: 'apiKey',
|
||||
title: 'API Key',
|
||||
type: 'short-input'
|
||||
type: 'short-input',
|
||||
placeholder: 'Your Pinecone API key',
|
||||
password: true,
|
||||
required: true,
|
||||
@@ -370,10 +438,11 @@ In addition, you will need to update the registries:
|
||||
}
|
||||
```
|
||||
|
||||
4. **Register Your Block:**
|
||||
Add your block to the blocks registry (`/apps/sim/blocks/registry.ts`):
|
||||
4. **Register Your Block:**
|
||||
Add your block to the blocks registry (`apps/sim/blocks/registry.ts`):
|
||||
|
||||
```typescript:/apps/sim/blocks/registry.ts
|
||||
```typescript
|
||||
// apps/sim/blocks/registry.ts
|
||||
import { PineconeBlock } from '@/blocks/blocks/pinecone'
|
||||
|
||||
// Registry of all available blocks
|
||||
@@ -385,24 +454,25 @@ In addition, you will need to update the registries:
|
||||
|
||||
The block will be automatically available to the application through the registry.
|
||||
|
||||
5. **Test Your Block:**
|
||||
5. **Test Your Block:**
|
||||
Ensure that the block displays correctly in the UI and that its functionality works as expected.
|
||||
|
||||
### How to Create a New Tool
|
||||
|
||||
1. **Create a New Directory:**
|
||||
Create a directory under `/apps/sim/tools` with the same name as the provider (e.g., `/apps/sim/tools/pinecone`).
|
||||
1. **Create a New Directory:**
|
||||
Create a directory under `apps/sim/tools/` with the same name as the provider (e.g., `apps/sim/tools/pinecone`).
|
||||
|
||||
2. **Create Tool Files:**
|
||||
2. **Create Tool Files:**
|
||||
Create separate files for each tool functionality with descriptive names (e.g., `fetch.ts`, `generate_embeddings.ts`, `search_text.ts`) in your tool directory.
|
||||
|
||||
3. **Create a Types File:**
|
||||
3. **Create a Types File:**
|
||||
Create a `types.ts` file in your tool directory to define and export all types related to your tools.
|
||||
|
||||
4. **Create an Index File:**
|
||||
4. **Create an Index File:**
|
||||
Create an `index.ts` file in your tool directory that imports and exports all tools:
|
||||
|
||||
```typescript:/apps/sim/tools/pinecone/index.ts
|
||||
```typescript
|
||||
// apps/sim/tools/pinecone/index.ts
|
||||
import { fetchTool } from './fetch'
|
||||
import { generateEmbeddingsTool } from './generate_embeddings'
|
||||
import { searchTextTool } from './search_text'
|
||||
@@ -410,10 +480,11 @@ In addition, you will need to update the registries:
|
||||
export { fetchTool, generateEmbeddingsTool, searchTextTool }
|
||||
```
|
||||
|
||||
5. **Define the Tool Configuration:**
|
||||
5. **Define the Tool Configuration:**
|
||||
Your tool should export a constant with a naming convention of `{toolName}Tool`. The tool ID should follow the format `{provider}_{tool_name}`. For example:
|
||||
|
||||
```typescript:/apps/sim/tools/pinecone/fetch.ts
|
||||
```typescript
|
||||
// apps/sim/tools/pinecone/fetch.ts
|
||||
import { ToolConfig, ToolResponse } from '@/tools/types'
|
||||
import { PineconeParams, PineconeResponse } from '@/tools/pinecone/types'
|
||||
|
||||
@@ -449,11 +520,12 @@ In addition, you will need to update the registries:
|
||||
}
|
||||
```
|
||||
|
||||
6. **Register Your Tool:**
|
||||
Update the tools registry in `/apps/sim/tools/index.ts` to include your new tool:
|
||||
6. **Register Your Tool:**
|
||||
Update the tools registry in `apps/sim/tools/index.ts` to include your new tool:
|
||||
|
||||
```typescript:/apps/sim/tools/index.ts
|
||||
import { fetchTool, generateEmbeddingsTool, searchTextTool } from '/@tools/pinecone'
|
||||
```typescript
|
||||
// apps/sim/tools/index.ts
|
||||
import { fetchTool, generateEmbeddingsTool, searchTextTool } from '@/tools/pinecone'
|
||||
// ... other imports
|
||||
|
||||
export const tools: Record<string, ToolConfig> = {
|
||||
@@ -464,13 +536,14 @@ In addition, you will need to update the registries:
|
||||
}
|
||||
```
|
||||
|
||||
7. **Test Your Tool:**
|
||||
7. **Test Your Tool:**
|
||||
Ensure that your tool functions correctly by making test requests and verifying the responses.
|
||||
|
||||
8. **Generate Documentation:**
|
||||
Run the documentation generator to create docs for your new tool:
|
||||
8. **Generate Documentation:**
|
||||
Run the documentation generator (from `apps/sim`) to create docs for your new tool:
|
||||
|
||||
```bash
|
||||
./scripts/generate-docs.sh
|
||||
cd apps/sim && bun run generate-docs
|
||||
```
|
||||
|
||||
### Naming Conventions
|
||||
@@ -480,7 +553,7 @@ Maintaining consistent naming across the codebase is critical for auto-generatio
|
||||
- **Block Files:** Name should match the provider (e.g., `pinecone.ts`)
|
||||
- **Block Export:** Should be named `{Provider}Block` (e.g., `PineconeBlock`)
|
||||
- **Icons:** Should be named `{Provider}Icon` (e.g., `PineconeIcon`)
|
||||
- **Tool Directories:** Should match the provider name (e.g., `/tools/pinecone/`)
|
||||
- **Tool Directories:** Should match the provider name (e.g., `tools/pinecone/`)
|
||||
- **Tool Files:** Should be named after their function (e.g., `fetch.ts`, `search_text.ts`)
|
||||
- **Tool Exports:** Should be named `{toolName}Tool` (e.g., `fetchTool`)
|
||||
- **Tool IDs:** Should follow the format `{provider}_{tool_name}` (e.g., `pinecone_fetch`)
|
||||
@@ -489,12 +562,12 @@ Maintaining consistent naming across the codebase is critical for auto-generatio
|
||||
|
||||
Sim implements a sophisticated parameter visibility system that controls how parameters are exposed to users and LLMs in agent workflows. Each parameter can have one of four visibility levels:
|
||||
|
||||
| Visibility | User Sees | LLM Sees | How It Gets Set |
|
||||
|-------------|-----------|----------|--------------------------------|
|
||||
| `user-only` | ✅ Yes | ❌ No | User provides in UI |
|
||||
| `user-or-llm` | ✅ Yes | ✅ Yes | User provides OR LLM generates |
|
||||
| `llm-only` | ❌ No | ✅ Yes | LLM generates only |
|
||||
| `hidden` | ❌ No | ❌ No | Application injects at runtime |
|
||||
| Visibility | User Sees | LLM Sees | How It Gets Set |
|
||||
| ------------- | --------- | -------- | ------------------------------ |
|
||||
| `user-only` | ✅ Yes | ❌ No | User provides in UI |
|
||||
| `user-or-llm` | ✅ Yes | ✅ Yes | User provides OR LLM generates |
|
||||
| `llm-only` | ❌ No | ✅ Yes | LLM generates only |
|
||||
| `hidden` | ❌ No | ❌ No | Application injects at runtime |
|
||||
|
||||
#### Visibility Guidelines
|
||||
|
||||
|
||||
@@ -10,13 +10,24 @@ import {
|
||||
MothershipStreamV1ToolOutcome,
|
||||
MothershipStreamV1ToolPhase,
|
||||
} from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
|
||||
vi.mock('@/lib/copilot/request/session', async () => {
|
||||
const actual = await vi.importActual<typeof import('@/lib/copilot/request/session')>(
|
||||
'@/lib/copilot/request/session'
|
||||
)
|
||||
return {
|
||||
...actual,
|
||||
hasAbortMarker: vi.fn().mockResolvedValue(false),
|
||||
}
|
||||
})
|
||||
|
||||
import {
|
||||
buildPreviewContentUpdate,
|
||||
decodeJsonStringPrefix,
|
||||
extractEditContent,
|
||||
runStreamLoop,
|
||||
} from '@/lib/copilot/request/go/stream'
|
||||
import { createEvent } from '@/lib/copilot/request/session'
|
||||
import { AbortReason, createEvent, hasAbortMarker } from '@/lib/copilot/request/session'
|
||||
import { RequestTraceV1Outcome, TraceCollector } from '@/lib/copilot/request/trace'
|
||||
import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types'
|
||||
|
||||
@@ -285,6 +296,137 @@ describe('copilot go stream helpers', () => {
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('reclassifies as aborted when the body closes without terminal but the abort marker is set', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
|
||||
expect(hasAbortMarker).toHaveBeenCalledWith(context.messageId)
|
||||
expect(context.wasAborted).toBe(true)
|
||||
expect(
|
||||
context.errors.some((message) =>
|
||||
message.includes('Copilot backend stream ended before a terminal event')
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('invokes onAbortObserved with MarkerObservedAtBodyClose when reclassifying via the abort marker', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(true)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
const onAbortObserved = vi.fn()
|
||||
|
||||
await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
onAbortObserved,
|
||||
})
|
||||
|
||||
expect(onAbortObserved).toHaveBeenCalledTimes(1)
|
||||
expect(onAbortObserved).toHaveBeenCalledWith(AbortReason.MarkerObservedAtBodyClose)
|
||||
expect(context.wasAborted).toBe(true)
|
||||
})
|
||||
|
||||
it('does not invoke onAbortObserved when no abort marker is present at body close', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockResolvedValueOnce(false)
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
const onAbortObserved = vi.fn()
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
onAbortObserved,
|
||||
})
|
||||
).rejects.toThrow('Copilot backend stream ended before a terminal event')
|
||||
|
||||
expect(onAbortObserved).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('still fails closed when the body closes without terminal and the abort marker check throws', async () => {
|
||||
const textEvent = createEvent({
|
||||
streamId: 'stream-1',
|
||||
cursor: '1',
|
||||
seq: 1,
|
||||
requestId: 'req-1',
|
||||
type: MothershipStreamV1EventType.text,
|
||||
payload: {
|
||||
channel: 'assistant',
|
||||
text: 'partial response',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent]))
|
||||
vi.mocked(hasAbortMarker).mockRejectedValueOnce(new Error('redis unavailable'))
|
||||
|
||||
const context = createStreamingContext()
|
||||
const execContext: ExecutionContext = {
|
||||
userId: 'user-1',
|
||||
workflowId: 'workflow-1',
|
||||
}
|
||||
|
||||
await expect(
|
||||
runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, {
|
||||
timeout: 1000,
|
||||
})
|
||||
).rejects.toThrow('Copilot backend stream ended before a terminal event')
|
||||
expect(context.wasAborted).toBe(false)
|
||||
})
|
||||
|
||||
it('fails closed when the shared stream receives an invalid event', async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce(
|
||||
createSseResponse([
|
||||
|
||||
@@ -30,7 +30,9 @@ import {
|
||||
} from '@/lib/copilot/request/handlers/types'
|
||||
import { getCopilotTracer } from '@/lib/copilot/request/otel'
|
||||
import {
|
||||
AbortReason,
|
||||
eventToStreamEvent,
|
||||
hasAbortMarker,
|
||||
isSubagentSpanStreamEvent,
|
||||
parsePersistedStreamEventEnvelope,
|
||||
} from '@/lib/copilot/request/session'
|
||||
@@ -436,16 +438,32 @@ export async function runStreamLoop(
|
||||
})
|
||||
|
||||
if (!context.streamComplete && !abortSignal?.aborted && !context.wasAborted) {
|
||||
const streamPath = new URL(fetchUrl).pathname
|
||||
const message = `Copilot backend stream ended before a terminal event on ${streamPath}`
|
||||
context.errors.push(message)
|
||||
logger.error('Copilot backend stream ended before a terminal event', {
|
||||
path: streamPath,
|
||||
requestId: context.requestId,
|
||||
messageId: context.messageId,
|
||||
})
|
||||
endedOn = CopilotSseCloseReason.ClosedNoTerminal
|
||||
throw new CopilotBackendError(message, { status: 503 })
|
||||
let abortRequested = false
|
||||
try {
|
||||
abortRequested = await hasAbortMarker(context.messageId)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read abort marker at body close', {
|
||||
streamId: context.messageId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
|
||||
if (abortRequested) {
|
||||
options.onAbortObserved?.(AbortReason.MarkerObservedAtBodyClose)
|
||||
context.wasAborted = true
|
||||
endedOn = CopilotSseCloseReason.Aborted
|
||||
} else {
|
||||
const streamPath = new URL(fetchUrl).pathname
|
||||
const message = `Copilot backend stream ended before a terminal event on ${streamPath}`
|
||||
context.errors.push(message)
|
||||
logger.error('Copilot backend stream ended before a terminal event', {
|
||||
path: streamPath,
|
||||
requestId: context.requestId,
|
||||
messageId: context.messageId,
|
||||
})
|
||||
endedOn = CopilotSseCloseReason.ClosedNoTerminal
|
||||
throw new CopilotBackendError(message, { status: 503 })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof FatalSseEventError && !context.errors.includes(error.message)) {
|
||||
|
||||
@@ -53,10 +53,10 @@ export async function runHeadlessCopilotLifecycle(
|
||||
simRequestId,
|
||||
otelContext,
|
||||
})
|
||||
outcome = options.abortSignal?.aborted
|
||||
? RequestTraceV1Outcome.cancelled
|
||||
: result.success
|
||||
? RequestTraceV1Outcome.success
|
||||
outcome = result.success
|
||||
? RequestTraceV1Outcome.success
|
||||
: options.abortSignal?.aborted || result.cancelled
|
||||
? RequestTraceV1Outcome.cancelled
|
||||
: RequestTraceV1Outcome.error
|
||||
return result
|
||||
} catch (error) {
|
||||
|
||||
@@ -6,7 +6,10 @@ import { propagation, trace } from '@opentelemetry/api'
|
||||
import { W3CTraceContextPropagator } from '@opentelemetry/core'
|
||||
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
import {
|
||||
MothershipStreamV1CompletionStatus,
|
||||
MothershipStreamV1EventType,
|
||||
} from '@/lib/copilot/generated/mothership-stream-v1'
|
||||
|
||||
const {
|
||||
runCopilotLifecycle,
|
||||
@@ -60,6 +63,7 @@ vi.mock('@/lib/copilot/request/session', () => ({
|
||||
registerActiveStream: vi.fn(),
|
||||
unregisterActiveStream: vi.fn(),
|
||||
startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)),
|
||||
isExplicitStopReason: vi.fn().mockReturnValue(false),
|
||||
SSE_RESPONSE_HEADERS: {},
|
||||
StreamWriter: vi.fn().mockImplementation(() => ({
|
||||
attach: vi.fn().mockImplementation((ctrl: ReadableStreamDefaultController) => {
|
||||
@@ -211,6 +215,46 @@ describe('createSSEStream terminal error handling', () => {
|
||||
expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1')
|
||||
})
|
||||
|
||||
it('publishes a cancelled completion (not an error) when the orchestrator reports cancelled without abortSignal aborted', async () => {
|
||||
runCopilotLifecycle.mockResolvedValue({
|
||||
success: false,
|
||||
cancelled: true,
|
||||
content: '',
|
||||
contentBlocks: [],
|
||||
toolCalls: [],
|
||||
})
|
||||
|
||||
const stream = createSSEStream({
|
||||
requestPayload: { message: 'hello' },
|
||||
userId: 'user-1',
|
||||
streamId: 'stream-1',
|
||||
executionId: 'exec-1',
|
||||
runId: 'run-1',
|
||||
currentChat: null,
|
||||
isNewChat: false,
|
||||
message: 'hello',
|
||||
titleModel: 'gpt-5.4',
|
||||
requestId: 'req-cancelled',
|
||||
orchestrateOptions: {},
|
||||
})
|
||||
|
||||
await drainStream(stream)
|
||||
|
||||
expect(appendEvent).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: MothershipStreamV1EventType.error,
|
||||
})
|
||||
)
|
||||
expect(appendEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: MothershipStreamV1EventType.complete,
|
||||
payload: expect.objectContaining({
|
||||
status: MothershipStreamV1CompletionStatus.cancelled,
|
||||
}),
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('passes an OTel context into the streaming lifecycle', async () => {
|
||||
let lifecycleTraceparent = ''
|
||||
runCopilotLifecycle.mockImplementation(async (_payload, options) => {
|
||||
|
||||
@@ -249,6 +249,11 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS
|
||||
onEvent: async (event) => {
|
||||
await publisher.publish(event)
|
||||
},
|
||||
onAbortObserved: (reason) => {
|
||||
if (!abortController.signal.aborted) {
|
||||
abortController.abort(reason)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
lifecycleResult = result
|
||||
@@ -266,7 +271,7 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS
|
||||
// 3. Otherwise → error.
|
||||
outcome = result.success
|
||||
? RequestTraceV1Outcome.success
|
||||
: abortController.signal.aborted || publisher.clientDisconnected
|
||||
: result.cancelled || abortController.signal.aborted || publisher.clientDisconnected
|
||||
? RequestTraceV1Outcome.cancelled
|
||||
: RequestTraceV1Outcome.error
|
||||
if (outcome === RequestTraceV1Outcome.cancelled) {
|
||||
|
||||
@@ -22,6 +22,12 @@ export const AbortReason = {
|
||||
* that the node that DID receive it wrote, and aborts on the poll.
|
||||
*/
|
||||
RedisPoller: 'redis_abort_marker:poller',
|
||||
/**
|
||||
* Cross-process stop: same root cause as `RedisPoller`, but observed
|
||||
* by `runStreamLoop` at body close (the Go body ended before the
|
||||
* 250ms poller's next tick) rather than by the polling timer.
|
||||
*/
|
||||
MarkerObservedAtBodyClose: 'redis_abort_marker:body_close',
|
||||
/** Internal timeout on the outbound explicit-abort fetch to Go. */
|
||||
ExplicitAbortFetchTimeout: 'timeout:go_explicit_abort_fetch',
|
||||
} as const
|
||||
@@ -38,5 +44,9 @@ export type AbortReasonValue = (typeof AbortReason)[keyof typeof AbortReason]
|
||||
* stops, mirroring `requestctx.IsExplicitUserStop` on the Go side.
|
||||
*/
|
||||
export function isExplicitStopReason(reason: unknown): boolean {
|
||||
return reason === AbortReason.UserStop || reason === AbortReason.RedisPoller
|
||||
return (
|
||||
reason === AbortReason.UserStop ||
|
||||
reason === AbortReason.RedisPoller ||
|
||||
reason === AbortReason.MarkerObservedAtBodyClose
|
||||
)
|
||||
}
|
||||
|
||||
@@ -98,6 +98,47 @@ describe('startAbortPoller heartbeat', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('aborts the controller before clearing the marker so the marker is never observable as cleared while the signal is still unaborted', async () => {
|
||||
const controller = new AbortController()
|
||||
const streamId = 'stream-order-1'
|
||||
|
||||
let signalAbortedWhenMarkerCleared: boolean | null = null
|
||||
mockClearAbortMarker.mockImplementationOnce(async () => {
|
||||
signalAbortedWhenMarkerCleared = controller.signal.aborted
|
||||
})
|
||||
mockHasAbortMarker.mockResolvedValueOnce(true)
|
||||
|
||||
const interval = startAbortPoller(streamId, controller, {})
|
||||
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(mockClearAbortMarker).toHaveBeenCalledWith(streamId)
|
||||
expect(signalAbortedWhenMarkerCleared).toBe(true)
|
||||
expect(controller.signal.aborted).toBe(true)
|
||||
} finally {
|
||||
clearInterval(interval)
|
||||
}
|
||||
})
|
||||
|
||||
it('does not clear the marker when the signal is already aborted (no double abort)', async () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort('preexisting')
|
||||
const streamId = 'stream-order-2'
|
||||
|
||||
mockHasAbortMarker.mockResolvedValueOnce(true)
|
||||
|
||||
const interval = startAbortPoller(streamId, controller, {})
|
||||
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(mockClearAbortMarker).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
clearInterval(interval)
|
||||
}
|
||||
})
|
||||
|
||||
it('stops heartbeating after ownership is lost', async () => {
|
||||
const controller = new AbortController()
|
||||
const streamId = 'stream-lost'
|
||||
|
||||
@@ -17,7 +17,7 @@ const pendingChatStreams = new Map<
|
||||
{ promise: Promise<void>; resolve: () => void; streamId: string }
|
||||
>()
|
||||
|
||||
const DEFAULT_ABORT_POLL_MS = 1000
|
||||
const DEFAULT_ABORT_POLL_MS = 250
|
||||
|
||||
/**
|
||||
* TTL for the per-chat stream lock. Kept short so that if the Sim pod
|
||||
|
||||
@@ -136,6 +136,14 @@ export interface OrchestratorOptions {
|
||||
onComplete?: (result: OrchestratorResult) => void | Promise<void>
|
||||
onError?: (error: Error) => void | Promise<void>
|
||||
abortSignal?: AbortSignal
|
||||
/**
|
||||
* Invoked when the orchestrator infers that the run was aborted via
|
||||
* an out-of-band signal (currently: a Redis abort marker observed
|
||||
* at SSE body close). Callers wire this to fire their local
|
||||
* `AbortController` so `signal.reason` is set and `recordCancelled`
|
||||
* classifies as `explicit_stop` rather than `unknown`.
|
||||
*/
|
||||
onAbortObserved?: (reason: string) => void
|
||||
interactive?: boolean
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user