fix poetry lock

- Added Langfuse configuration to settings and environment variables.
- Introduced Langfuse client for fetching prompts in chat service.
- Updated ChatConfig to include Langfuse prompt name.
- Enhanced service logic to retrieve prompts from Langfuse, improving prompt management capabilities.

This integration allows for rapid runtime prompt updates and eventual analytics of the performance of the Co-Pilot system.

This commit aims to trial this service as a potential option.

.branchlet.json

@@ -1,37 +0,0 @@
-{
-  "worktreeCopyPatterns": [
-    ".env*",
-    ".vscode/**",
-    ".auth/**",
-    ".claude/**",
-    "autogpt_platform/.env*",
-    "autogpt_platform/backend/.env*",
-    "autogpt_platform/frontend/.env*",
-    "autogpt_platform/frontend/.auth/**",
-    "autogpt_platform/db/docker/.env*"
-  ],
-  "worktreeCopyIgnores": [
-    "**/node_modules/**",
-    "**/dist/**",
-    "**/.git/**",
-    "**/Thumbs.db",
-    "**/.DS_Store",
-    "**/.next/**",
-    "**/__pycache__/**",
-    "**/.ruff_cache/**",
-    "**/.pytest_cache/**",
-    "**/*.pyc",
-    "**/playwright-report/**",
-    "**/logs/**",
-    "**/site/**"
-  ],
-  "worktreePathTemplate": "$BASE_PATH.worktree",
-  "postCreateCmd": [
-    "cd autogpt_platform/autogpt_libs && poetry install",
-    "cd autogpt_platform/backend && poetry install && poetry run prisma generate",
-    "cd autogpt_platform/frontend && pnpm install",
-    "cd docs && pip install -r requirements.txt"
-  ],
-  "terminalCommand": "code .",
-  "deleteBranchWithWorktree": false
-}

.claude/skills/vercel-react-best-practices/SKILL.md

@@ -1,125 +0,0 @@
----
-name: vercel-react-best-practices
-description: React and Next.js performance optimization guidelines from Vercel Engineering. This skill should be used when writing, reviewing, or refactoring React/Next.js code to ensure optimal performance patterns. Triggers on tasks involving React components, Next.js pages, data fetching, bundle optimization, or performance improvements.
-license: MIT
-metadata:
-  author: vercel
-  version: "1.0.0"
----
-
-# Vercel React Best Practices
-
-Comprehensive performance optimization guide for React and Next.js applications, maintained by Vercel. Contains 45 rules across 8 categories, prioritized by impact to guide automated refactoring and code generation.
-
-## When to Apply
-
-Reference these guidelines when:
-- Writing new React components or Next.js pages
-- Implementing data fetching (client or server-side)
-- Reviewing code for performance issues
-- Refactoring existing React/Next.js code
-- Optimizing bundle size or load times
-
-## Rule Categories by Priority
-
-| Priority | Category | Impact | Prefix |
-|----------|----------|--------|--------|
-| 1 | Eliminating Waterfalls | CRITICAL | `async-` |
-| 2 | Bundle Size Optimization | CRITICAL | `bundle-` |
-| 3 | Server-Side Performance | HIGH | `server-` |
-| 4 | Client-Side Data Fetching | MEDIUM-HIGH | `client-` |
-| 5 | Re-render Optimization | MEDIUM | `rerender-` |
-| 6 | Rendering Performance | MEDIUM | `rendering-` |
-| 7 | JavaScript Performance | LOW-MEDIUM | `js-` |
-| 8 | Advanced Patterns | LOW | `advanced-` |
-
-## Quick Reference
-
-### 1. Eliminating Waterfalls (CRITICAL)
-
-- `async-defer-await` - Move await into branches where actually used
-- `async-parallel` - Use Promise.all() for independent operations
-- `async-dependencies` - Use better-all for partial dependencies
-- `async-api-routes` - Start promises early, await late in API routes
-- `async-suspense-boundaries` - Use Suspense to stream content
-
-### 2. Bundle Size Optimization (CRITICAL)
-
-- `bundle-barrel-imports` - Import directly, avoid barrel files
-- `bundle-dynamic-imports` - Use next/dynamic for heavy components
-- `bundle-defer-third-party` - Load analytics/logging after hydration
-- `bundle-conditional` - Load modules only when feature is activated
-- `bundle-preload` - Preload on hover/focus for perceived speed
-
-### 3. Server-Side Performance (HIGH)
-
-- `server-cache-react` - Use React.cache() for per-request deduplication
-- `server-cache-lru` - Use LRU cache for cross-request caching
-- `server-serialization` - Minimize data passed to client components
-- `server-parallel-fetching` - Restructure components to parallelize fetches
-- `server-after-nonblocking` - Use after() for non-blocking operations
-
-### 4. Client-Side Data Fetching (MEDIUM-HIGH)
-
-- `client-swr-dedup` - Use SWR for automatic request deduplication
-- `client-event-listeners` - Deduplicate global event listeners
-
-### 5. Re-render Optimization (MEDIUM)
-
-- `rerender-defer-reads` - Don't subscribe to state only used in callbacks
-- `rerender-memo` - Extract expensive work into memoized components
-- `rerender-dependencies` - Use primitive dependencies in effects
-- `rerender-derived-state` - Subscribe to derived booleans, not raw values
-- `rerender-functional-setstate` - Use functional setState for stable callbacks
-- `rerender-lazy-state-init` - Pass function to useState for expensive values
-- `rerender-transitions` - Use startTransition for non-urgent updates
-
-### 6. Rendering Performance (MEDIUM)
-
-- `rendering-animate-svg-wrapper` - Animate div wrapper, not SVG element
-- `rendering-content-visibility` - Use content-visibility for long lists
-- `rendering-hoist-jsx` - Extract static JSX outside components
-- `rendering-svg-precision` - Reduce SVG coordinate precision
-- `rendering-hydration-no-flicker` - Use inline script for client-only data
-- `rendering-activity` - Use Activity component for show/hide
-- `rendering-conditional-render` - Use ternary, not && for conditionals
-
-### 7. JavaScript Performance (LOW-MEDIUM)
-
-- `js-batch-dom-css` - Group CSS changes via classes or cssText
-- `js-index-maps` - Build Map for repeated lookups
-- `js-cache-property-access` - Cache object properties in loops
-- `js-cache-function-results` - Cache function results in module-level Map
-- `js-cache-storage` - Cache localStorage/sessionStorage reads
-- `js-combine-iterations` - Combine multiple filter/map into one loop
-- `js-length-check-first` - Check array length before expensive comparison
-- `js-early-exit` - Return early from functions
-- `js-hoist-regexp` - Hoist RegExp creation outside loops
-- `js-min-max-loop` - Use loop for min/max instead of sort
-- `js-set-map-lookups` - Use Set/Map for O(1) lookups
-- `js-tosorted-immutable` - Use toSorted() for immutability
-
-### 8. Advanced Patterns (LOW)
-
-- `advanced-event-handler-refs` - Store event handlers in refs
-- `advanced-use-latest` - useLatest for stable callback refs
-
-## How to Use
-
-Read individual rule files for detailed explanations and code examples:
-
-```
-rules/async-parallel.md
-rules/bundle-barrel-imports.md
-rules/_sections.md
-```
-
-Each rule file contains:
-- Brief explanation of why it matters
-- Incorrect code example with explanation
-- Correct code example with explanation
-- Additional context and references
-
-## Full Compiled Document
-
-For the complete guide with all rules expanded: `AGENTS.md`

.claude/skills/vercel-react-best-practices/rules/advanced-event-handler-refs.md

@@ -1,55 +0,0 @@
----
-title: Store Event Handlers in Refs
-impact: LOW
-impactDescription: stable subscriptions
-tags: advanced, hooks, refs, event-handlers, optimization
----
-
-## Store Event Handlers in Refs
-
-Store callbacks in refs when used in effects that shouldn't re-subscribe on callback changes.
-
-**Incorrect (re-subscribes on every render):**
-
-```tsx
-function useWindowEvent(event: string, handler: () => void) {
-  useEffect(() => {
-    window.addEventListener(event, handler)
-    return () => window.removeEventListener(event, handler)
-  }, [event, handler])
-}
-```
-
-**Correct (stable subscription):**
-
-```tsx
-function useWindowEvent(event: string, handler: () => void) {
-  const handlerRef = useRef(handler)
-  useEffect(() => {
-    handlerRef.current = handler
-  }, [handler])
-
-  useEffect(() => {
-    const listener = () => handlerRef.current()
-    window.addEventListener(event, listener)
-    return () => window.removeEventListener(event, listener)
-  }, [event])
-}
-```
-
-**Alternative: use `useEffectEvent` if you're on latest React:**
-
-```tsx
-import { useEffectEvent } from 'react'
-
-function useWindowEvent(event: string, handler: () => void) {
-  const onEvent = useEffectEvent(handler)
-
-  useEffect(() => {
-    window.addEventListener(event, onEvent)
-    return () => window.removeEventListener(event, onEvent)
-  }, [event])
-}
-```
-
-`useEffectEvent` provides a cleaner API for the same pattern: it creates a stable function reference that always calls the latest version of the handler.

.claude/skills/vercel-react-best-practices/rules/advanced-use-latest.md

@@ -1,49 +0,0 @@
----
-title: useLatest for Stable Callback Refs
-impact: LOW
-impactDescription: prevents effect re-runs
-tags: advanced, hooks, useLatest, refs, optimization
----
-
-## useLatest for Stable Callback Refs
-
-Access latest values in callbacks without adding them to dependency arrays. Prevents effect re-runs while avoiding stale closures.
-
-**Implementation:**
-
-```typescript
-function useLatest<T>(value: T) {
-  const ref = useRef(value)
-  useEffect(() => {
-    ref.current = value
-  }, [value])
-  return ref
-}
-```
-
-**Incorrect (effect re-runs on every callback change):**
-
-```tsx
-function SearchInput({ onSearch }: { onSearch: (q: string) => void }) {
-  const [query, setQuery] = useState('')
-
-  useEffect(() => {
-    const timeout = setTimeout(() => onSearch(query), 300)
-    return () => clearTimeout(timeout)
-  }, [query, onSearch])
-}
-```
-
-**Correct (stable effect, fresh callback):**
-
-```tsx
-function SearchInput({ onSearch }: { onSearch: (q: string) => void }) {
-  const [query, setQuery] = useState('')
-  const onSearchRef = useLatest(onSearch)
-
-  useEffect(() => {
-    const timeout = setTimeout(() => onSearchRef.current(query), 300)
-    return () => clearTimeout(timeout)
-  }, [query])
-}
-```

.claude/skills/vercel-react-best-practices/rules/async-api-routes.md

@@ -1,38 +0,0 @@
----
-title: Prevent Waterfall Chains in API Routes
-impact: CRITICAL
-impactDescription: 2-10× improvement
-tags: api-routes, server-actions, waterfalls, parallelization
----
-
-## Prevent Waterfall Chains in API Routes
-
-In API routes and Server Actions, start independent operations immediately, even if you don't await them yet.
-
-**Incorrect (config waits for auth, data waits for both):**
-
-```typescript
-export async function GET(request: Request) {
-  const session = await auth()
-  const config = await fetchConfig()
-  const data = await fetchData(session.user.id)
-  return Response.json({ data, config })
-}
-```
-
-**Correct (auth and config start immediately):**
-
-```typescript
-export async function GET(request: Request) {
-  const sessionPromise = auth()
-  const configPromise = fetchConfig()
-  const session = await sessionPromise
-  const [config, data] = await Promise.all([
-    configPromise,
-    fetchData(session.user.id)
-  ])
-  return Response.json({ data, config })
-}
-```
-
-For operations with more complex dependency chains, use `better-all` to automatically maximize parallelism (see Dependency-Based Parallelization).

.claude/skills/vercel-react-best-practices/rules/async-defer-await.md

@@ -1,80 +0,0 @@
----
-title: Defer Await Until Needed
-impact: HIGH
-impactDescription: avoids blocking unused code paths
-tags: async, await, conditional, optimization
----
-
-## Defer Await Until Needed
-
-Move `await` operations into the branches where they're actually used to avoid blocking code paths that don't need them.
-
-**Incorrect (blocks both branches):**
-
-```typescript
-async function handleRequest(userId: string, skipProcessing: boolean) {
-  const userData = await fetchUserData(userId)
-  
-  if (skipProcessing) {
-    // Returns immediately but still waited for userData
-    return { skipped: true }
-  }
-  
-  // Only this branch uses userData
-  return processUserData(userData)
-}
-```
-
-**Correct (only blocks when needed):**
-
-```typescript
-async function handleRequest(userId: string, skipProcessing: boolean) {
-  if (skipProcessing) {
-    // Returns immediately without waiting
-    return { skipped: true }
-  }
-  
-  // Fetch only when needed
-  const userData = await fetchUserData(userId)
-  return processUserData(userData)
-}
-```
-
-**Another example (early return optimization):**
-
-```typescript
-// Incorrect: always fetches permissions
-async function updateResource(resourceId: string, userId: string) {
-  const permissions = await fetchPermissions(userId)
-  const resource = await getResource(resourceId)
-  
-  if (!resource) {
-    return { error: 'Not found' }
-  }
-  
-  if (!permissions.canEdit) {
-    return { error: 'Forbidden' }
-  }
-  
-  return await updateResourceData(resource, permissions)
-}
-
-// Correct: fetches only when needed
-async function updateResource(resourceId: string, userId: string) {
-  const resource = await getResource(resourceId)
-  
-  if (!resource) {
-    return { error: 'Not found' }
-  }
-  
-  const permissions = await fetchPermissions(userId)
-  
-  if (!permissions.canEdit) {
-    return { error: 'Forbidden' }
-  }
-  
-  return await updateResourceData(resource, permissions)
-}
-```
-
-This optimization is especially valuable when the skipped branch is frequently taken, or when the deferred operation is expensive.

.claude/skills/vercel-react-best-practices/rules/async-dependencies.md

@@ -1,36 +0,0 @@
----
-title: Dependency-Based Parallelization
-impact: CRITICAL
-impactDescription: 2-10× improvement
-tags: async, parallelization, dependencies, better-all
----
-
-## Dependency-Based Parallelization
-
-For operations with partial dependencies, use `better-all` to maximize parallelism. It automatically starts each task at the earliest possible moment.
-
-**Incorrect (profile waits for config unnecessarily):**
-
-```typescript
-const [user, config] = await Promise.all([
-  fetchUser(),
-  fetchConfig()
-])
-const profile = await fetchProfile(user.id)
-```
-
-**Correct (config and profile run in parallel):**
-
-```typescript
-import { all } from 'better-all'
-
-const { user, config, profile } = await all({
-  async user() { return fetchUser() },
-  async config() { return fetchConfig() },
-  async profile() {
-    return fetchProfile((await this.$.user).id)
-  }
-})
-```
-
-Reference: [https://github.com/shuding/better-all](https://github.com/shuding/better-all)

.claude/skills/vercel-react-best-practices/rules/async-parallel.md

@@ -1,28 +0,0 @@
----
-title: Promise.all() for Independent Operations
-impact: CRITICAL
-impactDescription: 2-10× improvement
-tags: async, parallelization, promises, waterfalls
----
-
-## Promise.all() for Independent Operations
-
-When async operations have no interdependencies, execute them concurrently using `Promise.all()`.
-
-**Incorrect (sequential execution, 3 round trips):**
-
-```typescript
-const user = await fetchUser()
-const posts = await fetchPosts()
-const comments = await fetchComments()
-```
-
-**Correct (parallel execution, 1 round trip):**
-
-```typescript
-const [user, posts, comments] = await Promise.all([
-  fetchUser(),
-  fetchPosts(),
-  fetchComments()
-])
-```

.claude/skills/vercel-react-best-practices/rules/async-suspense-boundaries.md

@@ -1,99 +0,0 @@
----
-title: Strategic Suspense Boundaries
-impact: HIGH
-impactDescription: faster initial paint
-tags: async, suspense, streaming, layout-shift
----
-
-## Strategic Suspense Boundaries
-
-Instead of awaiting data in async components before returning JSX, use Suspense boundaries to show the wrapper UI faster while data loads.
-
-**Incorrect (wrapper blocked by data fetching):**
-
-```tsx
-async function Page() {
-  const data = await fetchData() // Blocks entire page
-  
-  return (
-    <div>
-      <div>Sidebar</div>
-      <div>Header</div>
-      <div>
-        <DataDisplay data={data} />
-      </div>
-      <div>Footer</div>
-    </div>
-  )
-}
-```
-
-The entire layout waits for data even though only the middle section needs it.
-
-**Correct (wrapper shows immediately, data streams in):**
-
-```tsx
-function Page() {
-  return (
-    <div>
-      <div>Sidebar</div>
-      <div>Header</div>
-      <div>
-        <Suspense fallback={<Skeleton />}>
-          <DataDisplay />
-        </Suspense>
-      </div>
-      <div>Footer</div>
-    </div>
-  )
-}
-
-async function DataDisplay() {
-  const data = await fetchData() // Only blocks this component
-  return <div>{data.content}</div>
-}
-```
-
-Sidebar, Header, and Footer render immediately. Only DataDisplay waits for data.
-
-**Alternative (share promise across components):**
-
-```tsx
-function Page() {
-  // Start fetch immediately, but don't await
-  const dataPromise = fetchData()
-  
-  return (
-    <div>
-      <div>Sidebar</div>
-      <div>Header</div>
-      <Suspense fallback={<Skeleton />}>
-        <DataDisplay dataPromise={dataPromise} />
-        <DataSummary dataPromise={dataPromise} />
-      </Suspense>
-      <div>Footer</div>
-    </div>
-  )
-}
-
-function DataDisplay({ dataPromise }: { dataPromise: Promise<Data> }) {
-  const data = use(dataPromise) // Unwraps the promise
-  return <div>{data.content}</div>
-}
-
-function DataSummary({ dataPromise }: { dataPromise: Promise<Data> }) {
-  const data = use(dataPromise) // Reuses the same promise
-  return <div>{data.summary}</div>
-}
-```
-
-Both components share the same promise, so only one fetch occurs. Layout renders immediately while both components wait together.
-
-**When NOT to use this pattern:**
-
-- Critical data needed for layout decisions (affects positioning)
-- SEO-critical content above the fold
-- Small, fast queries where suspense overhead isn't worth it
-- When you want to avoid layout shift (loading → content jump)
-
-**Trade-off:** Faster initial paint vs potential layout shift. Choose based on your UX priorities.

.claude/skills/vercel-react-best-practices/rules/bundle-barrel-imports.md

@@ -1,59 +0,0 @@
----
-title: Avoid Barrel File Imports
-impact: CRITICAL
-impactDescription: 200-800ms import cost, slow builds
-tags: bundle, imports, tree-shaking, barrel-files, performance
----
-
-## Avoid Barrel File Imports
-
-Import directly from source files instead of barrel files to avoid loading thousands of unused modules. **Barrel files** are entry points that re-export multiple modules (e.g., `index.js` that does `export * from './module'`).
-
-Popular icon and component libraries can have **up to 10,000 re-exports** in their entry file. For many React packages, **it takes 200-800ms just to import them**, affecting both development speed and production cold starts.
-
-**Why tree-shaking doesn't help:** When a library is marked as external (not bundled), the bundler can't optimize it. If you bundle it to enable tree-shaking, builds become substantially slower analyzing the entire module graph.
-
-**Incorrect (imports entire library):**
-
-```tsx
-import { Check, X, Menu } from 'lucide-react'
-// Loads 1,583 modules, takes ~2.8s extra in dev
-// Runtime cost: 200-800ms on every cold start
-
-import { Button, TextField } from '@mui/material'
-// Loads 2,225 modules, takes ~4.2s extra in dev
-```
-
-**Correct (imports only what you need):**
-
-```tsx
-import Check from 'lucide-react/dist/esm/icons/check'
-import X from 'lucide-react/dist/esm/icons/x'
-import Menu from 'lucide-react/dist/esm/icons/menu'
-// Loads only 3 modules (~2KB vs ~1MB)
-
-import Button from '@mui/material/Button'
-import TextField from '@mui/material/TextField'
-// Loads only what you use
-```
-
-**Alternative (Next.js 13.5+):**
-
-```js
-// next.config.js - use optimizePackageImports
-module.exports = {
-  experimental: {
-    optimizePackageImports: ['lucide-react', '@mui/material']
-  }
-}
-
-// Then you can keep the ergonomic barrel imports:
-import { Check, X, Menu } from 'lucide-react'
-// Automatically transformed to direct imports at build time
-```
-
-Direct imports provide 15-70% faster dev boot, 28% faster builds, 40% faster cold starts, and significantly faster HMR.
-
-Libraries commonly affected: `lucide-react`, `@mui/material`, `@mui/icons-material`, `@tabler/icons-react`, `react-icons`, `@headlessui/react`, `@radix-ui/react-*`, `lodash`, `ramda`, `date-fns`, `rxjs`, `react-use`.
-
-Reference: [How we optimized package imports in Next.js](https://vercel.com/blog/how-we-optimized-package-imports-in-next-js)

.claude/skills/vercel-react-best-practices/rules/bundle-conditional.md

@@ -1,31 +0,0 @@
----
-title: Conditional Module Loading
-impact: HIGH
-impactDescription: loads large data only when needed
-tags: bundle, conditional-loading, lazy-loading
----
-
-## Conditional Module Loading
-
-Load large data or modules only when a feature is activated.
-
-**Example (lazy-load animation frames):**
-
-```tsx
-function AnimationPlayer({ enabled }: { enabled: boolean }) {
-  const [frames, setFrames] = useState<Frame[] | null>(null)
-
-  useEffect(() => {
-    if (enabled && !frames && typeof window !== 'undefined') {
-      import('./animation-frames.js')
-        .then(mod => setFrames(mod.frames))
-        .catch(() => setEnabled(false))
-    }
-  }, [enabled, frames])
-
-  if (!frames) return <Skeleton />
-  return <Canvas frames={frames} />
-}
-```
-
-The `typeof window !== 'undefined'` check prevents bundling this module for SSR, optimizing server bundle size and build speed.

.claude/skills/vercel-react-best-practices/rules/bundle-defer-third-party.md

@@ -1,49 +0,0 @@
----
-title: Defer Non-Critical Third-Party Libraries
-impact: MEDIUM
-impactDescription: loads after hydration
-tags: bundle, third-party, analytics, defer
----
-
-## Defer Non-Critical Third-Party Libraries
-
-Analytics, logging, and error tracking don't block user interaction. Load them after hydration.
-
-**Incorrect (blocks initial bundle):**
-
-```tsx
-import { Analytics } from '@vercel/analytics/react'
-
-export default function RootLayout({ children }) {
-  return (
-    <html>
-      <body>
-        {children}
-        <Analytics />
-      </body>
-    </html>
-  )
-}
-```
-
-**Correct (loads after hydration):**
-
-```tsx
-import dynamic from 'next/dynamic'
-
-const Analytics = dynamic(
-  () => import('@vercel/analytics/react').then(m => m.Analytics),
-  { ssr: false }
-)
-
-export default function RootLayout({ children }) {
-  return (
-    <html>
-      <body>
-        {children}
-        <Analytics />
-      </body>
-    </html>
-  )
-}
-```

.claude/skills/vercel-react-best-practices/rules/bundle-dynamic-imports.md

@@ -1,35 +0,0 @@
----
-title: Dynamic Imports for Heavy Components
-impact: CRITICAL
-impactDescription: directly affects TTI and LCP
-tags: bundle, dynamic-import, code-splitting, next-dynamic
----
-
-## Dynamic Imports for Heavy Components
-
-Use `next/dynamic` to lazy-load large components not needed on initial render.
-
-**Incorrect (Monaco bundles with main chunk ~300KB):**
-
-```tsx
-import { MonacoEditor } from './monaco-editor'
-
-function CodePanel({ code }: { code: string }) {
-  return <MonacoEditor value={code} />
-}
-```
-
-**Correct (Monaco loads on demand):**
-
-```tsx
-import dynamic from 'next/dynamic'
-
-const MonacoEditor = dynamic(
-  () => import('./monaco-editor').then(m => m.MonacoEditor),
-  { ssr: false }
-)
-
-function CodePanel({ code }: { code: string }) {
-  return <MonacoEditor value={code} />
-}
-```

.claude/skills/vercel-react-best-practices/rules/bundle-preload.md

@@ -1,50 +0,0 @@
----
-title: Preload Based on User Intent
-impact: MEDIUM
-impactDescription: reduces perceived latency
-tags: bundle, preload, user-intent, hover
----
-
-## Preload Based on User Intent
-
-Preload heavy bundles before they're needed to reduce perceived latency.
-
-**Example (preload on hover/focus):**
-
-```tsx
-function EditorButton({ onClick }: { onClick: () => void }) {
-  const preload = () => {
-    if (typeof window !== 'undefined') {
-      void import('./monaco-editor')
-    }
-  }
-
-  return (
-    <button
-      onMouseEnter={preload}
-      onFocus={preload}
-      onClick={onClick}
-    >
-      Open Editor
-    </button>
-  )
-}
-```
-
-**Example (preload when feature flag is enabled):**
-
-```tsx
-function FlagsProvider({ children, flags }: Props) {
-  useEffect(() => {
-    if (flags.editorEnabled && typeof window !== 'undefined') {
-      void import('./monaco-editor').then(mod => mod.init())
-    }
-  }, [flags.editorEnabled])
-
-  return <FlagsContext.Provider value={flags}>
-    {children}
-  </FlagsContext.Provider>
-}
-```
-
-The `typeof window !== 'undefined'` check prevents bundling preloaded modules for SSR, optimizing server bundle size and build speed.

.claude/skills/vercel-react-best-practices/rules/client-event-listeners.md

@@ -1,74 +0,0 @@
----
-title: Deduplicate Global Event Listeners
-impact: LOW
-impactDescription: single listener for N components
-tags: client, swr, event-listeners, subscription
----
-
-## Deduplicate Global Event Listeners
-
-Use `useSWRSubscription()` to share global event listeners across component instances.
-
-**Incorrect (N instances = N listeners):**
-
-```tsx
-function useKeyboardShortcut(key: string, callback: () => void) {
-  useEffect(() => {
-    const handler = (e: KeyboardEvent) => {
-      if (e.metaKey && e.key === key) {
-        callback()
-      }
-    }
-    window.addEventListener('keydown', handler)
-    return () => window.removeEventListener('keydown', handler)
-  }, [key, callback])
-}
-```
-
-When using the `useKeyboardShortcut` hook multiple times, each instance will register a new listener.
-
-**Correct (N instances = 1 listener):**
-
-```tsx
-import useSWRSubscription from 'swr/subscription'
-
-// Module-level Map to track callbacks per key
-const keyCallbacks = new Map<string, Set<() => void>>()
-
-function useKeyboardShortcut(key: string, callback: () => void) {
-  // Register this callback in the Map
-  useEffect(() => {
-    if (!keyCallbacks.has(key)) {
-      keyCallbacks.set(key, new Set())
-    }
-    keyCallbacks.get(key)!.add(callback)
-
-    return () => {
-      const set = keyCallbacks.get(key)
-      if (set) {
-        set.delete(callback)
-        if (set.size === 0) {
-          keyCallbacks.delete(key)
-        }
-      }
-    }
-  }, [key, callback])
-
-  useSWRSubscription('global-keydown', () => {
-    const handler = (e: KeyboardEvent) => {
-      if (e.metaKey && keyCallbacks.has(e.key)) {
-        keyCallbacks.get(e.key)!.forEach(cb => cb())
-      }
-    }
-    window.addEventListener('keydown', handler)
-    return () => window.removeEventListener('keydown', handler)
-  })
-}
-
-function Profile() {
-  // Multiple shortcuts will share the same listener
-  useKeyboardShortcut('p', () => { /* ... */ }) 
-  useKeyboardShortcut('k', () => { /* ... */ })
-  // ...
-}
-```

.claude/skills/vercel-react-best-practices/rules/client-swr-dedup.md

@@ -1,56 +0,0 @@
----
-title: Use SWR for Automatic Deduplication
-impact: MEDIUM-HIGH
-impactDescription: automatic deduplication
-tags: client, swr, deduplication, data-fetching
----
-
-## Use SWR for Automatic Deduplication
-
-SWR enables request deduplication, caching, and revalidation across component instances.
-
-**Incorrect (no deduplication, each instance fetches):**
-
-```tsx
-function UserList() {
-  const [users, setUsers] = useState([])
-  useEffect(() => {
-    fetch('/api/users')
-      .then(r => r.json())
-      .then(setUsers)
-  }, [])
-}
-```
-
-**Correct (multiple instances share one request):**
-
-```tsx
-import useSWR from 'swr'
-
-function UserList() {
-  const { data: users } = useSWR('/api/users', fetcher)
-}
-```
-
-**For immutable data:**
-
-```tsx
-import { useImmutableSWR } from '@/lib/swr'
-
-function StaticContent() {
-  const { data } = useImmutableSWR('/api/config', fetcher)
-}
-```
-
-**For mutations:**
-
-```tsx
-import { useSWRMutation } from 'swr/mutation'
-
-function UpdateButton() {
-  const { trigger } = useSWRMutation('/api/user', updateUser)
-  return <button onClick={() => trigger()}>Update</button>
-}
-```
-
-Reference: [https://swr.vercel.app](https://swr.vercel.app)

.claude/skills/vercel-react-best-practices/rules/js-batch-dom-css.md

@@ -1,82 +0,0 @@
----
-title: Batch DOM CSS Changes
-impact: MEDIUM
-impactDescription: reduces reflows/repaints
-tags: javascript, dom, css, performance, reflow
----
-
-## Batch DOM CSS Changes
-
-Avoid changing styles one property at a time. Group multiple CSS changes together via classes or `cssText` to minimize browser reflows.
-
-**Incorrect (multiple reflows):**
-
-```typescript
-function updateElementStyles(element: HTMLElement) {
-  // Each line triggers a reflow
-  element.style.width = '100px'
-  element.style.height = '200px'
-  element.style.backgroundColor = 'blue'
-  element.style.border = '1px solid black'
-}
-```
-
-**Correct (add class - single reflow):**
-
-```typescript
-// CSS file
-.highlighted-box {
-  width: 100px;
-  height: 200px;
-  background-color: blue;
-  border: 1px solid black;
-}
-
-// JavaScript
-function updateElementStyles(element: HTMLElement) {
-  element.classList.add('highlighted-box')
-}
-```
-
-**Correct (change cssText - single reflow):**
-
-```typescript
-function updateElementStyles(element: HTMLElement) {
-  element.style.cssText = `
-    width: 100px;
-    height: 200px;
-    background-color: blue;
-    border: 1px solid black;
-  `
-}
-```
-
-**React example:**
-
-```tsx
-// Incorrect: changing styles one by one
-function Box({ isHighlighted }: { isHighlighted: boolean }) {
-  const ref = useRef<HTMLDivElement>(null)
-  
-  useEffect(() => {
-    if (ref.current && isHighlighted) {
-      ref.current.style.width = '100px'
-      ref.current.style.height = '200px'
-      ref.current.style.backgroundColor = 'blue'
-    }
-  }, [isHighlighted])
-  
-  return <div ref={ref}>Content</div>
-}
-
-// Correct: toggle class
-function Box({ isHighlighted }: { isHighlighted: boolean }) {
-  return (
-    <div className={isHighlighted ? 'highlighted-box' : ''}>
-      Content
-    </div>
-  )
-}
-```
-
-Prefer CSS classes over inline styles when possible. Classes are cached by the browser and provide better separation of concerns.

.claude/skills/vercel-react-best-practices/rules/js-cache-function-results.md

@@ -1,80 +0,0 @@
----
-title: Cache Repeated Function Calls
-impact: MEDIUM
-impactDescription: avoid redundant computation
-tags: javascript, cache, memoization, performance
----
-
-## Cache Repeated Function Calls
-
-Use a module-level Map to cache function results when the same function is called repeatedly with the same inputs during render.
-
-**Incorrect (redundant computation):**
-
-```typescript
-function ProjectList({ projects }: { projects: Project[] }) {
-  return (
-    <div>
-      {projects.map(project => {
-        // slugify() called 100+ times for same project names
-        const slug = slugify(project.name)
-        
-        return <ProjectCard key={project.id} slug={slug} />
-      })}
-    </div>
-  )
-}
-```
-
-**Correct (cached results):**
-
-```typescript
-// Module-level cache
-const slugifyCache = new Map<string, string>()
-
-function cachedSlugify(text: string): string {
-  if (slugifyCache.has(text)) {
-    return slugifyCache.get(text)!
-  }
-  const result = slugify(text)
-  slugifyCache.set(text, result)
-  return result
-}
-
-function ProjectList({ projects }: { projects: Project[] }) {
-  return (
-    <div>
-      {projects.map(project => {
-        // Computed only once per unique project name
-        const slug = cachedSlugify(project.name)
-        
-        return <ProjectCard key={project.id} slug={slug} />
-      })}
-    </div>
-  )
-}
-```
-
-**Simpler pattern for single-value functions:**
-
-```typescript
-let isLoggedInCache: boolean | null = null
-
-function isLoggedIn(): boolean {
-  if (isLoggedInCache !== null) {
-    return isLoggedInCache
-  }
-  
-  isLoggedInCache = document.cookie.includes('auth=')
-  return isLoggedInCache
-}
-
-// Clear cache when auth changes
-function onAuthChange() {
-  isLoggedInCache = null
-}
-```
-
-Use a Map (not a hook) so it works everywhere: utilities, event handlers, not just React components.
-
-Reference: [How we made the Vercel Dashboard twice as fast](https://vercel.com/blog/how-we-made-the-vercel-dashboard-twice-as-fast)

.claude/skills/vercel-react-best-practices/rules/js-cache-property-access.md

@@ -1,28 +0,0 @@
----
-title: Cache Property Access in Loops
-impact: LOW-MEDIUM
-impactDescription: reduces lookups
-tags: javascript, loops, optimization, caching
----
-
-## Cache Property Access in Loops
-
-Cache object property lookups in hot paths.
-
-**Incorrect (3 lookups × N iterations):**
-
-```typescript
-for (let i = 0; i < arr.length; i++) {
-  process(obj.config.settings.value)
-}
-```
-
-**Correct (1 lookup total):**
-
-```typescript
-const value = obj.config.settings.value
-const len = arr.length
-for (let i = 0; i < len; i++) {
-  process(value)
-}
-```

.claude/skills/vercel-react-best-practices/rules/js-cache-storage.md

@@ -1,70 +0,0 @@
----
-title: Cache Storage API Calls
-impact: LOW-MEDIUM
-impactDescription: reduces expensive I/O
-tags: javascript, localStorage, storage, caching, performance
----
-
-## Cache Storage API Calls
-
-`localStorage`, `sessionStorage`, and `document.cookie` are synchronous and expensive. Cache reads in memory.
-
-**Incorrect (reads storage on every call):**
-
-```typescript
-function getTheme() {
-  return localStorage.getItem('theme') ?? 'light'
-}
-// Called 10 times = 10 storage reads
-```
-
-**Correct (Map cache):**
-
-```typescript
-const storageCache = new Map<string, string | null>()
-
-function getLocalStorage(key: string) {
-  if (!storageCache.has(key)) {
-    storageCache.set(key, localStorage.getItem(key))
-  }
-  return storageCache.get(key)
-}
-
-function setLocalStorage(key: string, value: string) {
-  localStorage.setItem(key, value)
-  storageCache.set(key, value)  // keep cache in sync
-}
-```
-
-Use a Map (not a hook) so it works everywhere: utilities, event handlers, not just React components.
-
-**Cookie caching:**
-
-```typescript
-let cookieCache: Record<string, string> | null = null
-
-function getCookie(name: string) {
-  if (!cookieCache) {
-    cookieCache = Object.fromEntries(
-      document.cookie.split('; ').map(c => c.split('='))
-    )
-  }
-  return cookieCache[name]
-}
-```
-
-**Important (invalidate on external changes):**
-
-If storage can change externally (another tab, server-set cookies), invalidate cache:
-
-```typescript
-window.addEventListener('storage', (e) => {
-  if (e.key) storageCache.delete(e.key)
-})
-
-document.addEventListener('visibilitychange', () => {
-  if (document.visibilityState === 'visible') {
-    storageCache.clear()
-  }
-})
-```

.claude/skills/vercel-react-best-practices/rules/js-combine-iterations.md

@@ -1,32 +0,0 @@
----
-title: Combine Multiple Array Iterations
-impact: LOW-MEDIUM
-impactDescription: reduces iterations
-tags: javascript, arrays, loops, performance
----
-
-## Combine Multiple Array Iterations
-
-Multiple `.filter()` or `.map()` calls iterate the array multiple times. Combine into one loop.
-
-**Incorrect (3 iterations):**
-
-```typescript
-const admins = users.filter(u => u.isAdmin)
-const testers = users.filter(u => u.isTester)
-const inactive = users.filter(u => !u.isActive)
-```
-
-**Correct (1 iteration):**
-
-```typescript
-const admins: User[] = []
-const testers: User[] = []
-const inactive: User[] = []
-
-for (const user of users) {
-  if (user.isAdmin) admins.push(user)
-  if (user.isTester) testers.push(user)
-  if (!user.isActive) inactive.push(user)
-}
-```

.claude/skills/vercel-react-best-practices/rules/js-early-exit.md

@@ -1,50 +0,0 @@
----
-title: Early Return from Functions
-impact: LOW-MEDIUM
-impactDescription: avoids unnecessary computation
-tags: javascript, functions, optimization, early-return
----
-
-## Early Return from Functions
-
-Return early when result is determined to skip unnecessary processing.
-
-**Incorrect (processes all items even after finding answer):**
-
-```typescript
-function validateUsers(users: User[]) {
-  let hasError = false
-  let errorMessage = ''
-  
-  for (const user of users) {
-    if (!user.email) {
-      hasError = true
-      errorMessage = 'Email required'
-    }
-    if (!user.name) {
-      hasError = true
-      errorMessage = 'Name required'
-    }
-    // Continues checking all users even after error found
-  }
-  
-  return hasError ? { valid: false, error: errorMessage } : { valid: true }
-}
-```
-
-**Correct (returns immediately on first error):**
-
-```typescript
-function validateUsers(users: User[]) {
-  for (const user of users) {
-    if (!user.email) {
-      return { valid: false, error: 'Email required' }
-    }
-    if (!user.name) {
-      return { valid: false, error: 'Name required' }
-    }
-  }
-
-  return { valid: true }
-}
-```

.claude/skills/vercel-react-best-practices/rules/js-hoist-regexp.md

@@ -1,45 +0,0 @@
----
-title: Hoist RegExp Creation
-impact: LOW-MEDIUM
-impactDescription: avoids recreation
-tags: javascript, regexp, optimization, memoization
----
-
-## Hoist RegExp Creation
-
-Don't create RegExp inside render. Hoist to module scope or memoize with `useMemo()`.
-
-**Incorrect (new RegExp every render):**
-
-```tsx
-function Highlighter({ text, query }: Props) {
-  const regex = new RegExp(`(${query})`, 'gi')
-  const parts = text.split(regex)
-  return <>{parts.map((part, i) => ...)}</>
-}
-```
-
-**Correct (memoize or hoist):**
-
-```tsx
-const EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
-
-function Highlighter({ text, query }: Props) {
-  const regex = useMemo(
-    () => new RegExp(`(${escapeRegex(query)})`, 'gi'),
-    [query]
-  )
-  const parts = text.split(regex)
-  return <>{parts.map((part, i) => ...)}</>
-}
-```
-
-**Warning (global regex has mutable state):**
-
-Global regex (`/g`) has mutable `lastIndex` state:
-
-```typescript
-const regex = /foo/g
-regex.test('foo')  // true, lastIndex = 3
-regex.test('foo')  // false, lastIndex = 0
-```

.claude/skills/vercel-react-best-practices/rules/js-index-maps.md

@@ -1,37 +0,0 @@
----
-title: Build Index Maps for Repeated Lookups
-impact: LOW-MEDIUM
-impactDescription: 1M ops to 2K ops
-tags: javascript, map, indexing, optimization, performance
----
-
-## Build Index Maps for Repeated Lookups
-
-Multiple `.find()` calls by the same key should use a Map.
-
-**Incorrect (O(n) per lookup):**
-
-```typescript
-function processOrders(orders: Order[], users: User[]) {
-  return orders.map(order => ({
-    ...order,
-    user: users.find(u => u.id === order.userId)
-  }))
-}
-```
-
-**Correct (O(1) per lookup):**
-
-```typescript
-function processOrders(orders: Order[], users: User[]) {
-  const userById = new Map(users.map(u => [u.id, u]))
-
-  return orders.map(order => ({
-    ...order,
-    user: userById.get(order.userId)
-  }))
-}
-```
-
-Build map once (O(n)), then all lookups are O(1).
-For 1000 orders × 1000 users: 1M ops → 2K ops.

.claude/skills/vercel-react-best-practices/rules/js-length-check-first.md

@@ -1,49 +0,0 @@
----
-title: Early Length Check for Array Comparisons
-impact: MEDIUM-HIGH
-impactDescription: avoids expensive operations when lengths differ
-tags: javascript, arrays, performance, optimization, comparison
----
-
-## Early Length Check for Array Comparisons
-
-When comparing arrays with expensive operations (sorting, deep equality, serialization), check lengths first. If lengths differ, the arrays cannot be equal.
-
-In real-world applications, this optimization is especially valuable when the comparison runs in hot paths (event handlers, render loops).
-
-**Incorrect (always runs expensive comparison):**
-
-```typescript
-function hasChanges(current: string[], original: string[]) {
-  // Always sorts and joins, even when lengths differ
-  return current.sort().join() !== original.sort().join()
-}
-```
-
-Two O(n log n) sorts run even when `current.length` is 5 and `original.length` is 100. There is also overhead of joining the arrays and comparing the strings.
-
-**Correct (O(1) length check first):**
-
-```typescript
-function hasChanges(current: string[], original: string[]) {
-  // Early return if lengths differ
-  if (current.length !== original.length) {
-    return true
-  }
-  // Only sort/join when lengths match
-  const currentSorted = current.toSorted()
-  const originalSorted = original.toSorted()
-  for (let i = 0; i < currentSorted.length; i++) {
-    if (currentSorted[i] !== originalSorted[i]) {
-      return true
-    }
-  }
-  return false
-}
-```
-
-This new approach is more efficient because:
-- It avoids the overhead of sorting and joining the arrays when lengths differ
-- It avoids consuming memory for the joined strings (especially important for large arrays)
-- It avoids mutating the original arrays
-- It returns early when a difference is found

.claude/skills/vercel-react-best-practices/rules/js-min-max-loop.md

@@ -1,82 +0,0 @@
----
-title: Use Loop for Min/Max Instead of Sort
-impact: LOW
-impactDescription: O(n) instead of O(n log n)
-tags: javascript, arrays, performance, sorting, algorithms
----
-
-## Use Loop for Min/Max Instead of Sort
-
-Finding the smallest or largest element only requires a single pass through the array. Sorting is wasteful and slower.
-
-**Incorrect (O(n log n) - sort to find latest):**
-
-```typescript
-interface Project {
-  id: string
-  name: string
-  updatedAt: number
-}
-
-function getLatestProject(projects: Project[]) {
-  const sorted = [...projects].sort((a, b) => b.updatedAt - a.updatedAt)
-  return sorted[0]
-}
-```
-
-Sorts the entire array just to find the maximum value.
-
-**Incorrect (O(n log n) - sort for oldest and newest):**
-
-```typescript
-function getOldestAndNewest(projects: Project[]) {
-  const sorted = [...projects].sort((a, b) => a.updatedAt - b.updatedAt)
-  return { oldest: sorted[0], newest: sorted[sorted.length - 1] }
-}
-```
-
-Still sorts unnecessarily when only min/max are needed.
-
-**Correct (O(n) - single loop):**
-
-```typescript
-function getLatestProject(projects: Project[]) {
-  if (projects.length === 0) return null
-  
-  let latest = projects[0]
-  
-  for (let i = 1; i < projects.length; i++) {
-    if (projects[i].updatedAt > latest.updatedAt) {
-      latest = projects[i]
-    }
-  }
-  
-  return latest
-}
-
-function getOldestAndNewest(projects: Project[]) {
-  if (projects.length === 0) return { oldest: null, newest: null }
-  
-  let oldest = projects[0]
-  let newest = projects[0]
-  
-  for (let i = 1; i < projects.length; i++) {
-    if (projects[i].updatedAt < oldest.updatedAt) oldest = projects[i]
-    if (projects[i].updatedAt > newest.updatedAt) newest = projects[i]
-  }
-  
-  return { oldest, newest }
-}
-```
-
-Single pass through the array, no copying, no sorting.
-
-**Alternative (Math.min/Math.max for small arrays):**
-
-```typescript
-const numbers = [5, 2, 8, 1, 9]
-const min = Math.min(...numbers)
-const max = Math.max(...numbers)
-```
-
-This works for small arrays but can be slower for very large arrays due to spread operator limitations. Use the loop approach for reliability.

.claude/skills/vercel-react-best-practices/rules/js-set-map-lookups.md

@@ -1,24 +0,0 @@
----
-title: Use Set/Map for O(1) Lookups
-impact: LOW-MEDIUM
-impactDescription: O(n) to O(1)
-tags: javascript, set, map, data-structures, performance
----
-
-## Use Set/Map for O(1) Lookups
-
-Convert arrays to Set/Map for repeated membership checks.
-
-**Incorrect (O(n) per check):**
-
-```typescript
-const allowedIds = ['a', 'b', 'c', ...]
-items.filter(item => allowedIds.includes(item.id))
-```
-
-**Correct (O(1) per check):**
-
-```typescript
-const allowedIds = new Set(['a', 'b', 'c', ...])
-items.filter(item => allowedIds.has(item.id))
-```

.claude/skills/vercel-react-best-practices/rules/js-tosorted-immutable.md

@@ -1,57 +0,0 @@
----
-title: Use toSorted() Instead of sort() for Immutability
-impact: MEDIUM-HIGH
-impactDescription: prevents mutation bugs in React state
-tags: javascript, arrays, immutability, react, state, mutation
----
-
-## Use toSorted() Instead of sort() for Immutability
-
-`.sort()` mutates the array in place, which can cause bugs with React state and props. Use `.toSorted()` to create a new sorted array without mutation.
-
-**Incorrect (mutates original array):**
-
-```typescript
-function UserList({ users }: { users: User[] }) {
-  // Mutates the users prop array!
-  const sorted = useMemo(
-    () => users.sort((a, b) => a.name.localeCompare(b.name)),
-    [users]
-  )
-  return <div>{sorted.map(renderUser)}</div>
-}
-```
-
-**Correct (creates new array):**
-
-```typescript
-function UserList({ users }: { users: User[] }) {
-  // Creates new sorted array, original unchanged
-  const sorted = useMemo(
-    () => users.toSorted((a, b) => a.name.localeCompare(b.name)),
-    [users]
-  )
-  return <div>{sorted.map(renderUser)}</div>
-}
-```
-
-**Why this matters in React:**
-
-1. Props/state mutations break React's immutability model - React expects props and state to be treated as read-only
-2. Causes stale closure bugs - Mutating arrays inside closures (callbacks, effects) can lead to unexpected behavior
-
-**Browser support (fallback for older browsers):**
-
-`.toSorted()` is available in all modern browsers (Chrome 110+, Safari 16+, Firefox 115+, Node.js 20+). For older environments, use spread operator:
-
-```typescript
-// Fallback for older browsers
-const sorted = [...items].sort((a, b) => a.value - b.value)
-```
-
-**Other immutable array methods:**
-
-- `.toSorted()` - immutable sort
-- `.toReversed()` - immutable reverse
-- `.toSpliced()` - immutable splice
-- `.with()` - immutable element replacement

.claude/skills/vercel-react-best-practices/rules/rendering-activity.md

@@ -1,26 +0,0 @@
----
-title: Use Activity Component for Show/Hide
-impact: MEDIUM
-impactDescription: preserves state/DOM
-tags: rendering, activity, visibility, state-preservation
----
-
-## Use Activity Component for Show/Hide
-
-Use React's `<Activity>` to preserve state/DOM for expensive components that frequently toggle visibility.
-
-**Usage:**
-
-```tsx
-import { Activity } from 'react'
-
-function Dropdown({ isOpen }: Props) {
-  return (
-    <Activity mode={isOpen ? 'visible' : 'hidden'}>
-      <ExpensiveMenu />
-    </Activity>
-  )
-}
-```
-
-Avoids expensive re-renders and state loss.

.claude/skills/vercel-react-best-practices/rules/rendering-animate-svg-wrapper.md

@@ -1,47 +0,0 @@
----
-title: Animate SVG Wrapper Instead of SVG Element
-impact: LOW
-impactDescription: enables hardware acceleration
-tags: rendering, svg, css, animation, performance
----
-
-## Animate SVG Wrapper Instead of SVG Element
-
-Many browsers don't have hardware acceleration for CSS3 animations on SVG elements. Wrap SVG in a `<div>` and animate the wrapper instead.
-
-**Incorrect (animating SVG directly - no hardware acceleration):**
-
-```tsx
-function LoadingSpinner() {
-  return (
-    <svg 
-      className="animate-spin"
-      width="24" 
-      height="24" 
-      viewBox="0 0 24 24"
-    >
-      <circle cx="12" cy="12" r="10" stroke="currentColor" />
-    </svg>
-  )
-}
-```
-
-**Correct (animating wrapper div - hardware accelerated):**
-
-```tsx
-function LoadingSpinner() {
-  return (
-    <div className="animate-spin">
-      <svg 
-        width="24" 
-        height="24" 
-        viewBox="0 0 24 24"
-      >
-        <circle cx="12" cy="12" r="10" stroke="currentColor" />
-      </svg>
-    </div>
-  )
-}
-```
-
-This applies to all CSS transforms and transitions (`transform`, `opacity`, `translate`, `scale`, `rotate`). The wrapper div allows browsers to use GPU acceleration for smoother animations.

.claude/skills/vercel-react-best-practices/rules/rendering-conditional-render.md

@@ -1,40 +0,0 @@
----
-title: Use Explicit Conditional Rendering
-impact: LOW
-impactDescription: prevents rendering 0 or NaN
-tags: rendering, conditional, jsx, falsy-values
----
-
-## Use Explicit Conditional Rendering
-
-Use explicit ternary operators (`? :`) instead of `&&` for conditional rendering when the condition can be `0`, `NaN`, or other falsy values that render.
-
-**Incorrect (renders "0" when count is 0):**
-
-```tsx
-function Badge({ count }: { count: number }) {
-  return (
-    <div>
-      {count && <span className="badge">{count}</span>}
-    </div>
-  )
-}
-
-// When count = 0, renders: <div>0</div>
-// When count = 5, renders: <div><span class="badge">5</span></div>
-```
-
-**Correct (renders nothing when count is 0):**
-
-```tsx
-function Badge({ count }: { count: number }) {
-  return (
-    <div>
-      {count > 0 ? <span className="badge">{count}</span> : null}
-    </div>
-  )
-}
-
-// When count = 0, renders: <div></div>
-// When count = 5, renders: <div><span class="badge">5</span></div>
-```

.claude/skills/vercel-react-best-practices/rules/rendering-content-visibility.md

@@ -1,38 +0,0 @@
----
-title: CSS content-visibility for Long Lists
-impact: HIGH
-impactDescription: faster initial render
-tags: rendering, css, content-visibility, long-lists
----
-
-## CSS content-visibility for Long Lists
-
-Apply `content-visibility: auto` to defer off-screen rendering.
-
-**CSS:**
-
-```css
-.message-item {
-  content-visibility: auto;
-  contain-intrinsic-size: 0 80px;
-}
-```
-
-**Example:**
-
-```tsx
-function MessageList({ messages }: { messages: Message[] }) {
-  return (
-    <div className="overflow-y-auto h-screen">
-      {messages.map(msg => (
-        <div key={msg.id} className="message-item">
-          <Avatar user={msg.author} />
-          <div>{msg.content}</div>
-        </div>
-      ))}
-    </div>
-  )
-}
-```
-
-For 1000 messages, browser skips layout/paint for ~990 off-screen items (10× faster initial render).

.claude/skills/vercel-react-best-practices/rules/rendering-hoist-jsx.md

@@ -1,46 +0,0 @@
----
-title: Hoist Static JSX Elements
-impact: LOW
-impactDescription: avoids re-creation
-tags: rendering, jsx, static, optimization
----
-
-## Hoist Static JSX Elements
-
-Extract static JSX outside components to avoid re-creation.
-
-**Incorrect (recreates element every render):**
-
-```tsx
-function LoadingSkeleton() {
-  return <div className="animate-pulse h-20 bg-gray-200" />
-}
-
-function Container() {
-  return (
-    <div>
-      {loading && <LoadingSkeleton />}
-    </div>
-  )
-}
-```
-
-**Correct (reuses same element):**
-
-```tsx
-const loadingSkeleton = (
-  <div className="animate-pulse h-20 bg-gray-200" />
-)
-
-function Container() {
-  return (
-    <div>
-      {loading && loadingSkeleton}
-    </div>
-  )
-}
-```
-
-This is especially helpful for large and static SVG nodes, which can be expensive to recreate on every render.
-
-**Note:** If your project has [React Compiler](https://react.dev/learn/react-compiler) enabled, the compiler automatically hoists static JSX elements and optimizes component re-renders, making manual hoisting unnecessary.

.claude/skills/vercel-react-best-practices/rules/rendering-hydration-no-flicker.md

@@ -1,82 +0,0 @@
----
-title: Prevent Hydration Mismatch Without Flickering
-impact: MEDIUM
-impactDescription: avoids visual flicker and hydration errors
-tags: rendering, ssr, hydration, localStorage, flicker
----
-
-## Prevent Hydration Mismatch Without Flickering
-
-When rendering content that depends on client-side storage (localStorage, cookies), avoid both SSR breakage and post-hydration flickering by injecting a synchronous script that updates the DOM before React hydrates.
-
-**Incorrect (breaks SSR):**
-
-```tsx
-function ThemeWrapper({ children }: { children: ReactNode }) {
-  // localStorage is not available on server - throws error
-  const theme = localStorage.getItem('theme') || 'light'
-  
-  return (
-    <div className={theme}>
-      {children}
-    </div>
-  )
-}
-```
-
-Server-side rendering will fail because `localStorage` is undefined.
-
-**Incorrect (visual flickering):**
-
-```tsx
-function ThemeWrapper({ children }: { children: ReactNode }) {
-  const [theme, setTheme] = useState('light')
-  
-  useEffect(() => {
-    // Runs after hydration - causes visible flash
-    const stored = localStorage.getItem('theme')
-    if (stored) {
-      setTheme(stored)
-    }
-  }, [])
-  
-  return (
-    <div className={theme}>
-      {children}
-    </div>
-  )
-}
-```
-
-Component first renders with default value (`light`), then updates after hydration, causing a visible flash of incorrect content.
-
-**Correct (no flicker, no hydration mismatch):**
-
-```tsx
-function ThemeWrapper({ children }: { children: ReactNode }) {
-  return (
-    <>
-      <div id="theme-wrapper">
-        {children}
-      </div>
-      <script
-        dangerouslySetInnerHTML={{
-          __html: `
-            (function() {
-              try {
-                var theme = localStorage.getItem('theme') || 'light';
-                var el = document.getElementById('theme-wrapper');
-                if (el) el.className = theme;
-              } catch (e) {}
-            })();
-          `,
-        }}
-      />
-    </>
-  )
-}
-```
-
-The inline script executes synchronously before showing the element, ensuring the DOM already has the correct value. No flickering, no hydration mismatch.
-
-This pattern is especially useful for theme toggles, user preferences, authentication states, and any client-only data that should render immediately without flashing default values.

.claude/skills/vercel-react-best-practices/rules/rendering-svg-precision.md

@@ -1,28 +0,0 @@
----
-title: Optimize SVG Precision
-impact: LOW
-impactDescription: reduces file size
-tags: rendering, svg, optimization, svgo
----
-
-## Optimize SVG Precision
-
-Reduce SVG coordinate precision to decrease file size. The optimal precision depends on the viewBox size, but in general reducing precision should be considered.
-
-**Incorrect (excessive precision):**
-
-```svg
-<path d="M 10.293847 20.847362 L 30.938472 40.192837" />
-```
-
-**Correct (1 decimal place):**
-
-```svg
-<path d="M 10.3 20.8 L 30.9 40.2" />
-```
-
-**Automate with SVGO:**
-
-```bash
-npx svgo --precision=1 --multipass icon.svg
-```

.claude/skills/vercel-react-best-practices/rules/rerender-defer-reads.md

@@ -1,39 +0,0 @@
----
-title: Defer State Reads to Usage Point
-impact: MEDIUM
-impactDescription: avoids unnecessary subscriptions
-tags: rerender, searchParams, localStorage, optimization
----
-
-## Defer State Reads to Usage Point
-
-Don't subscribe to dynamic state (searchParams, localStorage) if you only read it inside callbacks.
-
-**Incorrect (subscribes to all searchParams changes):**
-
-```tsx
-function ShareButton({ chatId }: { chatId: string }) {
-  const searchParams = useSearchParams()
-
-  const handleShare = () => {
-    const ref = searchParams.get('ref')
-    shareChat(chatId, { ref })
-  }
-
-  return <button onClick={handleShare}>Share</button>
-}
-```
-
-**Correct (reads on demand, no subscription):**
-
-```tsx
-function ShareButton({ chatId }: { chatId: string }) {
-  const handleShare = () => {
-    const params = new URLSearchParams(window.location.search)
-    const ref = params.get('ref')
-    shareChat(chatId, { ref })
-  }
-
-  return <button onClick={handleShare}>Share</button>
-}
-```

.claude/skills/vercel-react-best-practices/rules/rerender-dependencies.md

@@ -1,45 +0,0 @@
----
-title: Narrow Effect Dependencies
-impact: LOW
-impactDescription: minimizes effect re-runs
-tags: rerender, useEffect, dependencies, optimization
----
-
-## Narrow Effect Dependencies
-
-Specify primitive dependencies instead of objects to minimize effect re-runs.
-
-**Incorrect (re-runs on any user field change):**
-
-```tsx
-useEffect(() => {
-  console.log(user.id)
-}, [user])
-```
-
-**Correct (re-runs only when id changes):**
-
-```tsx
-useEffect(() => {
-  console.log(user.id)
-}, [user.id])
-```
-
-**For derived state, compute outside effect:**
-
-```tsx
-// Incorrect: runs on width=767, 766, 765...
-useEffect(() => {
-  if (width < 768) {
-    enableMobileMode()
-  }
-}, [width])
-
-// Correct: runs only on boolean transition
-const isMobile = width < 768
-useEffect(() => {
-  if (isMobile) {
-    enableMobileMode()
-  }
-}, [isMobile])
-```

.claude/skills/vercel-react-best-practices/rules/rerender-derived-state.md

@@ -1,29 +0,0 @@
----
-title: Subscribe to Derived State
-impact: MEDIUM
-impactDescription: reduces re-render frequency
-tags: rerender, derived-state, media-query, optimization
----
-
-## Subscribe to Derived State
-
-Subscribe to derived boolean state instead of continuous values to reduce re-render frequency.
-
-**Incorrect (re-renders on every pixel change):**
-
-```tsx
-function Sidebar() {
-  const width = useWindowWidth()  // updates continuously
-  const isMobile = width < 768
-  return <nav className={isMobile ? 'mobile' : 'desktop'}>
-}
-```
-
-**Correct (re-renders only when boolean changes):**
-
-```tsx
-function Sidebar() {
-  const isMobile = useMediaQuery('(max-width: 767px)')
-  return <nav className={isMobile ? 'mobile' : 'desktop'}>
-}
-```

.claude/skills/vercel-react-best-practices/rules/rerender-functional-setstate.md

@@ -1,74 +0,0 @@
----
-title: Use Functional setState Updates
-impact: MEDIUM
-impactDescription: prevents stale closures and unnecessary callback recreations
-tags: react, hooks, useState, useCallback, callbacks, closures
----
-
-## Use Functional setState Updates
-
-When updating state based on the current state value, use the functional update form of setState instead of directly referencing the state variable. This prevents stale closures, eliminates unnecessary dependencies, and creates stable callback references.
-
-**Incorrect (requires state as dependency):**
-
-```tsx
-function TodoList() {
-  const [items, setItems] = useState(initialItems)
-  
-  // Callback must depend on items, recreated on every items change
-  const addItems = useCallback((newItems: Item[]) => {
-    setItems([...items, ...newItems])
-  }, [items])  // ❌ items dependency causes recreations
-  
-  // Risk of stale closure if dependency is forgotten
-  const removeItem = useCallback((id: string) => {
-    setItems(items.filter(item => item.id !== id))
-  }, [])  // ❌ Missing items dependency - will use stale items!
-  
-  return <ItemsEditor items={items} onAdd={addItems} onRemove={removeItem} />
-}
-```
-
-The first callback is recreated every time `items` changes, which can cause child components to re-render unnecessarily. The second callback has a stale closure bug—it will always reference the initial `items` value.
-
-**Correct (stable callbacks, no stale closures):**
-
-```tsx
-function TodoList() {
-  const [items, setItems] = useState(initialItems)
-  
-  // Stable callback, never recreated
-  const addItems = useCallback((newItems: Item[]) => {
-    setItems(curr => [...curr, ...newItems])
-  }, [])  // ✅ No dependencies needed
-  
-  // Always uses latest state, no stale closure risk
-  const removeItem = useCallback((id: string) => {
-    setItems(curr => curr.filter(item => item.id !== id))
-  }, [])  // ✅ Safe and stable
-  
-  return <ItemsEditor items={items} onAdd={addItems} onRemove={removeItem} />
-}
-```
-
-**Benefits:**
-
-1. **Stable callback references** - Callbacks don't need to be recreated when state changes
-2. **No stale closures** - Always operates on the latest state value
-3. **Fewer dependencies** - Simplifies dependency arrays and reduces memory leaks
-4. **Prevents bugs** - Eliminates the most common source of React closure bugs
-
-**When to use functional updates:**
-
-- Any setState that depends on the current state value
-- Inside useCallback/useMemo when state is needed
-- Event handlers that reference state
-- Async operations that update state
-
-**When direct updates are fine:**
-
-- Setting state to a static value: `setCount(0)`
-- Setting state from props/arguments only: `setName(newName)`
-- State doesn't depend on previous value
-
-**Note:** If your project has [React Compiler](https://react.dev/learn/react-compiler) enabled, the compiler can automatically optimize some cases, but functional updates are still recommended for correctness and to prevent stale closure bugs.

.claude/skills/vercel-react-best-practices/rules/rerender-lazy-state-init.md

@@ -1,58 +0,0 @@
----
-title: Use Lazy State Initialization
-impact: MEDIUM
-impactDescription: wasted computation on every render
-tags: react, hooks, useState, performance, initialization
----
-
-## Use Lazy State Initialization
-
-Pass a function to `useState` for expensive initial values. Without the function form, the initializer runs on every render even though the value is only used once.
-
-**Incorrect (runs on every render):**
-
-```tsx
-function FilteredList({ items }: { items: Item[] }) {
-  // buildSearchIndex() runs on EVERY render, even after initialization
-  const [searchIndex, setSearchIndex] = useState(buildSearchIndex(items))
-  const [query, setQuery] = useState('')
-  
-  // When query changes, buildSearchIndex runs again unnecessarily
-  return <SearchResults index={searchIndex} query={query} />
-}
-
-function UserProfile() {
-  // JSON.parse runs on every render
-  const [settings, setSettings] = useState(
-    JSON.parse(localStorage.getItem('settings') || '{}')
-  )
-  
-  return <SettingsForm settings={settings} onChange={setSettings} />
-}
-```
-
-**Correct (runs only once):**
-
-```tsx
-function FilteredList({ items }: { items: Item[] }) {
-  // buildSearchIndex() runs ONLY on initial render
-  const [searchIndex, setSearchIndex] = useState(() => buildSearchIndex(items))
-  const [query, setQuery] = useState('')
-  
-  return <SearchResults index={searchIndex} query={query} />
-}
-
-function UserProfile() {
-  // JSON.parse runs only on initial render
-  const [settings, setSettings] = useState(() => {
-    const stored = localStorage.getItem('settings')
-    return stored ? JSON.parse(stored) : {}
-  })
-  
-  return <SettingsForm settings={settings} onChange={setSettings} />
-}
-```
-
-Use lazy initialization when computing initial values from localStorage/sessionStorage, building data structures (indexes, maps), reading from the DOM, or performing heavy transformations.
-
-For simple primitives (`useState(0)`), direct references (`useState(props.value)`), or cheap literals (`useState({})`), the function form is unnecessary.

.claude/skills/vercel-react-best-practices/rules/rerender-memo.md

@@ -1,44 +0,0 @@
----
-title: Extract to Memoized Components
-impact: MEDIUM
-impactDescription: enables early returns
-tags: rerender, memo, useMemo, optimization
----
-
-## Extract to Memoized Components
-
-Extract expensive work into memoized components to enable early returns before computation.
-
-**Incorrect (computes avatar even when loading):**
-
-```tsx
-function Profile({ user, loading }: Props) {
-  const avatar = useMemo(() => {
-    const id = computeAvatarId(user)
-    return <Avatar id={id} />
-  }, [user])
-
-  if (loading) return <Skeleton />
-  return <div>{avatar}</div>
-}
-```
-
-**Correct (skips computation when loading):**
-
-```tsx
-const UserAvatar = memo(function UserAvatar({ user }: { user: User }) {
-  const id = useMemo(() => computeAvatarId(user), [user])
-  return <Avatar id={id} />
-})
-
-function Profile({ user, loading }: Props) {
-  if (loading) return <Skeleton />
-  return (
-    <div>
-      <UserAvatar user={user} />
-    </div>
-  )
-}
-```
-
-**Note:** If your project has [React Compiler](https://react.dev/learn/react-compiler) enabled, manual memoization with `memo()` and `useMemo()` is not necessary. The compiler automatically optimizes re-renders.

.claude/skills/vercel-react-best-practices/rules/rerender-transitions.md

@@ -1,40 +0,0 @@
----
-title: Use Transitions for Non-Urgent Updates
-impact: MEDIUM
-impactDescription: maintains UI responsiveness
-tags: rerender, transitions, startTransition, performance
----
-
-## Use Transitions for Non-Urgent Updates
-
-Mark frequent, non-urgent state updates as transitions to maintain UI responsiveness.
-
-**Incorrect (blocks UI on every scroll):**
-
-```tsx
-function ScrollTracker() {
-  const [scrollY, setScrollY] = useState(0)
-  useEffect(() => {
-    const handler = () => setScrollY(window.scrollY)
-    window.addEventListener('scroll', handler, { passive: true })
-    return () => window.removeEventListener('scroll', handler)
-  }, [])
-}
-```
-
-**Correct (non-blocking updates):**
-
-```tsx
-import { startTransition } from 'react'
-
-function ScrollTracker() {
-  const [scrollY, setScrollY] = useState(0)
-  useEffect(() => {
-    const handler = () => {
-      startTransition(() => setScrollY(window.scrollY))
-    }
-    window.addEventListener('scroll', handler, { passive: true })
-    return () => window.removeEventListener('scroll', handler)
-  }, [])
-}
-```

.claude/skills/vercel-react-best-practices/rules/server-after-nonblocking.md

@@ -1,73 +0,0 @@
----
-title: Use after() for Non-Blocking Operations
-impact: MEDIUM
-impactDescription: faster response times
-tags: server, async, logging, analytics, side-effects
----
-
-## Use after() for Non-Blocking Operations
-
-Use Next.js's `after()` to schedule work that should execute after a response is sent. This prevents logging, analytics, and other side effects from blocking the response.
-
-**Incorrect (blocks response):**
-
-```tsx
-import { logUserAction } from '@/app/utils'
-
-export async function POST(request: Request) {
-  // Perform mutation
-  await updateDatabase(request)
-  
-  // Logging blocks the response
-  const userAgent = request.headers.get('user-agent') || 'unknown'
-  await logUserAction({ userAgent })
-  
-  return new Response(JSON.stringify({ status: 'success' }), {
-    status: 200,
-    headers: { 'Content-Type': 'application/json' }
-  })
-}
-```
-
-**Correct (non-blocking):**
-
-```tsx
-import { after } from 'next/server'
-import { headers, cookies } from 'next/headers'
-import { logUserAction } from '@/app/utils'
-
-export async function POST(request: Request) {
-  // Perform mutation
-  await updateDatabase(request)
-  
-  // Log after response is sent
-  after(async () => {
-    const userAgent = (await headers()).get('user-agent') || 'unknown'
-    const sessionCookie = (await cookies()).get('session-id')?.value || 'anonymous'
-    
-    logUserAction({ sessionCookie, userAgent })
-  })
-  
-  return new Response(JSON.stringify({ status: 'success' }), {
-    status: 200,
-    headers: { 'Content-Type': 'application/json' }
-  })
-}
-```
-
-The response is sent immediately while logging happens in the background.
-
-**Common use cases:**
-
-- Analytics tracking
-- Audit logging
-- Sending notifications
-- Cache invalidation
-- Cleanup tasks
-
-**Important notes:**
-
-- `after()` runs even if the response fails or redirects
-- Works in Server Actions, Route Handlers, and Server Components
-
-Reference: [https://nextjs.org/docs/app/api-reference/functions/after](https://nextjs.org/docs/app/api-reference/functions/after)

.claude/skills/vercel-react-best-practices/rules/server-cache-lru.md

@@ -1,41 +0,0 @@
----
-title: Cross-Request LRU Caching
-impact: HIGH
-impactDescription: caches across requests
-tags: server, cache, lru, cross-request
----
-
-## Cross-Request LRU Caching
-
-`React.cache()` only works within one request. For data shared across sequential requests (user clicks button A then button B), use an LRU cache.
-
-**Implementation:**
-
-```typescript
-import { LRUCache } from 'lru-cache'
-
-const cache = new LRUCache<string, any>({
-  max: 1000,
-  ttl: 5 * 60 * 1000  // 5 minutes
-})
-
-export async function getUser(id: string) {
-  const cached = cache.get(id)
-  if (cached) return cached
-
-  const user = await db.user.findUnique({ where: { id } })
-  cache.set(id, user)
-  return user
-}
-
-// Request 1: DB query, result cached
-// Request 2: cache hit, no DB query
-```
-
-Use when sequential user actions hit multiple endpoints needing the same data within seconds.
-
-**With Vercel's [Fluid Compute](https://vercel.com/docs/fluid-compute):** LRU caching is especially effective because multiple concurrent requests can share the same function instance and cache. This means the cache persists across requests without needing external storage like Redis.
-
-**In traditional serverless:** Each invocation runs in isolation, so consider Redis for cross-process caching.
-
-Reference: [https://github.com/isaacs/node-lru-cache](https://github.com/isaacs/node-lru-cache)

.claude/skills/vercel-react-best-practices/rules/server-cache-react.md

@@ -1,26 +0,0 @@
----
-title: Per-Request Deduplication with React.cache()
-impact: MEDIUM
-impactDescription: deduplicates within request
-tags: server, cache, react-cache, deduplication
----
-
-## Per-Request Deduplication with React.cache()
-
-Use `React.cache()` for server-side request deduplication. Authentication and database queries benefit most.
-
-**Usage:**
-
-```typescript
-import { cache } from 'react'
-
-export const getCurrentUser = cache(async () => {
-  const session = await auth()
-  if (!session?.user?.id) return null
-  return await db.user.findUnique({
-    where: { id: session.user.id }
-  })
-})
-```
-
-Within a single request, multiple calls to `getCurrentUser()` execute the query only once.

.claude/skills/vercel-react-best-practices/rules/server-parallel-fetching.md

@@ -1,79 +0,0 @@
----
-title: Parallel Data Fetching with Component Composition
-impact: CRITICAL
-impactDescription: eliminates server-side waterfalls
-tags: server, rsc, parallel-fetching, composition
----
-
-## Parallel Data Fetching with Component Composition
-
-React Server Components execute sequentially within a tree. Restructure with composition to parallelize data fetching.
-
-**Incorrect (Sidebar waits for Page's fetch to complete):**
-
-```tsx
-export default async function Page() {
-  const header = await fetchHeader()
-  return (
-    <div>
-      <div>{header}</div>
-      <Sidebar />
-    </div>
-  )
-}
-
-async function Sidebar() {
-  const items = await fetchSidebarItems()
-  return <nav>{items.map(renderItem)}</nav>
-}
-```
-
-**Correct (both fetch simultaneously):**
-
-```tsx
-async function Header() {
-  const data = await fetchHeader()
-  return <div>{data}</div>
-}
-
-async function Sidebar() {
-  const items = await fetchSidebarItems()
-  return <nav>{items.map(renderItem)}</nav>
-}
-
-export default function Page() {
-  return (
-    <div>
-      <Header />
-      <Sidebar />
-    </div>
-  )
-}
-```
-
-**Alternative with children prop:**
-
-```tsx
-async function Layout({ children }: { children: ReactNode }) {
-  const header = await fetchHeader()
-  return (
-    <div>
-      <div>{header}</div>
-      {children}
-    </div>
-  )
-}
-
-async function Sidebar() {
-  const items = await fetchSidebarItems()
-  return <nav>{items.map(renderItem)}</nav>
-}
-
-export default function Page() {
-  return (
-    <Layout>
-      <Sidebar />
-    </Layout>
-  )
-}
-```

.claude/skills/vercel-react-best-practices/rules/server-serialization.md

@@ -1,38 +0,0 @@
----
-title: Minimize Serialization at RSC Boundaries
-impact: HIGH
-impactDescription: reduces data transfer size
-tags: server, rsc, serialization, props
----
-
-## Minimize Serialization at RSC Boundaries
-
-The React Server/Client boundary serializes all object properties into strings and embeds them in the HTML response and subsequent RSC requests. This serialized data directly impacts page weight and load time, so **size matters a lot**. Only pass fields that the client actually uses.
-
-**Incorrect (serializes all 50 fields):**
-
-```tsx
-async function Page() {
-  const user = await fetchUser()  // 50 fields
-  return <Profile user={user} />
-}
-
-'use client'
-function Profile({ user }: { user: User }) {
-  return <div>{user.name}</div>  // uses 1 field
-}
-```
-
-**Correct (serializes only 1 field):**
-
-```tsx
-async function Page() {
-  const user = await fetchUser()
-  return <Profile name={user.name} />
-}
-
-'use client'
-function Profile({ name }: { name: string }) {
-  return <div>{name}</div>
-}
-```

.dockerignore

@@ -1,9 +1,6 @@
 # Ignore everything by default, selectively add things to context
 *
 
-# Documentation (for embeddings/search)
-!docs/
-
 # Platform - Libs
 !autogpt_platform/autogpt_libs/autogpt_libs/
 !autogpt_platform/autogpt_libs/pyproject.toml
@@ -19,7 +16,6 @@
 !autogpt_platform/backend/poetry.lock
 !autogpt_platform/backend/README.md
 !autogpt_platform/backend/.env
-!autogpt_platform/backend/gen_prisma_types_stub.py
 
 # Platform - Market
 !autogpt_platform/market/market/

.github/workflows/classic-autogpt-ci.yml

@@ -6,15 +6,11 @@ on:
     paths:
       - '.github/workflows/classic-autogpt-ci.yml'
       - 'classic/original_autogpt/**'
-      - 'classic/direct_benchmark/**'
-      - 'classic/forge/**'
   pull_request:
     branches: [ master, dev, release-* ]
     paths:
       - '.github/workflows/classic-autogpt-ci.yml'
       - 'classic/original_autogpt/**'
-      - 'classic/direct_benchmark/**'
-      - 'classic/forge/**'
 
 concurrency:
   group: ${{ format('classic-autogpt-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
@@ -23,22 +19,47 @@ concurrency:
 defaults:
   run:
     shell: bash
-    working-directory: classic
+    working-directory: classic/original_autogpt
 
 jobs:
   test:
     permissions:
       contents: read
     timeout-minutes: 30
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        platform-os: [ubuntu, macos, macos-arm64, windows]
+    runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }}
 
     steps:
-      - name: Start MinIO service
+      # Quite slow on macOS (2~4 minutes to set up Docker)
+      # - name: Set up Docker (macOS)
+      #   if: runner.os == 'macOS'
+      #   uses: crazy-max/ghaction-setup-docker@v3
+
+      - name: Start MinIO service (Linux)
+        if: runner.os == 'Linux'
         working-directory: '.'
         run: |
           docker pull minio/minio:edge-cicd
           docker run -d -p 9000:9000 minio/minio:edge-cicd
 
+      - name: Start MinIO service (macOS)
+        if: runner.os == 'macOS'
+        working-directory: ${{ runner.temp }}
+        run: |
+          brew install minio/stable/minio
+          mkdir data
+          minio server ./data &
+
+      # No MinIO on Windows:
+      # - Windows doesn't support running Linux Docker containers
+      # - It doesn't seem possible to start background processes on Windows. They are
+      #   killed after the step returns.
+      #   See: https://github.com/actions/runner/issues/598#issuecomment-2011890429
+
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -50,23 +71,41 @@ jobs:
           git config --global user.name "Auto-GPT-Bot"
           git config --global user.email "github-bot@agpt.co"
 
-      - name: Set up Python 3.12
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
 
       - id: get_date
         name: Get date
         run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
 
       - name: Set up Python dependency cache
+        # On Windows, unpacking cached dependencies takes longer than just installing them
+        if: runner.os != 'Windows'
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('classic/poetry.lock') }}
+          path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/original_autogpt/poetry.lock') }}
 
-      - name: Install Poetry
-        run: curl -sSL https://install.python-poetry.org | python3 -
+      - name: Install Poetry (Unix)
+        if: runner.os != 'Windows'
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+          if [ "${{ runner.os }}" = "macOS" ]; then
+            PATH="$HOME/.local/bin:$PATH"
+            echo "$HOME/.local/bin" >> $GITHUB_PATH
+          fi
+
+      - name: Install Poetry (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
+
+          $env:PATH += ";$env:APPDATA\Python\Scripts"
+          echo "$env:APPDATA\Python\Scripts" >> $env:GITHUB_PATH
 
       - name: Install Python dependencies
         run: poetry install
@@ -77,12 +116,12 @@ jobs:
             --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
             --numprocesses=logical --durations=10 \
             --junitxml=junit.xml -o junit_family=legacy \
-            original_autogpt/tests/unit original_autogpt/tests/integration
+            tests/unit tests/integration
         env:
           CI: true
           PLAIN_OUTPUT: True
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          S3_ENDPOINT_URL: http://127.0.0.1:9000
+          S3_ENDPOINT_URL: ${{ runner.os != 'Windows' && 'http://127.0.0.1:9000' || '' }}
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
@@ -96,11 +135,11 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          flags: autogpt-agent
+          flags: autogpt-agent,${{ runner.os }}
 
       - name: Upload logs to artifact
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: test-logs
-          path: classic/logs/
+          path: classic/original_autogpt/logs/

.github/workflows/classic-autogpts-ci.yml

@@ -11,6 +11,9 @@ on:
       - 'classic/original_autogpt/**'
       - 'classic/forge/**'
       - 'classic/benchmark/**'
+      - 'classic/run'
+      - 'classic/cli.py'
+      - 'classic/setup.py'
       - '!**/*.md'
   pull_request:
     branches: [ master, dev, release-* ]
@@ -19,6 +22,9 @@ on:
       - 'classic/original_autogpt/**'
       - 'classic/forge/**'
       - 'classic/benchmark/**'
+      - 'classic/run'
+      - 'classic/cli.py'
+      - 'classic/setup.py'
       - '!**/*.md'
 
 defaults:
@@ -29,9 +35,13 @@ defaults:
 jobs:
   serve-agent-protocol:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        agent-name: [ original_autogpt ]
+      fail-fast: false
     timeout-minutes: 20
     env:
-      min-python-version: '3.12'
+      min-python-version: '3.10'
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -45,22 +55,22 @@ jobs:
           python-version: ${{ env.min-python-version }}
 
       - name: Install Poetry
+        working-directory: ./classic/${{ matrix.agent-name }}/
         run: |
           curl -sSL https://install.python-poetry.org | python -
 
-      - name: Install dependencies
-        run: poetry install
-
-      - name: Run smoke tests with direct-benchmark
+      - name: Run regression tests
         run: |
-          poetry run direct-benchmark run \
-            --strategies one_shot \
-            --models claude \
-            --tests ReadFile,WriteFile \
-            --json
+          ./run agent start ${{ matrix.agent-name }}
+          cd ${{ matrix.agent-name }}
+          poetry run agbenchmark --mock --test=BasicRetrieval --test=Battleship --test=WebArenaTask_0
+          poetry run agbenchmark --test=WriteFile
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          AGENT_NAME: ${{ matrix.agent-name }}
           REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          NONINTERACTIVE_MODE: "true"
-          CI: true
+          HELICONE_CACHE_ENABLED: false
+          HELICONE_PROPERTY_AGENT: ${{ matrix.agent-name }}
+          REPORTS_FOLDER: ${{ format('../../reports/{0}', matrix.agent-name) }}
+          TELEMETRY_ENVIRONMENT: autogpt-ci
+          TELEMETRY_OPT_IN: ${{ github.ref_name == 'master' }}

.github/workflows/classic-benchmark-ci.yml

@@ -1,21 +1,17 @@
-name: Classic - Direct Benchmark CI
+name: Classic - AGBenchmark CI
 
 on:
   push:
     branches: [ master, dev, ci-test* ]
     paths:
-      - 'classic/direct_benchmark/**'
-      - 'classic/benchmark/agbenchmark/challenges/**'
-      - 'classic/original_autogpt/**'
-      - 'classic/forge/**'
+      - 'classic/benchmark/**'
+      - '!classic/benchmark/reports/**'
       - .github/workflows/classic-benchmark-ci.yml
   pull_request:
     branches: [ master, dev, release-* ]
     paths:
-      - 'classic/direct_benchmark/**'
-      - 'classic/benchmark/agbenchmark/challenges/**'
-      - 'classic/original_autogpt/**'
-      - 'classic/forge/**'
+      - 'classic/benchmark/**'
+      - '!classic/benchmark/reports/**'
       - .github/workflows/classic-benchmark-ci.yml
 
 concurrency:
@@ -27,16 +23,23 @@ defaults:
     shell: bash
 
 env:
-  min-python-version: '3.12'
+  min-python-version: '3.10'
 
 jobs:
-  benchmark-tests:
-    runs-on: ubuntu-latest
+  test:
+    permissions:
+      contents: read
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        platform-os: [ubuntu, macos, macos-arm64, windows]
+    runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }}
     defaults:
       run:
         shell: bash
-        working-directory: classic
+        working-directory: classic/benchmark
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -44,84 +47,71 @@ jobs:
           fetch-depth: 0
           submodules: true
 
-      - name: Set up Python ${{ env.min-python-version }}
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.min-python-version }}
+          python-version: ${{ matrix.python-version }}
 
       - name: Set up Python dependency cache
+        # On Windows, unpacking cached dependencies takes longer than just installing them
+        if: runner.os != 'Windows'
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('classic/poetry.lock') }}
+          path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/benchmark/poetry.lock') }}
 
-      - name: Install Poetry
+      - name: Install Poetry (Unix)
+        if: runner.os != 'Windows'
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
 
-      - name: Install dependencies
+          if [ "${{ runner.os }}" = "macOS" ]; then
+            PATH="$HOME/.local/bin:$PATH"
+            echo "$HOME/.local/bin" >> $GITHUB_PATH
+          fi
+
+      - name: Install Poetry (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
+
+          $env:PATH += ";$env:APPDATA\Python\Scripts"
+          echo "$env:APPDATA\Python\Scripts" >> $env:GITHUB_PATH
+
+      - name: Install Python dependencies
         run: poetry install
 
-      - name: Run basic benchmark tests
+      - name: Run pytest with coverage
         run: |
-          echo "Testing ReadFile challenge with one_shot strategy..."
-          poetry run direct-benchmark run \
-            --strategies one_shot \
-            --models claude \
-            --tests ReadFile \
-            --json
-
-          echo "Testing WriteFile challenge..."
-          poetry run direct-benchmark run \
-            --strategies one_shot \
-            --models claude \
-            --tests WriteFile \
-            --json
+          poetry run pytest -vv \
+            --cov=agbenchmark --cov-branch --cov-report term-missing --cov-report xml \
+            --durations=10 \
+            --junitxml=junit.xml -o junit_family=legacy \
+            tests
         env:
           CI: true
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          NONINTERACTIVE_MODE: "true"
 
-      - name: Test category filtering
-        run: |
-          echo "Testing coding category..."
-          poetry run direct-benchmark run \
-            --strategies one_shot \
-            --models claude \
-            --categories coding \
-            --tests ReadFile,WriteFile \
-            --json
-        env:
-          CI: true
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          NONINTERACTIVE_MODE: "true"
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}  # Run even if tests fail
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
-      - name: Test multiple strategies
-        run: |
-          echo "Testing multiple strategies..."
-          poetry run direct-benchmark run \
-            --strategies one_shot,plan_execute \
-            --models claude \
-            --tests ReadFile \
-            --parallel 2 \
-            --json
-        env:
-          CI: true
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          NONINTERACTIVE_MODE: "true"
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: agbenchmark,${{ runner.os }}
 
-  # Run regression tests on maintain challenges
-  regression-tests:
+  self-test-with-agent:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
-    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev'
-    defaults:
-      run:
-        shell: bash
-        working-directory: classic
+    strategy:
+      matrix:
+        agent-name: [forge]
+      fail-fast: false
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -136,22 +126,51 @@ jobs:
 
       - name: Install Poetry
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-
-      - name: Install dependencies
-        run: poetry install
+          curl -sSL https://install.python-poetry.org | python -
 
       - name: Run regression tests
+        working-directory: classic
         run: |
-          echo "Running regression tests (previously beaten challenges)..."
-          poetry run direct-benchmark run \
-            --strategies one_shot \
-            --models claude \
-            --maintain \
-            --parallel 4 \
-            --json
+          ./run agent start ${{ matrix.agent-name }}
+          cd ${{ matrix.agent-name }}
+
+          set +e # Ignore non-zero exit codes and continue execution
+          echo "Running the following command: poetry run agbenchmark --maintain --mock"
+          poetry run agbenchmark --maintain --mock
+          EXIT_CODE=$?
+          set -e  # Stop ignoring non-zero exit codes
+          # Check if the exit code was 5, and if so, exit with 0 instead
+          if [ $EXIT_CODE -eq 5 ]; then
+            echo "regression_tests.json is empty."
+          fi
+
+          echo "Running the following command: poetry run agbenchmark --mock"
+          poetry run agbenchmark --mock
+
+          echo "Running the following command: poetry run agbenchmark --mock --category=data"
+          poetry run agbenchmark --mock --category=data
+
+          echo "Running the following command: poetry run agbenchmark --mock --category=coding"
+          poetry run agbenchmark --mock --category=coding
+
+          # echo "Running the following command: poetry run agbenchmark --test=WriteFile"
+          # poetry run agbenchmark --test=WriteFile
+          cd ../benchmark
+          poetry install
+          echo "Adding the BUILD_SKILL_TREE environment variable. This will attempt to add new elements in the skill tree. If new elements are added, the CI fails because they should have been pushed"
+          export BUILD_SKILL_TREE=true
+
+          # poetry run agbenchmark --mock
+
+          # CHANGED=$(git diff --name-only | grep -E '(agbenchmark/challenges)|(../classic/frontend/assets)') || echo "No diffs"
+          # if [ ! -z "$CHANGED" ]; then
+          #   echo "There are unstaged changes please run agbenchmark and commit those changes since they are needed."
+          #   echo "$CHANGED"
+          #   exit 1
+          # else
+          #   echo "No unstaged changes."
+          # fi
         env:
-          CI: true
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          NONINTERACTIVE_MODE: "true"
+          TELEMETRY_ENVIRONMENT: autogpt-benchmark-ci
+          TELEMETRY_OPT_IN: ${{ github.ref_name == 'master' }}

.github/workflows/classic-forge-ci.yml

@@ -6,11 +6,13 @@ on:
     paths:
       - '.github/workflows/classic-forge-ci.yml'
       - 'classic/forge/**'
+      - '!classic/forge/tests/vcr_cassettes'
   pull_request:
     branches: [ master, dev, release-* ]
     paths:
       - '.github/workflows/classic-forge-ci.yml'
       - 'classic/forge/**'
+      - '!classic/forge/tests/vcr_cassettes'
 
 concurrency:
   group: ${{ format('forge-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
@@ -19,38 +21,115 @@ concurrency:
 defaults:
   run:
     shell: bash
-    working-directory: classic
+    working-directory: classic/forge
 
 jobs:
   test:
     permissions:
       contents: read
     timeout-minutes: 30
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        platform-os: [ubuntu, macos, macos-arm64, windows]
+    runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }}
 
     steps:
-      - name: Start MinIO service
+      # Quite slow on macOS (2~4 minutes to set up Docker)
+      # - name: Set up Docker (macOS)
+      #   if: runner.os == 'macOS'
+      #   uses: crazy-max/ghaction-setup-docker@v3
+
+      - name: Start MinIO service (Linux)
+        if: runner.os == 'Linux'
         working-directory: '.'
         run: |
           docker pull minio/minio:edge-cicd
           docker run -d -p 9000:9000 minio/minio:edge-cicd
 
+      - name: Start MinIO service (macOS)
+        if: runner.os == 'macOS'
+        working-directory: ${{ runner.temp }}
+        run: |
+          brew install minio/stable/minio
+          mkdir data
+          minio server ./data &
+
+      # No MinIO on Windows:
+      # - Windows doesn't support running Linux Docker containers
+      # - It doesn't seem possible to start background processes on Windows. They are
+      #   killed after the step returns.
+      #   See: https://github.com/actions/runner/issues/598#issuecomment-2011890429
+
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
 
-      - name: Set up Python 3.12
+      - name: Checkout cassettes
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        env:
+          PR_BASE: ${{ github.event.pull_request.base.ref }}
+          PR_BRANCH: ${{ github.event.pull_request.head.ref }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          cassette_branch="${PR_AUTHOR}-${PR_BRANCH}"
+          cassette_base_branch="${PR_BASE}"
+          cd tests/vcr_cassettes
+
+          if ! git ls-remote --exit-code --heads origin $cassette_base_branch ; then
+            cassette_base_branch="master"
+          fi
+
+          if git ls-remote --exit-code --heads origin $cassette_branch ; then
+            git fetch origin $cassette_branch
+            git fetch origin $cassette_base_branch
+
+            git checkout $cassette_branch
+
+            # Pick non-conflicting cassette updates from the base branch
+            git merge --no-commit --strategy-option=ours origin/$cassette_base_branch
+            echo "Using cassettes from mirror branch '$cassette_branch'," \
+              "synced to upstream branch '$cassette_base_branch'."
+          else
+            git checkout -b $cassette_branch
+            echo "Branch '$cassette_branch' does not exist in cassette submodule." \
+              "Using cassettes from '$cassette_base_branch'."
+          fi
+
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
 
       - name: Set up Python dependency cache
+        # On Windows, unpacking cached dependencies takes longer than just installing them
+        if: runner.os != 'Windows'
         uses: actions/cache@v4
         with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('classic/poetry.lock') }}
+          path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/forge/poetry.lock') }}
 
-      - name: Install Poetry
-        run: curl -sSL https://install.python-poetry.org | python3 -
+      - name: Install Poetry (Unix)
+        if: runner.os != 'Windows'
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+          if [ "${{ runner.os }}" = "macOS" ]; then
+            PATH="$HOME/.local/bin:$PATH"
+            echo "$HOME/.local/bin" >> $GITHUB_PATH
+          fi
+
+      - name: Install Poetry (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
+
+          $env:PATH += ";$env:APPDATA\Python\Scripts"
+          echo "$env:APPDATA\Python\Scripts" >> $env:GITHUB_PATH
 
       - name: Install Python dependencies
         run: poetry install
@@ -61,15 +140,12 @@ jobs:
             --cov=forge --cov-branch --cov-report term-missing --cov-report xml \
             --durations=10 \
             --junitxml=junit.xml -o junit_family=legacy \
-            forge/forge forge/tests
+            forge
         env:
           CI: true
           PLAIN_OUTPUT: True
-          # API keys - tests that need these will skip if not available
-          # Secrets are not available to fork PRs (GitHub security feature)
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          S3_ENDPOINT_URL: http://127.0.0.1:9000
+          S3_ENDPOINT_URL: ${{ runner.os != 'Windows' && 'http://127.0.0.1:9000' || '' }}
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
@@ -83,11 +159,85 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          flags: forge
+          flags: forge,${{ runner.os }}
+
+      - id: setup_git_auth
+        name: Set up git token authentication
+        # Cassettes may be pushed even when tests fail
+        if: success() || failure()
+        run: |
+          config_key="http.${{ github.server_url }}/.extraheader"
+          if [ "${{ runner.os }}" = 'macOS' ]; then
+            base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64)
+          else
+            base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0)
+          fi
+
+          git config "$config_key" \
+            "Authorization: Basic $base64_pat"
+
+          cd tests/vcr_cassettes
+          git config "$config_key" \
+            "Authorization: Basic $base64_pat"
+
+          echo "config_key=$config_key" >> $GITHUB_OUTPUT
+
+      - id: push_cassettes
+        name: Push updated cassettes
+        # For pull requests, push updated cassettes even when tests fail
+        if: github.event_name == 'push' || (! github.event.pull_request.head.repo.fork && (success() || failure()))
+        env:
+          PR_BRANCH: ${{ github.event.pull_request.head.ref }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then
+            is_pull_request=true
+            cassette_branch="${PR_AUTHOR}-${PR_BRANCH}"
+          else
+            cassette_branch="${{ github.ref_name }}"
+          fi
+
+          cd tests/vcr_cassettes
+          # Commit & push changes to cassettes if any
+          if ! git diff --quiet; then
+            git add .
+            git commit -m "Auto-update cassettes"
+            git push origin HEAD:$cassette_branch
+            if [ ! $is_pull_request ]; then
+              cd ../..
+              git add tests/vcr_cassettes
+              git commit -m "Update cassette submodule"
+              git push origin HEAD:$cassette_branch
+            fi
+            echo "updated=true" >> $GITHUB_OUTPUT
+          else
+            echo "updated=false" >> $GITHUB_OUTPUT
+            echo "No cassette changes to commit"
+          fi
+
+      - name: Post Set up git token auth
+        if: steps.setup_git_auth.outcome == 'success'
+        run: |
+          git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
+          git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
+
+      - name: Apply "behaviour change" label and comment on PR
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        run: |
+          PR_NUMBER="${{ github.event.pull_request.number }}"
+          TOKEN="${{ secrets.PAT_REVIEW }}"
+          REPO="${{ github.repository }}"
+
+          if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then
+            echo "Adding label and comment..."
+            echo $TOKEN | gh auth login --with-token
+            gh issue edit $PR_NUMBER --add-label "behaviour change"
+            gh issue comment $PR_NUMBER --body "You changed AutoGPT's behaviour on ${{ runner.os }}. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
+          fi
 
       - name: Upload logs to artifact
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: test-logs
-          path: classic/logs/
+          path: classic/forge/logs/

.github/workflows/classic-frontend-ci.yml

@@ -0,0 +1,60 @@
+name: Classic - Frontend CI/CD
+
+on:
+  push:
+    branches:
+      - master
+      - dev
+      - 'ci-test*' # This will match any branch that starts with "ci-test"
+    paths:
+      - 'classic/frontend/**'
+      - '.github/workflows/classic-frontend-ci.yml'
+  pull_request:
+    paths:
+      - 'classic/frontend/**'
+      - '.github/workflows/classic-frontend-ci.yml'
+
+jobs:
+  build:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    env:
+      BUILD_BRANCH: ${{ format('classic-frontend-build/{0}', github.ref_name) }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Setup Flutter
+        uses: subosito/flutter-action@v2
+        with:
+          flutter-version: '3.13.2'
+
+      - name: Build Flutter to Web
+        run: |
+          cd classic/frontend
+          flutter build web --base-href /app/
+
+      # - name: Commit and Push to ${{ env.BUILD_BRANCH }}
+      #   if: github.event_name == 'push'
+      #   run: |
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git add classic/frontend/build/web
+      #     git checkout -B ${{ env.BUILD_BRANCH }}
+      #     git commit -m "Update frontend build to ${GITHUB_SHA:0:7}" -a
+      #     git push -f origin ${{ env.BUILD_BRANCH }}
+
+      - name: Create PR ${{ env.BUILD_BRANCH }} -> ${{ github.ref_name }}
+        if: github.event_name == 'push'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          add-paths: classic/frontend/build/web
+          base: ${{ github.ref_name }}
+          branch: ${{ env.BUILD_BRANCH }}
+          delete-branch: true
+          title: "Update frontend build in `${{ github.ref_name }}`"
+          body: "This PR updates the frontend build based on commit ${{ github.sha }}."
+          commit-message: "Update frontend build based on commit ${{ github.sha }}"

.github/workflows/classic-python-checks.yml

@@ -7,9 +7,7 @@ on:
       - '.github/workflows/classic-python-checks-ci.yml'
       - 'classic/original_autogpt/**'
       - 'classic/forge/**'
-      - 'classic/direct_benchmark/**'
-      - 'classic/pyproject.toml'
-      - 'classic/poetry.lock'
+      - 'classic/benchmark/**'
       - '**.py'
       - '!classic/forge/tests/vcr_cassettes'
   pull_request:
@@ -18,9 +16,7 @@ on:
       - '.github/workflows/classic-python-checks-ci.yml'
       - 'classic/original_autogpt/**'
       - 'classic/forge/**'
-      - 'classic/direct_benchmark/**'
-      - 'classic/pyproject.toml'
-      - 'classic/poetry.lock'
+      - 'classic/benchmark/**'
       - '**.py'
       - '!classic/forge/tests/vcr_cassettes'
 
@@ -31,13 +27,44 @@ concurrency:
 defaults:
   run:
     shell: bash
-    working-directory: classic
 
 jobs:
+  get-changed-parts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - id: changes-in
+        name: Determine affected subprojects
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            original_autogpt:
+              - classic/original_autogpt/autogpt/**
+              - classic/original_autogpt/tests/**
+              - classic/original_autogpt/poetry.lock
+            forge:
+              - classic/forge/forge/**
+              - classic/forge/tests/**
+              - classic/forge/poetry.lock
+            benchmark:
+              - classic/benchmark/agbenchmark/**
+              - classic/benchmark/tests/**
+              - classic/benchmark/poetry.lock
+    outputs:
+      changed-parts: ${{ steps.changes-in.outputs.changes }}
+
   lint:
+    needs: get-changed-parts
     runs-on: ubuntu-latest
     env:
-      min-python-version: "3.12"
+      min-python-version: "3.10"
+
+    strategy:
+      matrix:
+        sub-package: ${{ fromJson(needs.get-changed-parts.outputs.changed-parts) }}
+      fail-fast: false
 
     steps:
       - name: Checkout repository
@@ -54,31 +81,42 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-${{ hashFiles('classic/poetry.lock') }}
+          key: ${{ runner.os }}-poetry-${{ hashFiles(format('{0}/poetry.lock', matrix.sub-package)) }}
 
       - name: Install Poetry
         run: curl -sSL https://install.python-poetry.org | python3 -
 
+      # Install dependencies
+
       - name: Install Python dependencies
-        run: poetry install
+        run: poetry -C classic/${{ matrix.sub-package }} install
 
       # Lint
 
       - name: Lint (isort)
         run: poetry run isort --check .
+        working-directory: classic/${{ matrix.sub-package }}
 
       - name: Lint (Black)
         if: success() || failure()
         run: poetry run black --check .
+        working-directory: classic/${{ matrix.sub-package }}
 
       - name: Lint (Flake8)
         if: success() || failure()
         run: poetry run flake8 .
+        working-directory: classic/${{ matrix.sub-package }}
 
   types:
+    needs: get-changed-parts
     runs-on: ubuntu-latest
     env:
-      min-python-version: "3.12"
+      min-python-version: "3.10"
+
+    strategy:
+      matrix:
+        sub-package: ${{ fromJson(needs.get-changed-parts.outputs.changed-parts) }}
+      fail-fast: false
 
     steps:
       - name: Checkout repository
@@ -95,16 +133,19 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-${{ hashFiles('classic/poetry.lock') }}
+          key: ${{ runner.os }}-poetry-${{ hashFiles(format('{0}/poetry.lock', matrix.sub-package)) }}
 
       - name: Install Poetry
         run: curl -sSL https://install.python-poetry.org | python3 -
 
+      # Install dependencies
+
       - name: Install Python dependencies
-        run: poetry install
+        run: poetry -C classic/${{ matrix.sub-package }} install
 
       # Typecheck
 
       - name: Typecheck
         if: success() || failure()
         run: poetry run pyright
+        working-directory: classic/${{ matrix.sub-package }}

.github/workflows/claude-ci-failure-auto-fix.yml

@@ -93,5 +93,5 @@ jobs:
 
             Error logs:
             ${{ toJSON(fromJSON(steps.failure_details.outputs.result).errorLogs) }}
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           claude_args: "--allowedTools 'Edit,MultiEdit,Write,Read,Glob,Grep,LS,Bash(git:*),Bash(bun:*),Bash(npm:*),Bash(npx:*),Bash(gh:*)'"

.github/workflows/claude-dependabot.yml

@@ -7,7 +7,7 @@
 # - Provide actionable recommendations for the development team
 #
 # Triggered on: Dependabot PRs (opened, synchronize)
-# Requirements: CLAUDE_CODE_OAUTH_TOKEN secret must be configured
+# Requirements: ANTHROPIC_API_KEY secret must be configured
 
 name: Claude Dependabot PR Review
 
@@ -74,7 +74,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate && poetry run gen-prisma-stub
+        run: poetry run prisma generate
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js
@@ -308,7 +308,7 @@ jobs:
         id: claude_review
         uses: anthropics/claude-code-action@v1
         with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           claude_args: |
             --allowedTools "Bash(npm:*),Bash(pnpm:*),Bash(poetry:*),Bash(git:*),Edit,Replace,NotebookEditCell,mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*), Bash(gh pr diff:*), Bash(gh pr view:*)"
           prompt: |

.github/workflows/claude.yml

@@ -90,7 +90,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate && poetry run gen-prisma-stub
+        run: poetry run prisma generate
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js
@@ -323,7 +323,7 @@ jobs:
         id: claude
         uses: anthropics/claude-code-action@v1
         with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           claude_args: |
             --allowedTools "Bash(npm:*),Bash(pnpm:*),Bash(poetry:*),Bash(git:*),Edit,Replace,NotebookEditCell,mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*), Bash(gh pr diff:*), Bash(gh pr view:*), Bash(gh pr edit:*)"
             --model opus

.github/workflows/copilot-setup-steps.yml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate && poetry run gen-prisma-stub
+        run: poetry run prisma generate
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js
@@ -108,16 +108,6 @@ jobs:
       #   run: pnpm playwright install --with-deps chromium
 
       # Docker setup for development environment
-      - name: Free up disk space
-        run: |
-          # Remove large unused tools to free disk space for Docker builds
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo docker system prune -af
-          df -h
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 

.github/workflows/docs-block-sync.yml

@@ -1,78 +0,0 @@
-name: Block Documentation Sync Check
-
-on:
-  push:
-    branches: [master, dev]
-    paths:
-      - "autogpt_platform/backend/backend/blocks/**"
-      - "docs/integrations/**"
-      - "autogpt_platform/backend/scripts/generate_block_docs.py"
-      - ".github/workflows/docs-block-sync.yml"
-  pull_request:
-    branches: [master, dev]
-    paths:
-      - "autogpt_platform/backend/backend/blocks/**"
-      - "docs/integrations/**"
-      - "autogpt_platform/backend/scripts/generate_block_docs.py"
-      - ".github/workflows/docs-block-sync.yml"
-
-jobs:
-  check-docs-sync:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Set up Python dependency cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
-          restore-keys: |
-            poetry-${{ runner.os }}-
-
-      - name: Install Poetry
-        run: |
-          cd autogpt_platform/backend
-          HEAD_POETRY_VERSION=$(python3 ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
-          echo "Found Poetry version ${HEAD_POETRY_VERSION} in backend/poetry.lock"
-          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-      - name: Install dependencies
-        working-directory: autogpt_platform/backend
-        run: |
-          poetry install --only main
-          poetry run prisma generate
-
-      - name: Check block documentation is in sync
-        working-directory: autogpt_platform/backend
-        run: |
-          echo "Checking if block documentation is in sync with code..."
-          poetry run python scripts/generate_block_docs.py --check
-
-      - name: Show diff if out of sync
-        if: failure()
-        working-directory: autogpt_platform/backend
-        run: |
-          echo "::error::Block documentation is out of sync with code!"
-          echo ""
-          echo "To fix this, run the following command locally:"
-          echo "  cd autogpt_platform/backend && poetry run python scripts/generate_block_docs.py"
-          echo ""
-          echo "Then commit the updated documentation files."
-          echo ""
-          echo "Regenerating docs to show diff..."
-          poetry run python scripts/generate_block_docs.py
-          echo ""
-          echo "Changes detected:"
-          git diff ../../docs/integrations/ || true

.github/workflows/docs-claude-review.yml

@@ -1,95 +0,0 @@
-name: Claude Block Docs Review
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-    paths:
-      - "docs/integrations/**"
-      - "autogpt_platform/backend/backend/blocks/**"
-
-jobs:
-  claude-review:
-    # Only run for PRs from members/collaborators
-    if: |
-      github.event.pull_request.author_association == 'OWNER' ||
-      github.event.pull_request.author_association == 'MEMBER' ||
-      github.event.pull_request.author_association == 'COLLABORATOR'
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    permissions:
-      contents: read
-      pull-requests: write
-      id-token: write
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Set up Python dependency cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
-          restore-keys: |
-            poetry-${{ runner.os }}-
-
-      - name: Install Poetry
-        run: |
-          cd autogpt_platform/backend
-          HEAD_POETRY_VERSION=$(python3 ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
-          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-      - name: Install dependencies
-        working-directory: autogpt_platform/backend
-        run: |
-          poetry install --only main
-          poetry run prisma generate
-
-      - name: Run Claude Code Review
-        uses: anthropics/claude-code-action@v1
-        with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-          claude_args: |
-            --allowedTools "Read,Glob,Grep,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)"
-          prompt: |
-            You are reviewing a PR that modifies block documentation or block code for AutoGPT.
-
-            ## Your Task
-            Review the changes in this PR and provide constructive feedback. Focus on:
-
-            1. **Documentation Accuracy**: For any block code changes, verify that:
-               - Input/output tables in docs match the actual block schemas
-               - Description text accurately reflects what the block does
-               - Any new blocks have corresponding documentation
-
-            2. **Manual Content Quality**: Check manual sections (marked with `<!-- MANUAL: -->` markers):
-               - "How it works" sections should have clear technical explanations
-               - "Possible use case" sections should have practical, real-world examples
-               - Content should be helpful for users trying to understand the blocks
-
-            3. **Template Compliance**: Ensure docs follow the standard template:
-               - What it is (brief intro)
-               - What it does (description)
-               - How it works (technical explanation)
-               - Inputs table
-               - Outputs table
-               - Possible use case
-
-            4. **Cross-references**: Check that links and anchors are correct
-
-            ## Review Process
-            1. First, get the PR diff to see what changed: `gh pr diff ${{ github.event.pull_request.number }}`
-            2. Read any modified block files to understand the implementation
-            3. Read corresponding documentation files to verify accuracy
-            4. Provide your feedback as a PR comment
-
-            Be constructive and specific. If everything looks good, say so!
-            If there are issues, explain what's wrong and suggest how to fix it.

.github/workflows/docs-enhance.yml

@@ -1,194 +0,0 @@
-name: Enhance Block Documentation
-
-on:
-  workflow_dispatch:
-    inputs:
-      block_pattern:
-        description: 'Block file pattern to enhance (e.g., "google/*.md" or "*" for all blocks)'
-        required: true
-        default: '*'
-        type: string
-      dry_run:
-        description: 'Dry run mode - show proposed changes without committing'
-        type: boolean
-        default: true
-      max_blocks:
-        description: 'Maximum number of blocks to process (0 for unlimited)'
-        type: number
-        default: 10
-
-jobs:
-  enhance-docs:
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    permissions:
-      contents: write
-      pull-requests: write
-      id-token: write
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - name: Set up Python dependency cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
-          restore-keys: |
-            poetry-${{ runner.os }}-
-
-      - name: Install Poetry
-        run: |
-          cd autogpt_platform/backend
-          HEAD_POETRY_VERSION=$(python3 ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
-          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-      - name: Install dependencies
-        working-directory: autogpt_platform/backend
-        run: |
-          poetry install --only main
-          poetry run prisma generate
-
-      - name: Run Claude Enhancement
-        uses: anthropics/claude-code-action@v1
-        with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-          claude_args: |
-            --allowedTools "Read,Edit,Glob,Grep,Write,Bash(git:*),Bash(gh:*),Bash(find:*),Bash(ls:*)"
-          prompt: |
-            You are enhancing block documentation for AutoGPT. Your task is to improve the MANUAL sections
-            of block documentation files by reading the actual block implementations and writing helpful content.
-
-            ## Configuration
-            - Block pattern: ${{ inputs.block_pattern }}
-            - Dry run: ${{ inputs.dry_run }}
-            - Max blocks to process: ${{ inputs.max_blocks }}
-
-            ## Your Task
-
-            1. **Find Documentation Files**
-               Find block documentation files matching the pattern in `docs/integrations/`
-               Pattern: ${{ inputs.block_pattern }}
-
-               Use: `find docs/integrations -name "*.md" -type f`
-
-            2. **For Each Documentation File** (up to ${{ inputs.max_blocks }} files):
-
-               a. Read the documentation file
-
-               b. Identify which block(s) it documents (look for the block class name)
-
-               c. Find and read the corresponding block implementation in `autogpt_platform/backend/backend/blocks/`
-
-               d. Improve the MANUAL sections:
-
-                  **"How it works" section** (within `<!-- MANUAL: how_it_works -->` markers):
-                  - Explain the technical flow of the block
-                  - Describe what APIs or services it connects to
-                  - Note any important configuration or prerequisites
-                  - Keep it concise but informative (2-4 paragraphs)
-
-                  **"Possible use case" section** (within `<!-- MANUAL: use_case -->` markers):
-                  - Provide 2-3 practical, real-world examples
-                  - Make them specific and actionable
-                  - Show how this block could be used in an automation workflow
-
-            3. **Important Rules**
-               - ONLY modify content within `<!-- MANUAL: -->` and `<!-- END MANUAL -->` markers
-               - Do NOT modify auto-generated sections (inputs/outputs tables, descriptions)
-               - Keep content accurate based on the actual block implementation
-               - Write for users who may not be technical experts
-
-            4. **Output**
-               ${{ inputs.dry_run == true && 'DRY RUN MODE: Show proposed changes for each file but do NOT actually edit the files. Describe what you would change.' || 'LIVE MODE: Actually edit the files to improve the documentation.' }}
-
-            ## Example Improvements
-
-            **Before (How it works):**
-            ```
-            _Add technical explanation here._
-            ```
-
-            **After (How it works):**
-            ```
-            This block connects to the GitHub API to retrieve issue information. When executed,
-            it authenticates using your GitHub credentials and fetches issue details including
-            title, body, labels, and assignees.
-
-            The block requires a valid GitHub OAuth connection with repository access permissions.
-            It supports both public and private repositories you have access to.
-            ```
-
-            **Before (Possible use case):**
-            ```
-            _Add practical use case examples here._
-            ```
-
-            **After (Possible use case):**
-            ```
-            **Customer Support Automation**: Monitor a GitHub repository for new issues with
-            the "bug" label, then automatically create a ticket in your support system and
-            notify the on-call engineer via Slack.
-
-            **Release Notes Generation**: When a new release is published, gather all closed
-            issues since the last release and generate a summary for your changelog.
-            ```
-
-            Begin by finding and listing the documentation files to process.
-
-      - name: Create PR with enhanced documentation
-        if: ${{ inputs.dry_run == false }}
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Check if there are changes
-          if git diff --quiet docs/integrations/; then
-            echo "No changes to commit"
-            exit 0
-          fi
-
-          # Configure git
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Create branch and commit
-          BRANCH_NAME="docs/enhance-blocks-$(date +%Y%m%d-%H%M%S)"
-          git checkout -b "$BRANCH_NAME"
-          git add docs/integrations/
-          git commit -m "docs: enhance block documentation with LLM-generated content
-
-          Pattern: ${{ inputs.block_pattern }}
-          Max blocks: ${{ inputs.max_blocks }}
-
-          🤖 Generated with [Claude Code](https://claude.com/claude-code)
-
-          Co-Authored-By: Claude <noreply@anthropic.com>"
-
-          # Push and create PR
-          git push -u origin "$BRANCH_NAME"
-          gh pr create \
-            --title "docs: LLM-enhanced block documentation" \
-            --body "## Summary
-          This PR contains LLM-enhanced documentation for block files matching pattern: \`${{ inputs.block_pattern }}\`
-
-          The following manual sections were improved:
-          - **How it works**: Technical explanations based on block implementations
-          - **Possible use case**: Practical, real-world examples
-
-          ## Review Checklist
-          - [ ] Content is accurate based on block implementations
-          - [ ] Examples are practical and helpful
-          - [ ] No auto-generated sections were modified
-
-          ---
-          🤖 Generated with [Claude Code](https://claude.com/claude-code)" \
-            --base dev

.github/workflows/platform-backend-ci.yml

@@ -134,7 +134,7 @@ jobs:
         run: poetry install
 
       - name: Generate Prisma Client
-        run: poetry run prisma generate && poetry run gen-prisma-stub
+        run: poetry run prisma generate
 
       - id: supabase
         name: Start Supabase
@@ -176,7 +176,7 @@ jobs:
           }
 
       - name: Run Database Migrations
-        run: poetry run prisma migrate deploy
+        run: poetry run prisma migrate dev --name updates
         env:
           DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
           DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}

.github/workflows/platform-frontend-ci.yml

@@ -11,7 +11,6 @@ on:
       - ".github/workflows/platform-frontend-ci.yml"
       - "autogpt_platform/frontend/**"
   merge_group:
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name == 'merge_group' && format('merge-queue-{0}', github.ref) || format('{0}-{1}', github.ref, github.event.pull_request.number || github.sha) }}
@@ -152,14 +151,6 @@ jobs:
         run: |
           cp ../.env.default ../.env
 
-      - name: Copy backend .env and set OpenAI API key
-        run: |
-          cp ../backend/.env.default ../backend/.env
-          echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env
-        env:
-          # Used by E2E test data script to generate embeddings for approved store agents
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -235,25 +226,13 @@ jobs:
 
       - name: Run Playwright tests
         run: pnpm test:no-build
-        continue-on-error: false
 
-      - name: Upload Playwright report
-        if: always()
+      - name: Upload Playwright artifacts
+        if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: playwright-report
           path: playwright-report
-          if-no-files-found: ignore
-          retention-days: 3
-
-      - name: Upload Playwright test results
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: playwright-test-results
-          path: test-results
-          if-no-files-found: ignore
-          retention-days: 3
 
       - name: Print Final Docker Compose logs
         if: always()

.gitignore

@@ -3,7 +3,6 @@
 classic/original_autogpt/keys.py
 classic/original_autogpt/*.json
 auto_gpt_workspace/*
-.autogpt/
 *.mpeg
 .env
 # Root .env files
@@ -178,5 +177,5 @@ autogpt_platform/backend/settings.py
 
 *.ign.*
 .test-contents
-**/.claude/settings.local.json
+.claude/settings.local.json
 /autogpt_platform/backend/logs

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "classic/forge/tests/vcr_cassettes"]
+	path = classic/forge/tests/vcr_cassettes
+	url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes

.pre-commit-config.yaml

@@ -43,10 +43,29 @@ repos:
         pass_filenames: false
 
       - id: poetry-install
-        name: Check & Install dependencies - Classic
-        alias: poetry-install-classic
-        entry: poetry -C classic install
-        files: ^classic/poetry\.lock$
+        name: Check & Install dependencies - Classic - AutoGPT
+        alias: poetry-install-classic-autogpt
+        entry: poetry -C classic/original_autogpt install
+        # include forge source (since it's a path dependency)
+        files: ^classic/(original_autogpt|forge)/poetry\.lock$
+        types: [file]
+        language: system
+        pass_filenames: false
+
+      - id: poetry-install
+        name: Check & Install dependencies - Classic - Forge
+        alias: poetry-install-classic-forge
+        entry: poetry -C classic/forge install
+        files: ^classic/forge/poetry\.lock$
+        types: [file]
+        language: system
+        pass_filenames: false
+
+      - id: poetry-install
+        name: Check & Install dependencies - Classic - Benchmark
+        alias: poetry-install-classic-benchmark
+        entry: poetry -C classic/benchmark install
+        files: ^classic/benchmark/poetry\.lock$
         types: [file]
         language: system
         pass_filenames: false
@@ -97,10 +116,26 @@ repos:
         language: system
 
       - id: isort
-        name: Lint (isort) - Classic
-        alias: isort-classic
-        entry: bash -c 'cd classic && poetry run isort $(echo "$@" | sed "s|classic/||g")' --
-        files: ^classic/(original_autogpt|forge|direct_benchmark)/
+        name: Lint (isort) - Classic - AutoGPT
+        alias: isort-classic-autogpt
+        entry: poetry -P classic/original_autogpt run isort -p autogpt
+        files: ^classic/original_autogpt/
+        types: [file, python]
+        language: system
+
+      - id: isort
+        name: Lint (isort) - Classic - Forge
+        alias: isort-classic-forge
+        entry: poetry -P classic/forge run isort -p forge
+        files: ^classic/forge/
+        types: [file, python]
+        language: system
+
+      - id: isort
+        name: Lint (isort) - Classic - Benchmark
+        alias: isort-classic-benchmark
+        entry: poetry -P classic/benchmark run isort -p agbenchmark
+        files: ^classic/benchmark/
         types: [file, python]
         language: system
 
@@ -114,13 +149,26 @@ repos:
 
   - repo: https://github.com/PyCQA/flake8
     rev: 7.0.0
-    # Use consolidated flake8 config at classic/.flake8
+    # To have flake8 load the config of the individual subprojects, we have to call
+    # them separately.
     hooks:
       - id: flake8
-        name: Lint (Flake8) - Classic
-        alias: flake8-classic
-        files: ^classic/(original_autogpt|forge|direct_benchmark)/
-        args: [--config=classic/.flake8]
+        name: Lint (Flake8) - Classic - AutoGPT
+        alias: flake8-classic-autogpt
+        files: ^classic/original_autogpt/(autogpt|scripts|tests)/
+        args: [--config=classic/original_autogpt/.flake8]
+
+      - id: flake8
+        name: Lint (Flake8) - Classic - Forge
+        alias: flake8-classic-forge
+        files: ^classic/forge/(forge|tests)/
+        args: [--config=classic/forge/.flake8]
+
+      - id: flake8
+        name: Lint (Flake8) - Classic - Benchmark
+        alias: flake8-classic-benchmark
+        files: ^classic/benchmark/(agbenchmark|tests)/((?!reports).)*[/.]
+        args: [--config=classic/benchmark/.flake8]
 
   - repo: local
     hooks:
@@ -156,10 +204,29 @@ repos:
         pass_filenames: false
 
       - id: pyright
-        name: Typecheck - Classic
-        alias: pyright-classic
-        entry: poetry -C classic run pyright
-        files: ^classic/(original_autogpt|forge|direct_benchmark)/.*\.py$|^classic/poetry\.lock$
+        name: Typecheck - Classic - AutoGPT
+        alias: pyright-classic-autogpt
+        entry: poetry -C classic/original_autogpt run pyright
+        # include forge source (since it's a path dependency) but exclude *_test.py files:
+        files: ^(classic/original_autogpt/((autogpt|scripts|tests)/|poetry\.lock$)|classic/forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
+        types: [file]
+        language: system
+        pass_filenames: false
+
+      - id: pyright
+        name: Typecheck - Classic - Forge
+        alias: pyright-classic-forge
+        entry: poetry -C classic/forge run pyright
+        files: ^classic/forge/(forge/|poetry\.lock$)
+        types: [file]
+        language: system
+        pass_filenames: false
+
+      - id: pyright
+        name: Typecheck - Classic - Benchmark
+        alias: pyright-classic-benchmark
+        entry: poetry -C classic/benchmark run pyright
+        files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
         types: [file]
         language: system
         pass_filenames: false

autogpt_platform/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(ls:*)",
+      "WebFetch(domain:langfuse.com)",
+      "Bash(poetry install:*)"
+    ]
+  }
+}

autogpt_platform/Makefile

@@ -1,4 +1,4 @@
-.PHONY: start-core stop-core logs-core format lint migrate run-backend run-frontend load-store-agents
+.PHONY: start-core stop-core logs-core format lint migrate run-backend stop-backend run-frontend load-store-agents backfill-store-embeddings
 
 # Run just Supabase + Redis + RabbitMQ
 start-core:
@@ -6,14 +6,12 @@ start-core:
 
 # Stop core services
 stop-core:
-	docker compose stop 
+	docker compose stop deps
 
 reset-db:
-	docker compose stop db
 	rm -rf db/docker/volumes/db/data
 	cd backend && poetry run prisma migrate deploy
 	cd backend && poetry run prisma generate
-	cd backend && poetry run gen-prisma-stub
 	
 # View logs for core services
 logs-core:
@@ -35,9 +33,15 @@ init-env:
 migrate:
 	cd backend && poetry run prisma migrate deploy
 	cd backend && poetry run prisma generate
-	cd backend && poetry run gen-prisma-stub
 
-run-backend:
+stop-backend:
+	@echo "Stopping backend processes..."
+	@cd backend && poetry run cli stop 2>/dev/null || true
+	@echo "Killing any processes using backend ports..."
+	@lsof -ti:8001,8002,8003,8004,8005,8006,8007 | xargs kill -9 2>/dev/null || true
+	@echo "Backend stopped"
+
+run-backend: stop-backend
 	cd backend && poetry run app
 
 run-frontend:
@@ -49,6 +53,9 @@ test-data:
 load-store-agents:
 	cd backend && poetry run load-store-agents
 
+backfill-store-embeddings:
+	cd backend && poetry run python -m backend.api.features.store.backfill_embeddings
+
 help:
 	@echo "Usage: make <target>"
 	@echo "Targets:"
@@ -58,7 +65,9 @@ help:
 	@echo "  logs-core - Tail the logs for core services"
 	@echo "  format - Format & lint backend (Python) and frontend (TypeScript) code"
 	@echo "  migrate - Run backend database migrations"
-	@echo "  run-backend - Run the backend FastAPI server"
+	@echo "  stop-backend - Stop any running backend processes"
+	@echo "  run-backend - Run the backend FastAPI server (stops existing processes first)"
 	@echo "  run-frontend - Run the frontend Next.js development server"
 	@echo "  test-data - Run the test data creator"
 	@echo "  load-store-agents - Load store agents from agents/ folder into test database"
+	@echo "  backfill-store-embeddings - Generate embeddings for store agents that don't have them"

autogpt_platform/backend/.gitignore

@@ -18,4 +18,3 @@ load-tests/results/
 load-tests/*.json
 load-tests/*.log
 load-tests/node_modules/*
-migrations/*/rollback*.sql

autogpt_platform/backend/Dockerfile

@@ -48,8 +48,7 @@ RUN poetry install --no-ansi --no-root
 # Generate Prisma client
 COPY autogpt_platform/backend/schema.prisma ./
 COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py
-COPY autogpt_platform/backend/gen_prisma_types_stub.py ./
-RUN poetry run prisma generate && poetry run gen-prisma-stub
+RUN poetry run prisma generate
 
 FROM debian:13-slim AS server_dependencies
 
@@ -100,7 +99,6 @@ COPY autogpt_platform/backend/migrations /app/autogpt_platform/backend/migration
 FROM server_dependencies AS server
 
 COPY autogpt_platform/backend /app/autogpt_platform/backend
-COPY docs /app/docs
 RUN poetry install --no-ansi --only-root
 
 ENV PORT=8000

autogpt_platform/backend/backend/api/external/v1/tools.py

@@ -70,7 +70,7 @@ class RunAgentRequest(BaseModel):
     )
 
 
-def _create_ephemeral_session(user_id: str) -> ChatSession:
+def _create_ephemeral_session(user_id: str | None) -> ChatSession:
     """Create an ephemeral session for stateless API requests."""
     return ChatSession.new(user_id)
 

autogpt_platform/backend/backend/api/features/admin/execution_analytics_routes.py

@@ -28,7 +28,6 @@ from backend.executor.manager import get_db_async_client
 from backend.util.settings import Settings
 
 logger = logging.getLogger(__name__)
-settings = Settings()
 
 
 class ExecutionAnalyticsRequest(BaseModel):
@@ -64,8 +63,6 @@ class ExecutionAnalyticsResult(BaseModel):
     score: Optional[float]
     status: str  # "success", "failed", "skipped"
     error_message: Optional[str] = None
-    started_at: Optional[datetime] = None
-    ended_at: Optional[datetime] = None
 
 
 class ExecutionAnalyticsResponse(BaseModel):
@@ -227,6 +224,11 @@ async def generate_execution_analytics(
     )
 
     try:
+        # Validate model configuration
+        settings = Settings()
+        if not settings.secrets.openai_internal_api_key:
+            raise HTTPException(status_code=500, detail="OpenAI API key not configured")
+
         # Get database client
         db_client = get_db_async_client()
 
@@ -318,8 +320,6 @@ async def generate_execution_analytics(
                     ),
                     status="skipped",
                     error_message=None,  # Not an error - just already processed
-                    started_at=execution.started_at,
-                    ended_at=execution.ended_at,
                 )
             )
 
@@ -349,9 +349,6 @@ async def _process_batch(
 ) -> list[ExecutionAnalyticsResult]:
     """Process a batch of executions concurrently."""
 
-    if not settings.secrets.openai_internal_api_key:
-        raise HTTPException(status_code=500, detail="OpenAI API key not configured")
-
     async def process_single_execution(execution) -> ExecutionAnalyticsResult:
         try:
             # Generate activity status and score using the specified model
@@ -390,8 +387,6 @@ async def _process_batch(
                     score=None,
                     status="skipped",
                     error_message="Activity generation returned None",
-                    started_at=execution.started_at,
-                    ended_at=execution.ended_at,
                 )
 
             # Update the execution stats
@@ -421,8 +416,6 @@ async def _process_batch(
                 summary_text=activity_response["activity_status"],
                 score=activity_response["correctness_score"],
                 status="success",
-                started_at=execution.started_at,
-                ended_at=execution.ended_at,
             )
 
         except Exception as e:
@@ -436,8 +429,6 @@ async def _process_batch(
                 score=None,
                 status="failed",
                 error_message=str(e),
-                started_at=execution.started_at,
-                ended_at=execution.ended_at,
             )
 
     # Process all executions in the batch concurrently

autogpt_platform/backend/backend/api/features/admin/store_admin_routes.py

@@ -9,6 +9,7 @@ import prisma.enums
 
 import backend.api.features.store.cache as store_cache
 import backend.api.features.store.db as store_db
+import backend.api.features.store.embeddings as store_embeddings
 import backend.api.features.store.model as store_model
 import backend.util.json
 
@@ -150,3 +151,54 @@ async def admin_download_agent_file(
         return fastapi.responses.FileResponse(
             tmp_file.name, filename=file_name, media_type="application/json"
         )
+
+
+@router.get(
+    "/embeddings/stats",
+    summary="Get Embedding Statistics",
+)
+async def get_embedding_stats() -> dict[str, typing.Any]:
+    """
+    Get statistics about embedding coverage for store listings.
+
+    Returns counts of total approved listings, listings with embeddings,
+    listings without embeddings, and coverage percentage.
+    """
+    try:
+        stats = await store_embeddings.get_embedding_stats()
+        return stats
+    except Exception as e:
+        logger.exception("Error getting embedding stats: %s", e)
+        raise fastapi.HTTPException(
+            status_code=500,
+            detail="An error occurred while retrieving embedding stats",
+        )
+
+
+@router.post(
+    "/embeddings/backfill",
+    summary="Backfill Missing Embeddings",
+)
+async def backfill_embeddings(
+    batch_size: int = 10,
+) -> dict[str, typing.Any]:
+    """
+    Trigger backfill of embeddings for approved listings that don't have them.
+
+    Args:
+        batch_size: Number of embeddings to generate in one call (default 10)
+
+    Returns:
+        Dict with processed count, success count, failure count, and message
+    """
+    try:
+        result = await store_embeddings.backfill_missing_embeddings(
+            batch_size=batch_size
+        )
+        return result
+    except Exception as e:
+        logger.exception("Error backfilling embeddings: %s", e)
+        raise fastapi.HTTPException(
+            status_code=500,
+            detail="An error occurred while backfilling embeddings",
+        )

autogpt_platform/backend/backend/api/features/chat/config.py

@@ -1,6 +1,7 @@
 """Configuration management for chat system."""
 
 import os
+from pathlib import Path
 
 from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings
@@ -26,6 +27,12 @@ class ChatConfig(BaseSettings):
     # Session TTL Configuration - 12 hours
     session_ttl: int = Field(default=43200, description="Session TTL in seconds")
 
+    # System Prompt Configuration
+    system_prompt_path: str = Field(
+        default="prompts/chat_system.md",
+        description="Path to system prompt file relative to chat module",
+    )
+
     # Streaming Configuration
     max_context_messages: int = Field(
         default=50, ge=1, le=200, description="Maximum context messages"
@@ -82,6 +89,73 @@ class ChatConfig(BaseSettings):
         "onboarding": "prompts/onboarding_system.md",
     }
 
+    def get_system_prompt_for_type(
+        self, prompt_type: str = "default", **template_vars
+    ) -> str:
+        """Load and render a system prompt by type.
+
+        Args:
+            prompt_type: The type of prompt to load ("default" or "onboarding")
+            **template_vars: Variables to substitute in the template
+
+        Returns:
+            Rendered system prompt string
+        """
+        prompt_path_str = self.PROMPT_PATHS.get(
+            prompt_type, self.PROMPT_PATHS["default"]
+        )
+        return self._load_prompt_from_path(prompt_path_str, **template_vars)
+
+    def get_system_prompt(self, **template_vars) -> str:
+        """Load and render the default system prompt from file.
+
+        Args:
+            **template_vars: Variables to substitute in the template
+
+        Returns:
+            Rendered system prompt string
+
+        """
+        return self._load_prompt_from_path(self.system_prompt_path, **template_vars)
+
+    def _load_prompt_from_path(self, prompt_path_str: str, **template_vars) -> str:
+        """Load and render a system prompt from a given path.
+
+        Args:
+            prompt_path_str: Path to the prompt file relative to chat module
+            **template_vars: Variables to substitute in the template
+
+        Returns:
+            Rendered system prompt string
+        """
+        # Get the path relative to this module
+        module_dir = Path(__file__).parent
+        prompt_path = module_dir / prompt_path_str
+
+        # Check for .j2 extension first (Jinja2 template)
+        j2_path = Path(str(prompt_path) + ".j2")
+        if j2_path.exists():
+            try:
+                from jinja2 import Template
+
+                template = Template(j2_path.read_text())
+                return template.render(**template_vars)
+            except ImportError:
+                # Jinja2 not installed, fall back to reading as plain text
+                return j2_path.read_text()
+
+        # Check for markdown file
+        if prompt_path.exists():
+            content = prompt_path.read_text()
+
+            # Simple variable substitution if Jinja2 is not available
+            for key, value in template_vars.items():
+                placeholder = f"{{{key}}}"
+                content = content.replace(placeholder, str(value))
+
+            return content
+        raise FileNotFoundError(f"System prompt file not found: {prompt_path}")
+
     class Config:
         """Pydantic config."""
 

autogpt_platform/backend/backend/api/features/chat/db.py

@@ -1,20 +1,13 @@
 """Database operations for chat sessions."""
 
-import asyncio
 import logging
 from datetime import UTC, datetime
-from typing import Any, cast
+from typing import Any
 
 from prisma.models import ChatMessage as PrismaChatMessage
 from prisma.models import ChatSession as PrismaChatSession
-from prisma.types import (
-    ChatMessageCreateInput,
-    ChatSessionCreateInput,
-    ChatSessionUpdateInput,
-    ChatSessionWhereInput,
-)
+from prisma.types import ChatSessionUpdateInput
 
-from backend.data.db import transaction
 from backend.util.json import SafeJson
 
 logger = logging.getLogger(__name__)
@@ -27,24 +20,23 @@ async def get_chat_session(session_id: str) -> PrismaChatSession | None:
         include={"Messages": True},
     )
     if session and session.Messages:
-        # Sort messages by sequence in Python - Prisma Python client doesn't support
-        # order_by in include clauses (unlike Prisma JS), so we sort after fetching
+        # Sort messages by sequence in Python since Prisma doesn't support order_by in include
         session.Messages.sort(key=lambda m: m.sequence)
     return session
 
 
 async def create_chat_session(
     session_id: str,
-    user_id: str,
+    user_id: str | None,
 ) -> PrismaChatSession:
     """Create a new chat session in the database."""
-    data = ChatSessionCreateInput(
-        id=session_id,
-        userId=user_id,
-        credentials=SafeJson({}),
-        successfulAgentRuns=SafeJson({}),
-        successfulAgentSchedules=SafeJson({}),
-    )
+    data = {
+        "id": session_id,
+        "userId": user_id,
+        "credentials": SafeJson({}),
+        "successfulAgentRuns": SafeJson({}),
+        "successfulAgentSchedules": SafeJson({}),
+    }
     return await PrismaChatSession.prisma().create(
         data=data,
         include={"Messages": True},
@@ -82,7 +74,6 @@ async def update_chat_session(
         include={"Messages": True},
     )
     if session and session.Messages:
-        # Sort in Python - Prisma Python doesn't support order_by in include clauses
         session.Messages.sort(key=lambda m: m.sequence)
     return session
 
@@ -99,16 +90,12 @@ async def add_chat_message(
     function_call: dict[str, Any] | None = None,
 ) -> PrismaChatMessage:
     """Add a message to a chat session."""
-    # Build input dict dynamically rather than using ChatMessageCreateInput directly
-    # because Prisma's TypedDict validation rejects optional fields set to None.
-    # We only include fields that have values, then cast at the end.
     data: dict[str, Any] = {
         "Session": {"connect": {"id": session_id}},
         "role": role,
         "sequence": sequence,
     }
 
-    # Add optional string fields
     if content is not None:
         data["content"] = content
     if name is not None:
@@ -117,22 +104,18 @@ async def add_chat_message(
         data["toolCallId"] = tool_call_id
     if refusal is not None:
         data["refusal"] = refusal
-
-    # Add optional JSON fields only when they have values
     if tool_calls is not None:
         data["toolCalls"] = SafeJson(tool_calls)
     if function_call is not None:
         data["functionCall"] = SafeJson(function_call)
 
-    # Run message create and session timestamp update in parallel for lower latency
-    _, message = await asyncio.gather(
-        PrismaChatSession.prisma().update(
-            where={"id": session_id},
-            data={"updatedAt": datetime.now(UTC)},
-        ),
-        PrismaChatMessage.prisma().create(data=cast(ChatMessageCreateInput, data)),
+    # Update session's updatedAt timestamp
+    await PrismaChatSession.prisma().update(
+        where={"id": session_id},
+        data={"updatedAt": datetime.now(UTC)},
     )
-    return message
+
+    return await PrismaChatMessage.prisma().create(data=data)
 
 
 async def add_chat_messages_batch(
@@ -140,55 +123,39 @@ async def add_chat_messages_batch(
     messages: list[dict[str, Any]],
     start_sequence: int,
 ) -> list[PrismaChatMessage]:
-    """Add multiple messages to a chat session in a batch.
-
-    Uses a transaction for atomicity - if any message creation fails,
-    the entire batch is rolled back.
-    """
+    """Add multiple messages to a chat session in a batch."""
     if not messages:
         return []
 
     created_messages = []
+    for i, msg in enumerate(messages):
+        data: dict[str, Any] = {
+            "Session": {"connect": {"id": session_id}},
+            "role": msg["role"],
+            "sequence": start_sequence + i,
+        }
 
-    async with transaction() as tx:
-        for i, msg in enumerate(messages):
-            # Build input dict dynamically rather than using ChatMessageCreateInput
-            # directly because Prisma's TypedDict validation rejects optional fields
-            # set to None. We only include fields that have values, then cast.
-            data: dict[str, Any] = {
-                "Session": {"connect": {"id": session_id}},
-                "role": msg["role"],
-                "sequence": start_sequence + i,
-            }
+        if msg.get("content") is not None:
+            data["content"] = msg["content"]
+        if msg.get("name") is not None:
+            data["name"] = msg["name"]
+        if msg.get("tool_call_id") is not None:
+            data["toolCallId"] = msg["tool_call_id"]
+        if msg.get("refusal") is not None:
+            data["refusal"] = msg["refusal"]
+        if msg.get("tool_calls") is not None:
+            data["toolCalls"] = SafeJson(msg["tool_calls"])
+        if msg.get("function_call") is not None:
+            data["functionCall"] = SafeJson(msg["function_call"])
 
-            # Add optional string fields
-            if msg.get("content") is not None:
-                data["content"] = msg["content"]
-            if msg.get("name") is not None:
-                data["name"] = msg["name"]
-            if msg.get("tool_call_id") is not None:
-                data["toolCallId"] = msg["tool_call_id"]
-            if msg.get("refusal") is not None:
-                data["refusal"] = msg["refusal"]
+        created = await PrismaChatMessage.prisma().create(data=data)
+        created_messages.append(created)
 
-            # Add optional JSON fields only when they have values
-            if msg.get("tool_calls") is not None:
-                data["toolCalls"] = SafeJson(msg["tool_calls"])
-            if msg.get("function_call") is not None:
-                data["functionCall"] = SafeJson(msg["function_call"])
-
-            created = await PrismaChatMessage.prisma(tx).create(
-                data=cast(ChatMessageCreateInput, data)
-            )
-            created_messages.append(created)
-
-        # Update session's updatedAt timestamp within the same transaction.
-        # Note: Token usage (total_prompt_tokens, total_completion_tokens) is updated
-        # separately via update_chat_session() after streaming completes.
-        await PrismaChatSession.prisma(tx).update(
-            where={"id": session_id},
-            data={"updatedAt": datetime.now(UTC)},
-        )
+    # Update session's updatedAt timestamp
+    await PrismaChatSession.prisma().update(
+        where={"id": session_id},
+        data={"updatedAt": datetime.now(UTC)},
+    )
 
     return created_messages
 
@@ -212,31 +179,10 @@ async def get_user_session_count(user_id: str) -> int:
     return await PrismaChatSession.prisma().count(where={"userId": user_id})
 
 
-async def delete_chat_session(session_id: str, user_id: str | None = None) -> bool:
-    """Delete a chat session and all its messages.
-
-    Args:
-        session_id: The session ID to delete.
-        user_id: If provided, validates that the session belongs to this user
-            before deletion. This prevents unauthorized deletion of other
-            users' sessions.
-
-    Returns:
-        True if deleted successfully, False otherwise.
-    """
+async def delete_chat_session(session_id: str) -> bool:
+    """Delete a chat session and all its messages."""
     try:
-        # Build typed where clause with optional user_id validation
-        where_clause: ChatSessionWhereInput = {"id": session_id}
-        if user_id is not None:
-            where_clause["userId"] = user_id
-
-        result = await PrismaChatSession.prisma().delete_many(where=where_clause)
-        if result == 0:
-            logger.warning(
-                f"No session deleted for {session_id} "
-                f"(user_id validation: {user_id is not None})"
-            )
-            return False
+        await PrismaChatSession.prisma().delete(where={"id": session_id})
         return True
     except Exception as e:
         logger.error(f"Failed to delete chat session {session_id}: {e}")

autogpt_platform/backend/backend/api/features/chat/model.py

@@ -1,9 +1,6 @@
-import asyncio
 import logging
 import uuid
 from datetime import UTC, datetime
-from typing import Any
-from weakref import WeakValueDictionary
 
 from openai.types.chat import (
     ChatCompletionAssistantMessageParam,
@@ -25,7 +22,7 @@ from pydantic import BaseModel
 
 from backend.data.redis_client import get_redis_async
 from backend.util import json
-from backend.util.exceptions import DatabaseError, RedisError
+from backend.util.exceptions import RedisError
 
 from . import db as chat_db
 from .config import ChatConfig
@@ -34,48 +31,6 @@ logger = logging.getLogger(__name__)
 config = ChatConfig()
 
 
-def _parse_json_field(value: str | dict | list | None, default: Any = None) -> Any:
-    """Parse a JSON field that may be stored as string or already parsed."""
-    if value is None:
-        return default
-    if isinstance(value, str):
-        return json.loads(value)
-    return value
-
-
-# Redis cache key prefix for chat sessions
-CHAT_SESSION_CACHE_PREFIX = "chat:session:"
-
-
-def _get_session_cache_key(session_id: str) -> str:
-    """Get the Redis cache key for a chat session."""
-    return f"{CHAT_SESSION_CACHE_PREFIX}{session_id}"
-
-
-# Session-level locks to prevent race conditions during concurrent upserts.
-# Uses WeakValueDictionary to automatically garbage collect locks when no longer referenced,
-# preventing unbounded memory growth while maintaining lock semantics for active sessions.
-# Invalidation: Locks are auto-removed by GC when no coroutine holds a reference (after
-# async with lock: completes). Explicit cleanup also occurs in delete_chat_session().
-_session_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary()
-_session_locks_mutex = asyncio.Lock()
-
-
-async def _get_session_lock(session_id: str) -> asyncio.Lock:
-    """Get or create a lock for a specific session to prevent concurrent upserts.
-
-    Uses WeakValueDictionary for automatic cleanup: locks are garbage collected
-    when no coroutine holds a reference to them, preventing memory leaks from
-    unbounded growth of session locks.
-    """
-    async with _session_locks_mutex:
-        lock = _session_locks.get(session_id)
-        if lock is None:
-            lock = asyncio.Lock()
-            _session_locks[session_id] = lock
-        return lock
-
-
 class ChatMessage(BaseModel):
     role: str
     content: str | None = None
@@ -94,7 +49,7 @@ class Usage(BaseModel):
 
 class ChatSession(BaseModel):
     session_id: str
-    user_id: str
+    user_id: str | None
     title: str | None = None
     messages: list[ChatMessage]
     usage: list[Usage]
@@ -105,7 +60,7 @@ class ChatSession(BaseModel):
     successful_agent_schedules: dict[str, int] = {}
 
     @staticmethod
-    def new(user_id: str) -> "ChatSession":
+    def new(user_id: str | None) -> "ChatSession":
         return ChatSession(
             session_id=str(uuid.uuid4()),
             user_id=user_id,
@@ -118,7 +73,7 @@ class ChatSession(BaseModel):
         )
 
     @staticmethod
-    def from_db(
+    def from_prisma(
         prisma_session: PrismaChatSession,
         prisma_messages: list[PrismaChatMessage] | None = None,
     ) -> "ChatSession":
@@ -126,6 +81,22 @@ class ChatSession(BaseModel):
         messages = []
         if prisma_messages:
             for msg in prisma_messages:
+                tool_calls = None
+                if msg.toolCalls:
+                    tool_calls = (
+                        json.loads(msg.toolCalls)
+                        if isinstance(msg.toolCalls, str)
+                        else msg.toolCalls
+                    )
+
+                function_call = None
+                if msg.functionCall:
+                    function_call = (
+                        json.loads(msg.functionCall)
+                        if isinstance(msg.functionCall, str)
+                        else msg.functionCall
+                    )
+
                 messages.append(
                     ChatMessage(
                         role=msg.role,
@@ -133,18 +104,26 @@ class ChatSession(BaseModel):
                         name=msg.name,
                         tool_call_id=msg.toolCallId,
                         refusal=msg.refusal,
-                        tool_calls=_parse_json_field(msg.toolCalls),
-                        function_call=_parse_json_field(msg.functionCall),
+                        tool_calls=tool_calls,
+                        function_call=function_call,
                     )
                 )
 
         # Parse JSON fields from Prisma
-        credentials = _parse_json_field(prisma_session.credentials, default={})
-        successful_agent_runs = _parse_json_field(
-            prisma_session.successfulAgentRuns, default={}
+        credentials = (
+            json.loads(prisma_session.credentials)
+            if isinstance(prisma_session.credentials, str)
+            else prisma_session.credentials or {}
         )
-        successful_agent_schedules = _parse_json_field(
-            prisma_session.successfulAgentSchedules, default={}
+        successful_agent_runs = (
+            json.loads(prisma_session.successfulAgentRuns)
+            if isinstance(prisma_session.successfulAgentRuns, str)
+            else prisma_session.successfulAgentRuns or {}
+        )
+        successful_agent_schedules = (
+            json.loads(prisma_session.successfulAgentSchedules)
+            if isinstance(prisma_session.successfulAgentSchedules, str)
+            else prisma_session.successfulAgentSchedules or {}
         )
 
         # Calculate usage from token counts
@@ -263,7 +242,7 @@ class ChatSession(BaseModel):
 
 async def _get_session_from_cache(session_id: str) -> ChatSession | None:
     """Get a chat session from Redis cache."""
-    redis_key = _get_session_cache_key(session_id)
+    redis_key = f"chat:session:{session_id}"
     async_redis = await get_redis_async()
     raw_session: bytes | None = await async_redis.get(redis_key)
 
@@ -285,7 +264,7 @@ async def _get_session_from_cache(session_id: str) -> ChatSession | None:
 
 async def _cache_session(session: ChatSession) -> None:
     """Cache a chat session in Redis."""
-    redis_key = _get_session_cache_key(session.session_id)
+    redis_key = f"chat:session:{session.session_id}"
     async_redis = await get_redis_async()
     await async_redis.setex(redis_key, config.session_ttl, session.model_dump_json())
 
@@ -304,7 +283,7 @@ async def _get_session_from_db(session_id: str) -> ChatSession | None:
         f"roles={[m.role for m in messages] if messages else []}"
     )
 
-    return ChatSession.from_db(prisma_session, messages)
+    return ChatSession.from_prisma(prisma_session, messages)
 
 
 async def _save_session_to_db(
@@ -366,24 +345,19 @@ async def _save_session_to_db(
 
 async def get_chat_session(
     session_id: str,
-    user_id: str | None = None,
+    user_id: str | None,
 ) -> ChatSession | None:
     """Get a chat session by ID.
 
     Checks Redis cache first, falls back to database if not found.
     Caches database results back to Redis.
-
-    Args:
-        session_id: The session ID to fetch.
-        user_id: If provided, validates that the session belongs to this user.
-            If None, ownership is not validated (admin/system access).
     """
     # Try cache first
     try:
         session = await _get_session_from_cache(session_id)
         if session:
-            # Verify user ownership if user_id was provided for validation
-            if user_id is not None and session.user_id != user_id:
+            # Verify user ownership
+            if session.user_id is not None and session.user_id != user_id:
                 logger.warning(
                     f"Session {session_id} user id mismatch: {session.user_id} != {user_id}"
                 )
@@ -402,8 +376,8 @@ async def get_chat_session(
         logger.warning(f"Session {session_id} not found in cache or database")
         return None
 
-    # Verify user ownership if user_id was provided for validation
-    if user_id is not None and session.user_id != user_id:
+    # Verify user ownership
+    if session.user_id is not None and session.user_id != user_id:
         logger.warning(
             f"Session {session_id} user id mismatch: {session.user_id} != {user_id}"
         )
@@ -422,88 +396,49 @@ async def get_chat_session(
 async def upsert_chat_session(
     session: ChatSession,
 ) -> ChatSession:
-    """Update a chat session in both cache and database.
+    """Update a chat session in both cache and database."""
+    # Get existing message count from DB for incremental saves
+    existing_message_count = await chat_db.get_chat_session_message_count(
+        session.session_id
+    )
 
-    Uses session-level locking to prevent race conditions when concurrent
-    operations (e.g., background title update and main stream handler)
-    attempt to upsert the same session simultaneously.
+    # Save to database
+    try:
+        await _save_session_to_db(session, existing_message_count)
+    except Exception as e:
+        logger.error(f"Failed to save session {session.session_id} to database: {e}")
+        # Continue to cache even if DB fails
 
-    Raises:
-        DatabaseError: If the database write fails. The cache is still updated
-            as a best-effort optimization, but the error is propagated to ensure
-            callers are aware of the persistence failure.
-        RedisError: If the cache write fails (after successful DB write).
-    """
-    # Acquire session-specific lock to prevent concurrent upserts
-    lock = await _get_session_lock(session.session_id)
+    # Save to cache
+    try:
+        await _cache_session(session)
+    except Exception as e:
+        raise RedisError(
+            f"Failed to persist chat session {session.session_id} to Redis: {e}"
+        ) from e
 
-    async with lock:
-        # Get existing message count from DB for incremental saves
-        existing_message_count = await chat_db.get_chat_session_message_count(
-            session.session_id
-        )
-
-        db_error: Exception | None = None
-
-        # Save to database (primary storage)
-        try:
-            await _save_session_to_db(session, existing_message_count)
-        except Exception as e:
-            logger.error(
-                f"Failed to save session {session.session_id} to database: {e}"
-            )
-            db_error = e
-
-        # Save to cache (best-effort, even if DB failed)
-        try:
-            await _cache_session(session)
-        except Exception as e:
-            # If DB succeeded but cache failed, raise cache error
-            if db_error is None:
-                raise RedisError(
-                    f"Failed to persist chat session {session.session_id} to Redis: {e}"
-                ) from e
-            # If both failed, log cache error but raise DB error (more critical)
-            logger.warning(
-                f"Cache write also failed for session {session.session_id}: {e}"
-            )
-
-        # Propagate DB error after attempting cache (prevents data loss)
-        if db_error is not None:
-            raise DatabaseError(
-                f"Failed to persist chat session {session.session_id} to database"
-            ) from db_error
-
-        return session
+    return session
 
 
-async def create_chat_session(user_id: str) -> ChatSession:
-    """Create a new chat session and persist it.
-
-    Raises:
-        DatabaseError: If the database write fails. We fail fast to ensure
-            callers never receive a non-persisted session that only exists
-            in cache (which would be lost when the cache expires).
-    """
+async def create_chat_session(user_id: str | None) -> ChatSession:
+    """Create a new chat session and persist it."""
     session = ChatSession.new(user_id)
 
-    # Create in database first - fail fast if this fails
+    # Create in database first
     try:
         await chat_db.create_chat_session(
             session_id=session.session_id,
             user_id=user_id,
         )
     except Exception as e:
-        logger.error(f"Failed to create session {session.session_id} in database: {e}")
-        raise DatabaseError(
-            f"Failed to create chat session {session.session_id} in database"
-        ) from e
+        logger.error(f"Failed to create session in database: {e}")
+        # Continue even if DB fails - cache will still work
 
-    # Cache the session (best-effort optimization, DB is source of truth)
+    # Cache the session
     try:
         await _cache_session(session)
     except Exception as e:
-        logger.warning(f"Failed to cache new session {session.session_id}: {e}")
+        logger.warning(f"Failed to cache new session: {e}")
 
     return session
 
@@ -512,86 +447,27 @@ async def get_user_sessions(
     user_id: str,
     limit: int = 50,
     offset: int = 0,
-) -> tuple[list[ChatSession], int]:
-    """Get chat sessions for a user from the database with total count.
-
-    Returns:
-        A tuple of (sessions, total_count) where total_count is the overall
-        number of sessions for the user (not just the current page).
-    """
+) -> list[ChatSession]:
+    """Get all chat sessions for a user from the database."""
     prisma_sessions = await chat_db.get_user_chat_sessions(user_id, limit, offset)
-    total_count = await chat_db.get_user_session_count(user_id)
 
     sessions = []
     for prisma_session in prisma_sessions:
         # Convert without messages for listing (lighter weight)
-        sessions.append(ChatSession.from_db(prisma_session, None))
+        sessions.append(ChatSession.from_prisma(prisma_session, None))
 
-    return sessions, total_count
+    return sessions
 
 
-async def delete_chat_session(session_id: str, user_id: str | None = None) -> bool:
-    """Delete a chat session from both cache and database.
-
-    Args:
-        session_id: The session ID to delete.
-        user_id: If provided, validates that the session belongs to this user
-            before deletion. This prevents unauthorized deletion.
-
-    Returns:
-        True if deleted successfully, False otherwise.
-    """
-    # Delete from database first (with optional user_id validation)
-    # This confirms ownership before invalidating cache
-    deleted = await chat_db.delete_chat_session(session_id, user_id)
-
-    if not deleted:
-        return False
-
-    # Only invalidate cache and clean up lock after DB confirms deletion
+async def delete_chat_session(session_id: str) -> bool:
+    """Delete a chat session from both cache and database."""
+    # Delete from cache
     try:
-        redis_key = _get_session_cache_key(session_id)
+        redis_key = f"chat:session:{session_id}"
         async_redis = await get_redis_async()
         await async_redis.delete(redis_key)
     except Exception as e:
         logger.warning(f"Failed to delete session {session_id} from cache: {e}")
 
-    # Clean up session lock (belt-and-suspenders with WeakValueDictionary)
-    async with _session_locks_mutex:
-        _session_locks.pop(session_id, None)
-
-    return True
-
-
-async def update_session_title(session_id: str, title: str) -> bool:
-    """Update only the title of a chat session.
-
-    This is a lightweight operation that doesn't touch messages, avoiding
-    race conditions with concurrent message updates. Use this for background
-    title generation instead of upsert_chat_session.
-
-    Args:
-        session_id: The session ID to update.
-        title: The new title to set.
-
-    Returns:
-        True if updated successfully, False otherwise.
-    """
-    try:
-        result = await chat_db.update_chat_session(session_id=session_id, title=title)
-        if result is None:
-            logger.warning(f"Session {session_id} not found for title update")
-            return False
-
-        # Invalidate cache so next fetch gets updated title
-        try:
-            redis_key = _get_session_cache_key(session_id)
-            async_redis = await get_redis_async()
-            await async_redis.delete(redis_key)
-        except Exception as e:
-            logger.warning(f"Failed to invalidate cache for session {session_id}: {e}")
-
-        return True
-    except Exception as e:
-        logger.error(f"Failed to update title for session {session_id}: {e}")
-        return False
+    # Delete from database
+    return await chat_db.delete_chat_session(session_id)

autogpt_platform/backend/backend/api/features/chat/model_test.py

@@ -43,9 +43,9 @@ async def test_chatsession_serialization_deserialization():
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_chatsession_redis_storage(setup_test_user, test_user_id):
+async def test_chatsession_redis_storage():
 
-    s = ChatSession.new(user_id=test_user_id)
+    s = ChatSession.new(user_id=None)
     s.messages = messages
 
     s = await upsert_chat_session(s)
@@ -59,28 +59,26 @@ async def test_chatsession_redis_storage(setup_test_user, test_user_id):
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_chatsession_redis_storage_user_id_mismatch(
-    setup_test_user, test_user_id
-):
+async def test_chatsession_redis_storage_user_id_mismatch():
 
-    s = ChatSession.new(user_id=test_user_id)
+    s = ChatSession.new(user_id="abc123")
     s.messages = messages
     s = await upsert_chat_session(s)
 
-    s2 = await get_chat_session(s.session_id, "different_user_id")
+    s2 = await get_chat_session(s.session_id, None)
 
     assert s2 is None
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_chatsession_db_storage(setup_test_user, test_user_id):
+async def test_chatsession_db_storage():
     """Test that messages are correctly saved to and loaded from DB (not cache)."""
     from backend.data.redis_client import get_redis_async
 
     # Create session with messages including assistant message
-    s = ChatSession.new(user_id=test_user_id)
+    s = ChatSession.new(user_id=None)
     s.messages = messages  # Contains user, assistant, and tool messages
-    assert s.session_id is not None, "Session id is not set"
+
     # Upsert to save to both cache and DB
     s = await upsert_chat_session(s)
 

autogpt_platform/backend/backend/api/features/chat/prompts/chat_system.md

@@ -0,0 +1,192 @@
+You are Otto, an AI Co-Pilot and Forward Deployed Engineer for AutoGPT, an AI Business Automation tool. Your mission is to help users quickly find, create, and set up AutoGPT agents to solve their business problems.
+
+Here are the functions available to you:
+
+<functions>
+**Understanding & Discovery:**
+1. **add_understanding** - Save information about the user's business context (use this as you learn about them)
+2. **find_agent** - Search the marketplace for pre-built agents that solve the user's problem
+3. **find_library_agent** - Search the user's personal library of saved agents
+4. **find_block** - Search for individual blocks (building components for agents)
+5. **search_platform_docs** - Search AutoGPT documentation for help
+
+**Agent Creation & Editing:**
+6. **create_agent** - Create a new custom agent from scratch based on user requirements
+7. **edit_agent** - Modify an existing agent (add/remove blocks, change configuration)
+
+**Execution & Output:**
+8. **run_agent** - Run or schedule an agent (automatically handles setup)
+9. **run_block** - Run a single block directly without creating an agent
+10. **agent_output** - Get the output/results from a running or completed agent execution
+</functions>
+
+## ALWAYS GET THE USER'S NAME
+
+**This is critical:** If you don't know the user's name, ask for it in your first response. Use a friendly, natural approach:
+- "Hi! I'm Otto. What's your name?"
+- "Hey there! Before we dive in, what should I call you?"
+
+Once you have their name, immediately save it with `add_understanding(user_name="...")` and use it throughout the conversation.
+
+## BUILDING USER UNDERSTANDING
+
+**If no User Business Context is provided below**, gather information naturally during conversation - don't interrogate them.
+
+**Key information to gather (in priority order):**
+1. Their name (ALWAYS first if unknown)
+2. Their job title and role
+3. Their business/company and industry
+4. Pain points and what they want to automate
+5. Tools they currently use
+
+**How to gather this information:**
+- Ask naturally as part of helping them (e.g., "What's your role?" or "What industry are you in?")
+- When they share information, immediately save it using `add_understanding`
+- Don't ask all questions at once - spread them across the conversation
+- Prioritize understanding their immediate problem first
+
+**Example:**
+```
+User: "I need help automating my social media"
+Otto: I can help with that! I'm Otto - what's your name?
+User: "I'm Sarah"
+Otto: [calls add_understanding with user_name="Sarah"]
+Nice to meet you, Sarah! What's your role - are you a social media manager or business owner?
+User: "I'm the marketing director at a fintech startup"
+Otto: [calls add_understanding with job_title="Marketing Director", industry="fintech", business_size="startup"]
+Great! Let me find social media automation agents for you.
+[calls find_agent with query="social media automation marketing"]
+```
+
+## WHEN TO USE WHICH TOOL
+
+**Finding existing agents:**
+- `find_agent` - Search the marketplace for pre-built agents others have created
+- `find_library_agent` - Search agents the user has already saved to their library
+
+**Creating/editing agents:**
+- `create_agent` - When user wants a custom agent that doesn't exist, or has specific requirements
+- `edit_agent` - When user wants to modify an existing agent (change inputs, add blocks, etc.)
+
+**Running agents:**
+- `run_agent` - To execute an agent (handles credentials and inputs automatically)
+- `agent_output` - To check the results of a running or completed agent execution
+
+**Direct execution:**
+- `run_block` - Run a single block directly without needing a full agent
+
+## HOW run_agent WORKS
+
+The `run_agent` tool automatically handles the entire setup flow:
+
+1. **First call** (no inputs) → Returns available inputs so user can decide what values to use
+2. **Credentials check** → If missing, UI automatically prompts user to add them (you don't need to mention this)
+3. **Execution** → Runs when you provide `inputs` OR set `use_defaults=true`
+
+Parameters:
+- `username_agent_slug` (required): Agent identifier like "creator/agent-name"
+- `inputs`: Object with input values for the agent
+- `use_defaults`: Set to `true` to run with default values (only after user confirms)
+- `schedule_name` + `cron`: For scheduled execution
+
+## HOW create_agent WORKS
+
+Use `create_agent` when the user wants to build a custom automation:
+- Describe what the agent should do
+- The tool will create the agent structure with appropriate blocks
+- Returns the agent ID for further editing or running
+
+## HOW agent_output WORKS
+
+Use `agent_output` to get results from agent executions:
+- Pass the execution_id from a run_agent response
+- Returns the current status and any outputs produced
+- Useful for checking if an agent has completed and what it produced
+
+## WORKFLOW
+
+1. **Get their name** - If unknown, ask for it first
+2. **Understand context** - Ask 1-2 questions about their problem while helping
+3. **Find or create** - Use find_agent for existing solutions, create_agent for custom needs
+4. **Set up and run** - Use run_agent to execute, agent_output to get results
+
+## YOUR APPROACH
+
+**Step 1: Greet and Identify**
+- If you don't know their name, ask for it
+- Be friendly and conversational
+
+**Step 2: Understand the Problem**
+- Ask maximum 1-2 targeted questions
+- Focus on: What business problem are they solving?
+- If they want to create/edit an agent, understand what it should do
+
+**Step 3: Find or Create**
+- For existing solutions: Use `find_agent` with relevant keywords
+- For custom needs: Use `create_agent` with their requirements
+- For modifications: Use `edit_agent` on an existing agent
+
+**Step 4: Execute**
+- Call `run_agent` without inputs first to see what's available
+- Ask user what values they want or if defaults are okay
+- Call `run_agent` again with inputs or `use_defaults=true`
+- Use `agent_output` to check results when needed
+
+## USING add_understanding
+
+Call `add_understanding` whenever you learn something about the user:
+
+**User info:** `user_name`, `job_title`
+**Business:** `business_name`, `industry`, `business_size` (1-10, 11-50, 51-200, 201-1000, 1000+), `user_role` (decision maker, implementer, end user)
+**Processes:** `key_workflows` (array), `daily_activities` (array)
+**Pain points:** `pain_points` (array), `bottlenecks` (array), `manual_tasks` (array), `automation_goals` (array)
+**Tools:** `current_software` (array), `existing_automation` (array)
+**Other:** `additional_notes`
+
+Example: `add_understanding(user_name="Sarah", job_title="Marketing Director", industry="fintech")`
+
+## KEY RULES
+
+**What You DON'T Do:**
+- Don't help with login (frontend handles this)
+- Don't mention or explain credentials to the user (frontend handles this automatically)
+- Don't run agents without first showing available inputs to the user
+- Don't use `use_defaults=true` without user explicitly confirming
+- Don't write responses longer than 3 sentences
+- Don't interrogate users with many questions - gather info naturally
+
+**What You DO:**
+- ALWAYS ask for user's name if you don't have it
+- Save user information with `add_understanding` as you learn it
+- Use their name when addressing them
+- Always call run_agent first without inputs to see what's available
+- Ask user what values they want OR if they want to use defaults
+- Keep all responses to maximum 3 sentences
+- Include the agent link in your response after successful execution
+
+**Error Handling:**
+- Authentication needed → "Please sign in via the interface"
+- Credentials missing → The UI handles this automatically. Focus on asking the user about input values instead.
+
+## RESPONSE STRUCTURE
+
+Before responding, wrap your analysis in <thinking> tags to systematically plan your approach:
+- Check if you know the user's name - if not, ask for it
+- Check if you have user context - if not, plan to gather some naturally
+- Extract the key business problem or request from the user's message
+- Determine what function call (if any) you need to make next
+- Plan your response to stay under the 3-sentence maximum
+
+Example interaction:
+```
+User: "Hi, I want to build an agent that monitors my competitors"
+Otto: <thinking>I don't know this user's name. I should ask for it while acknowledging their request.</thinking>
+Hi! I'm Otto and I'd love to help you build a competitor monitoring agent. What's your name?
+User: "I'm Mike"
+Otto: [calls add_understanding with user_name="Mike"]
+<thinking>Now I know Mike wants competitor monitoring. I should search for existing agents first.</thinking>
+Great to meet you, Mike! Let me search for competitor monitoring agents.
+[calls find_agent with query="competitor monitoring analysis"]
+```
+
+KEEP ANSWERS TO 3 SENTENCES

autogpt_platform/backend/backend/api/features/chat/prompts/onboarding_system.md

@@ -0,0 +1,155 @@
+You are Otto, an AI Co-Pilot helping new users get started with AutoGPT, an AI Business Automation platform. Your mission is to welcome them, learn about their needs, and help them run their first successful agent.
+
+Here are the functions available to you:
+
+<functions>
+**Understanding & Discovery:**
+1. **add_understanding** - Save information about the user's business context (use this as you learn about them)
+2. **find_agent** - Search the marketplace for pre-built agents that solve the user's problem
+3. **find_library_agent** - Search the user's personal library of saved agents
+4. **find_block** - Search for individual blocks (building components for agents)
+5. **search_platform_docs** - Search AutoGPT documentation for help
+
+**Agent Creation & Editing:**
+6. **create_agent** - Create a new custom agent from scratch based on user requirements
+7. **edit_agent** - Modify an existing agent (add/remove blocks, change configuration)
+
+**Execution & Output:**
+8. **run_agent** - Run or schedule an agent (automatically handles setup)
+9. **run_block** - Run a single block directly without creating an agent
+10. **agent_output** - Get the output/results from a running or completed agent execution
+</functions>
+
+## YOUR ONBOARDING MISSION
+
+You are guiding a new user through their first experience with AutoGPT. Your goal is to:
+1. Welcome them warmly and get their name
+2. Learn about them and their business
+3. Find or create an agent that solves a real problem for them
+4. Get that agent running successfully
+5. Celebrate their success and point them to next steps
+
+## PHASE 1: WELCOME & INTRODUCTION
+
+**Start every conversation by:**
+- Giving a warm, friendly greeting
+- Introducing yourself as Otto, their AI assistant
+- Asking for their name immediately
+
+**Example opening:**
+```
+Hi! I'm Otto, your AI assistant. Welcome to AutoGPT! I'm here to help you set up your first automation. What's your name?
+```
+
+Once you have their name, save it immediately with `add_understanding(user_name="...")` and use it throughout.
+
+## PHASE 2: DISCOVERY
+
+**After getting their name, learn about them:**
+- What's their role/job title?
+- What industry/business are they in?
+- What's one thing they'd love to automate?
+
+**Keep it conversational - don't interrogate. Example:**
+```
+Nice to meet you, Sarah! What do you do for work, and what's one task you wish you could automate?
+```
+
+Save everything you learn with `add_understanding`.
+
+## PHASE 3: FIND OR CREATE AN AGENT
+
+**Once you understand their need:**
+- Search for existing agents with `find_agent`
+- Present the best match and explain how it helps them
+- If nothing fits, offer to create a custom agent with `create_agent`
+
+**Be enthusiastic about the solution:**
+```
+I found a great agent for you! The "Social Media Scheduler" can automatically post to your accounts on a schedule. Want to try it?
+```
+
+## PHASE 4: SETUP & RUN
+
+**Guide them through running the agent:**
+1. Call `run_agent` without inputs first to see what's needed
+2. Explain each input in simple terms
+3. Ask what values they want to use
+4. Run the agent with their inputs or defaults
+
+**Don't mention credentials** - the UI handles that automatically.
+
+## PHASE 5: CELEBRATE & HANDOFF
+
+**After successful execution:**
+- Congratulate them on their first automation!
+- Tell them where to find this agent (their Library)
+- Mention they can explore more agents in the Marketplace
+- Offer to help with anything else
+
+**Example:**
+```
+You did it! Your first agent is running. You can find it anytime in your Library. Ready to explore more automations?
+```
+
+## KEY RULES
+
+**What You DON'T Do:**
+- Don't help with login (frontend handles this)
+- Don't mention credentials (UI handles automatically)
+- Don't run agents without showing inputs first
+- Don't use `use_defaults=true` without explicit confirmation
+- Don't write responses longer than 3 sentences
+- Don't overwhelm with too many questions at once
+
+**What You DO:**
+- ALWAYS get the user's name first
+- Be warm, encouraging, and celebratory
+- Save info with `add_understanding` as you learn it
+- Use their name when addressing them
+- Keep responses to maximum 3 sentences
+- Make them feel successful at each step
+
+## USING add_understanding
+
+Save information as you learn it:
+
+**User info:** `user_name`, `job_title`
+**Business:** `business_name`, `industry`, `business_size`, `user_role`
+**Pain points:** `pain_points`, `manual_tasks`, `automation_goals`
+**Tools:** `current_software`
+
+Example: `add_understanding(user_name="Sarah", job_title="Marketing Manager", automation_goals=["social media scheduling"])`
+
+## HOW run_agent WORKS
+
+1. **First call** (no inputs) → Shows available inputs
+2. **Credentials** → UI handles automatically (don't mention)
+3. **Execution** → Run with `inputs={...}` or `use_defaults=true`
+
+## RESPONSE STRUCTURE
+
+Before responding, plan your approach in <thinking> tags:
+- What phase am I in? (Welcome/Discovery/Find/Setup/Celebrate)
+- Do I know their name? If not, ask for it
+- What's the next step to move them forward?
+- Keep response under 3 sentences
+
+**Example flow:**
+```
+User: "Hi"
+Otto: <thinking>Phase 1 - I need to welcome them and get their name.</thinking>
+Hi! I'm Otto, welcome to AutoGPT! I'm here to help you set up your first automation - what's your name?
+
+User: "I'm Alex"
+Otto: [calls add_understanding with user_name="Alex"]
+<thinking>Got their name. Phase 2 - learn about them.</thinking>
+Great to meet you, Alex! What do you do for work, and what's one task you'd love to automate?
+
+User: "I run an e-commerce store and spend hours on customer support emails"
+Otto: [calls add_understanding with industry="e-commerce", pain_points=["customer support emails"]]
+<thinking>Phase 3 - search for agents.</thinking>
+[calls find_agent with query="customer support email automation"]
+```
+
+KEEP ANSWERS TO 3 SENTENCES - Be warm, helpful, and focused on their success!

autogpt_platform/backend/backend/api/features/chat/response_model.py

@@ -1,10 +1,3 @@
-"""
-Response models for Vercel AI SDK UI Stream Protocol.
-
-This module implements the AI SDK UI Stream Protocol (v1) for streaming chat responses.
-See: https://ai-sdk.dev/docs/ai-sdk-ui/stream-protocol
-"""
-
 from enum import Enum
 from typing import Any
 
@@ -12,133 +5,97 @@ from pydantic import BaseModel, Field
 
 
 class ResponseType(str, Enum):
-    """Types of streaming responses following AI SDK protocol."""
+    """Types of streaming responses."""
 
-    # Message lifecycle
-    START = "start"
-    FINISH = "finish"
-
-    # Text streaming
-    TEXT_START = "text-start"
-    TEXT_DELTA = "text-delta"
-    TEXT_END = "text-end"
-
-    # Tool interaction
-    TOOL_INPUT_START = "tool-input-start"
-    TOOL_INPUT_AVAILABLE = "tool-input-available"
-    TOOL_OUTPUT_AVAILABLE = "tool-output-available"
-
-    # Other
+    TEXT_CHUNK = "text_chunk"
+    TEXT_ENDED = "text_ended"
+    TOOL_CALL = "tool_call"
+    TOOL_CALL_START = "tool_call_start"
+    TOOL_RESPONSE = "tool_response"
     ERROR = "error"
     USAGE = "usage"
+    STREAM_END = "stream_end"
 
 
 class StreamBaseResponse(BaseModel):
     """Base response model for all streaming responses."""
 
     type: ResponseType
+    timestamp: str | None = None
 
     def to_sse(self) -> str:
         """Convert to SSE format."""
         return f"data: {self.model_dump_json()}\n\n"
 
 
-# ========== Message Lifecycle ==========
+class StreamTextChunk(StreamBaseResponse):
+    """Streaming text content from the assistant."""
+
+    type: ResponseType = ResponseType.TEXT_CHUNK
+    content: str = Field(..., description="Text content chunk")
 
 
-class StreamStart(StreamBaseResponse):
-    """Start of a new message."""
-
-    type: ResponseType = ResponseType.START
-    messageId: str = Field(..., description="Unique message ID")
-
-
-class StreamFinish(StreamBaseResponse):
-    """End of message/stream."""
-
-    type: ResponseType = ResponseType.FINISH
-
-
-# ========== Text Streaming ==========
-
-
-class StreamTextStart(StreamBaseResponse):
-    """Start of a text block."""
-
-    type: ResponseType = ResponseType.TEXT_START
-    id: str = Field(..., description="Text block ID")
-
-
-class StreamTextDelta(StreamBaseResponse):
-    """Streaming text content delta."""
-
-    type: ResponseType = ResponseType.TEXT_DELTA
-    id: str = Field(..., description="Text block ID")
-    delta: str = Field(..., description="Text content delta")
-
-
-class StreamTextEnd(StreamBaseResponse):
-    """End of a text block."""
-
-    type: ResponseType = ResponseType.TEXT_END
-    id: str = Field(..., description="Text block ID")
-
-
-# ========== Tool Interaction ==========
-
-
-class StreamToolInputStart(StreamBaseResponse):
+class StreamToolCallStart(StreamBaseResponse):
     """Tool call started notification."""
 
-    type: ResponseType = ResponseType.TOOL_INPUT_START
-    toolCallId: str = Field(..., description="Unique tool call ID")
-    toolName: str = Field(..., description="Name of the tool being called")
+    type: ResponseType = ResponseType.TOOL_CALL_START
+    tool_name: str = Field(..., description="Name of the tool that was executed")
+    tool_id: str = Field(..., description="Unique tool call ID")
 
 
-class StreamToolInputAvailable(StreamBaseResponse):
-    """Tool input is ready for execution."""
+class StreamToolCall(StreamBaseResponse):
+    """Tool invocation notification."""
 
-    type: ResponseType = ResponseType.TOOL_INPUT_AVAILABLE
-    toolCallId: str = Field(..., description="Unique tool call ID")
-    toolName: str = Field(..., description="Name of the tool being called")
-    input: dict[str, Any] = Field(
-        default_factory=dict, description="Tool input arguments"
+    type: ResponseType = ResponseType.TOOL_CALL
+    tool_id: str = Field(..., description="Unique tool call ID")
+    tool_name: str = Field(..., description="Name of the tool being called")
+    arguments: dict[str, Any] = Field(
+        default_factory=dict, description="Tool arguments"
     )
 
 
-class StreamToolOutputAvailable(StreamBaseResponse):
+class StreamToolExecutionResult(StreamBaseResponse):
     """Tool execution result."""
 
-    type: ResponseType = ResponseType.TOOL_OUTPUT_AVAILABLE
-    toolCallId: str = Field(..., description="Tool call ID this responds to")
-    output: str | dict[str, Any] = Field(..., description="Tool execution output")
-    # Additional fields for internal use (not part of AI SDK spec but useful)
-    toolName: str | None = Field(
-        default=None, description="Name of the tool that was executed"
-    )
+    type: ResponseType = ResponseType.TOOL_RESPONSE
+    tool_id: str = Field(..., description="Tool call ID this responds to")
+    tool_name: str = Field(..., description="Name of the tool that was executed")
+    result: str | dict[str, Any] = Field(..., description="Tool execution result")
     success: bool = Field(
         default=True, description="Whether the tool execution succeeded"
     )
 
 
-# ========== Other ==========
-
-
 class StreamUsage(StreamBaseResponse):
     """Token usage statistics."""
 
     type: ResponseType = ResponseType.USAGE
-    promptTokens: int = Field(..., description="Number of prompt tokens")
-    completionTokens: int = Field(..., description="Number of completion tokens")
-    totalTokens: int = Field(..., description="Total number of tokens")
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
 
 
 class StreamError(StreamBaseResponse):
     """Error response."""
 
     type: ResponseType = ResponseType.ERROR
-    errorText: str = Field(..., description="Error message text")
+    message: str = Field(..., description="Error message")
     code: str | None = Field(default=None, description="Error code")
     details: dict[str, Any] | None = Field(
         default=None, description="Additional error details"
     )
+
+
+class StreamTextEnded(StreamBaseResponse):
+    """Text streaming completed marker."""
+
+    type: ResponseType = ResponseType.TEXT_ENDED
+
+
+class StreamEnd(StreamBaseResponse):
+    """End of stream marker."""
+
+    type: ResponseType = ResponseType.STREAM_END
+    summary: dict[str, Any] | None = Field(
+        default=None, description="Stream summary statistics"
+    )

autogpt_platform/backend/backend/api/features/chat/routes.py

@@ -13,25 +13,12 @@ from backend.util.exceptions import NotFoundError
 
 from . import service as chat_service
 from .config import ChatConfig
-from .model import ChatSession, create_chat_session, get_chat_session, get_user_sessions
 
 config = ChatConfig()
 
 
 logger = logging.getLogger(__name__)
 
-
-async def _validate_and_get_session(
-    session_id: str,
-    user_id: str | None,
-) -> ChatSession:
-    """Validate session exists and belongs to user."""
-    session = await get_chat_session(session_id, user_id)
-    if not session:
-        raise NotFoundError(f"Session {session_id} not found.")
-    return session
-
-
 router = APIRouter(
     tags=["chat"],
 )
@@ -107,7 +94,7 @@ async def list_sessions(
     Returns:
         ListSessionsResponse: List of session summaries and total count.
     """
-    sessions, total_count = await get_user_sessions(user_id, limit, offset)
+    sessions = await chat_service.get_user_sessions(user_id, limit, offset)
 
     return ListSessionsResponse(
         sessions=[
@@ -115,11 +102,11 @@ async def list_sessions(
                 id=session.session_id,
                 created_at=session.started_at.isoformat(),
                 updated_at=session.updated_at.isoformat(),
-                title=session.title,
+                title=None,  # TODO: Add title support
             )
             for session in sessions
         ],
-        total=total_count,
+        total=len(sessions),
     )
 
 
@@ -127,15 +114,15 @@ async def list_sessions(
     "/sessions",
 )
 async def create_session(
-    user_id: Annotated[str, Depends(auth.get_user_id)],
+    user_id: Annotated[str | None, Depends(auth.get_user_id)],
 ) -> CreateSessionResponse:
     """
     Create a new chat session.
 
-    Initiates a new chat session for the authenticated user.
+    Initiates a new chat session for either an authenticated or anonymous user.
 
     Args:
-        user_id: The authenticated user ID parsed from the JWT (required).
+        user_id: The optional authenticated user ID parsed from the JWT. If missing, creates an anonymous session.
 
     Returns:
         CreateSessionResponse: Details of the created session.
@@ -143,15 +130,15 @@ async def create_session(
     """
     logger.info(
         f"Creating session with user_id: "
-        f"...{user_id[-8:] if len(user_id) > 8 else '<redacted>'}"
+        f"...{user_id[-8:] if user_id and len(user_id) > 8 else '<redacted>'}"
     )
 
-    session = await create_chat_session(user_id)
+    session = await chat_service.create_chat_session(user_id)
 
     return CreateSessionResponse(
         id=session.session_id,
         created_at=session.started_at.isoformat(),
-        user_id=session.user_id,
+        user_id=session.user_id or None,
     )
 
 
@@ -175,7 +162,7 @@ async def get_session(
         SessionDetailResponse: Details for the requested session; raises NotFoundError if not found.
 
     """
-    session = await get_chat_session(session_id, user_id)
+    session = await chat_service.get_session(session_id, user_id)
     if not session:
         raise NotFoundError(f"Session {session_id} not found")
 
@@ -219,7 +206,14 @@ async def stream_chat_post(
         StreamingResponse: SSE-formatted response chunks.
 
     """
-    session = await _validate_and_get_session(session_id, user_id)
+    # Validate session exists before starting the stream
+    # This prevents errors after the response has already started
+    session = await chat_service.get_session(session_id, user_id)
+
+    if not session:
+        raise NotFoundError(f"Session {session_id} not found. ")
+    if session.user_id is None and user_id is not None:
+        session = await chat_service.assign_user_to_session(session_id, user_id)
 
     async def event_generator() -> AsyncGenerator[str, None]:
         async for chunk in chat_service.stream_chat_completion(
@@ -231,8 +225,6 @@ async def stream_chat_post(
             context=request.context,
         ):
             yield chunk.to_sse()
-        # AI SDK protocol termination
-        yield "data: [DONE]\n\n"
 
     return StreamingResponse(
         event_generator(),
@@ -241,7 +233,6 @@ async def stream_chat_post(
             "Cache-Control": "no-cache",
             "Connection": "keep-alive",
             "X-Accel-Buffering": "no",  # Disable nginx buffering
-            "x-vercel-ai-ui-message-stream": "v1",  # AI SDK protocol header
         },
     )
 
@@ -272,7 +263,14 @@ async def stream_chat_get(
         StreamingResponse: SSE-formatted response chunks.
 
     """
-    session = await _validate_and_get_session(session_id, user_id)
+    # Validate session exists before starting the stream
+    # This prevents errors after the response has already started
+    session = await chat_service.get_session(session_id, user_id)
+
+    if not session:
+        raise NotFoundError(f"Session {session_id} not found. ")
+    if session.user_id is None and user_id is not None:
+        session = await chat_service.assign_user_to_session(session_id, user_id)
 
     async def event_generator() -> AsyncGenerator[str, None]:
         async for chunk in chat_service.stream_chat_completion(
@@ -283,8 +281,6 @@ async def stream_chat_get(
             session=session,  # Pass pre-fetched session to avoid double-fetch
         ):
             yield chunk.to_sse()
-        # AI SDK protocol termination
-        yield "data: [DONE]\n\n"
 
     return StreamingResponse(
         event_generator(),
@@ -293,7 +289,6 @@ async def stream_chat_get(
             "Cache-Control": "no-cache",
             "Connection": "keep-alive",
             "X-Accel-Buffering": "no",  # Disable nginx buffering
-            "x-vercel-ai-ui-message-stream": "v1",  # AI SDK protocol header
         },
     )
 
@@ -324,6 +319,133 @@ async def session_assign_user(
     return {"status": "ok"}
 
 
+# ========== Onboarding Routes ==========
+# These routes use a specialized onboarding system prompt
+
+
+@router.post(
+    "/onboarding/sessions",
+)
+async def create_onboarding_session(
+    user_id: Annotated[str | None, Depends(auth.get_user_id)],
+) -> CreateSessionResponse:
+    """
+    Create a new onboarding chat session.
+
+    Initiates a new chat session specifically for user onboarding,
+    using a specialized prompt that guides users through their first
+    experience with AutoGPT.
+
+    Args:
+        user_id: The optional authenticated user ID parsed from the JWT.
+
+    Returns:
+        CreateSessionResponse: Details of the created onboarding session.
+    """
+    logger.info(
+        f"Creating onboarding session with user_id: "
+        f"...{user_id[-8:] if user_id and len(user_id) > 8 else '<redacted>'}"
+    )
+
+    session = await chat_service.create_chat_session(user_id)
+
+    return CreateSessionResponse(
+        id=session.session_id,
+        created_at=session.started_at.isoformat(),
+        user_id=session.user_id or None,
+    )
+
+
+@router.get(
+    "/onboarding/sessions/{session_id}",
+)
+async def get_onboarding_session(
+    session_id: str,
+    user_id: Annotated[str | None, Depends(auth.get_user_id)],
+) -> SessionDetailResponse:
+    """
+    Retrieve the details of an onboarding chat session.
+
+    Args:
+        session_id: The unique identifier for the onboarding session.
+        user_id: The optional authenticated user ID.
+
+    Returns:
+        SessionDetailResponse: Details for the requested session.
+    """
+    session = await chat_service.get_session(session_id, user_id)
+    if not session:
+        raise NotFoundError(f"Session {session_id} not found")
+
+    messages = [message.model_dump() for message in session.messages]
+    logger.info(
+        f"Returning onboarding session {session_id}: "
+        f"message_count={len(messages)}, "
+        f"roles={[m.get('role') for m in messages]}"
+    )
+
+    return SessionDetailResponse(
+        id=session.session_id,
+        created_at=session.started_at.isoformat(),
+        updated_at=session.updated_at.isoformat(),
+        user_id=session.user_id or None,
+        messages=messages,
+    )
+
+
+@router.post(
+    "/onboarding/sessions/{session_id}/stream",
+)
+async def stream_onboarding_chat(
+    session_id: str,
+    request: StreamChatRequest,
+    user_id: str | None = Depends(auth.get_user_id),
+):
+    """
+    Stream onboarding chat responses for a session.
+
+    Uses the specialized onboarding system prompt to guide new users
+    through their first experience with AutoGPT. Streams AI responses
+    in real time over Server-Sent Events (SSE).
+
+    Args:
+        session_id: The onboarding session identifier.
+        request: Request body containing message and optional context.
+        user_id: Optional authenticated user ID.
+
+    Returns:
+        StreamingResponse: SSE-formatted response chunks.
+    """
+    session = await chat_service.get_session(session_id, user_id)
+
+    if not session:
+        raise NotFoundError(f"Session {session_id} not found.")
+    if session.user_id is None and user_id is not None:
+        session = await chat_service.assign_user_to_session(session_id, user_id)
+
+    async def event_generator() -> AsyncGenerator[str, None]:
+        async for chunk in chat_service.stream_chat_completion(
+            session_id,
+            request.message,
+            is_user_message=request.is_user_message,
+            user_id=user_id,
+            session=session,
+            context=request.context,
+            prompt_type="onboarding",  # Use onboarding system prompt
+        ):
+            yield chunk.to_sse()
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
 # ========== Health Check ==========
 
 
@@ -332,28 +454,16 @@ async def health_check() -> dict:
     """
     Health check endpoint for the chat service.
 
-    Performs a full cycle test of session creation and retrieval. Should always return healthy
+    Performs a full cycle test of session creation, assignment, and retrieval. Should always return healthy
     if the service and data layer are operational.
 
     Returns:
         dict: A status dictionary indicating health, service name, and API version.
 
     """
-    from backend.data.user import get_or_create_user
-
-    # Ensure health check user exists (required for FK constraint)
-    health_check_user_id = "health-check-user"
-    await get_or_create_user(
-        {
-            "sub": health_check_user_id,
-            "email": "health-check@system.local",
-            "user_metadata": {"name": "Health Check User"},
-        }
-    )
-
-    # Create and retrieve session to verify full data layer
-    session = await create_chat_session(health_check_user_id)
-    await get_chat_session(session.session_id, health_check_user_id)
+    session = await chat_service.create_chat_session(None)
+    await chat_service.assign_user_to_session(session.session_id, "test_user")
+    await chat_service.get_session(session.session_id, "test_user")
 
     return {
         "status": "healthy",

autogpt_platform/backend/backend/api/features/chat/service_test.py

@@ -4,19 +4,18 @@ from os import getenv
 import pytest
 
 from . import service as chat_service
-from .model import create_chat_session, get_chat_session, upsert_chat_session
 from .response_model import (
+    StreamEnd,
     StreamError,
-    StreamFinish,
-    StreamTextDelta,
-    StreamToolOutputAvailable,
+    StreamTextChunk,
+    StreamToolExecutionResult,
 )
 
 logger = logging.getLogger(__name__)
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_stream_chat_completion(setup_test_user, test_user_id):
+async def test_stream_chat_completion():
     """
     Test the stream_chat_completion function.
     """
@@ -24,7 +23,7 @@ async def test_stream_chat_completion(setup_test_user, test_user_id):
     if not api_key:
         return pytest.skip("OPEN_ROUTER_API_KEY is not set, skipping test")
 
-    session = await create_chat_session(test_user_id)
+    session = await chat_service.create_chat_session()
 
     has_errors = False
     has_ended = False
@@ -35,9 +34,9 @@ async def test_stream_chat_completion(setup_test_user, test_user_id):
         logger.info(chunk)
         if isinstance(chunk, StreamError):
             has_errors = True
-        if isinstance(chunk, StreamTextDelta):
-            assistant_message += chunk.delta
-        if isinstance(chunk, StreamFinish):
+        if isinstance(chunk, StreamTextChunk):
+            assistant_message += chunk.content
+        if isinstance(chunk, StreamEnd):
             has_ended = True
 
     assert has_ended, "Chat completion did not end"
@@ -46,7 +45,7 @@ async def test_stream_chat_completion(setup_test_user, test_user_id):
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user_id):
+async def test_stream_chat_completion_with_tool_calls():
     """
     Test the stream_chat_completion function.
     """
@@ -54,8 +53,8 @@ async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user
     if not api_key:
         return pytest.skip("OPEN_ROUTER_API_KEY is not set, skipping test")
 
-    session = await create_chat_session(test_user_id)
-    session = await upsert_chat_session(session)
+    session = await chat_service.create_chat_session()
+    session = await chat_service.upsert_chat_session(session)
 
     has_errors = False
     has_ended = False
@@ -69,14 +68,14 @@ async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user
         if isinstance(chunk, StreamError):
             has_errors = True
 
-        if isinstance(chunk, StreamFinish):
+        if isinstance(chunk, StreamEnd):
             has_ended = True
-        if isinstance(chunk, StreamToolOutputAvailable):
+        if isinstance(chunk, StreamToolExecutionResult):
             had_tool_calls = True
 
     assert has_ended, "Chat completion did not end"
     assert not has_errors, "Error occurred while streaming chat completion"
     assert had_tool_calls, "Tool calls did not occur"
-    session = await get_chat_session(session.session_id)
+    session = await chat_service.get_session(session.session_id)
     assert session, "Session not found"
     assert session.usage, "Usage is empty"

autogpt_platform/backend/backend/api/features/chat/tools/__init__.py

@@ -12,36 +12,37 @@ from .edit_agent import EditAgentTool
 from .find_agent import FindAgentTool
 from .find_block import FindBlockTool
 from .find_library_agent import FindLibraryAgentTool
-from .get_doc_page import GetDocPageTool
 from .run_agent import RunAgentTool
 from .run_block import RunBlockTool
 from .search_docs import SearchDocsTool
 
 if TYPE_CHECKING:
-    from backend.api.features.chat.response_model import StreamToolOutputAvailable
+    from backend.api.features.chat.response_model import StreamToolExecutionResult
 
-# Single source of truth for all tools
-TOOL_REGISTRY: dict[str, BaseTool] = {
-    "add_understanding": AddUnderstandingTool(),
-    "create_agent": CreateAgentTool(),
-    "edit_agent": EditAgentTool(),
-    "find_agent": FindAgentTool(),
-    "find_block": FindBlockTool(),
-    "find_library_agent": FindLibraryAgentTool(),
-    "run_agent": RunAgentTool(),
-    "run_block": RunBlockTool(),
-    "view_agent_output": AgentOutputTool(),
-    "search_docs": SearchDocsTool(),
-    "get_doc_page": GetDocPageTool(),
-}
+# Initialize tool instances
+add_understanding_tool = AddUnderstandingTool()
+create_agent_tool = CreateAgentTool()
+edit_agent_tool = EditAgentTool()
+find_agent_tool = FindAgentTool()
+find_block_tool = FindBlockTool()
+find_library_agent_tool = FindLibraryAgentTool()
+run_agent_tool = RunAgentTool()
+run_block_tool = RunBlockTool()
+search_docs_tool = SearchDocsTool()
+agent_output_tool = AgentOutputTool()
 
-# Export individual tool instances for backwards compatibility
-find_agent_tool = TOOL_REGISTRY["find_agent"]
-run_agent_tool = TOOL_REGISTRY["run_agent"]
-
-# Generated from registry for OpenAI API
+# Export tools as OpenAI format
 tools: list[ChatCompletionToolParam] = [
-    tool.as_openai_tool() for tool in TOOL_REGISTRY.values()
+    add_understanding_tool.as_openai_tool(),
+    create_agent_tool.as_openai_tool(),
+    edit_agent_tool.as_openai_tool(),
+    find_agent_tool.as_openai_tool(),
+    find_block_tool.as_openai_tool(),
+    find_library_agent_tool.as_openai_tool(),
+    run_agent_tool.as_openai_tool(),
+    run_block_tool.as_openai_tool(),
+    search_docs_tool.as_openai_tool(),
+    agent_output_tool.as_openai_tool(),
 ]
 
 
@@ -51,9 +52,22 @@ async def execute_tool(
     user_id: str | None,
     session: ChatSession,
     tool_call_id: str,
-) -> "StreamToolOutputAvailable":
-    """Execute a tool by name."""
-    tool = TOOL_REGISTRY.get(tool_name)
-    if not tool:
+) -> "StreamToolExecutionResult":
+
+    tool_map: dict[str, BaseTool] = {
+        "add_understanding": add_understanding_tool,
+        "create_agent": create_agent_tool,
+        "edit_agent": edit_agent_tool,
+        "find_agent": find_agent_tool,
+        "find_block": find_block_tool,
+        "find_library_agent": find_library_agent_tool,
+        "run_agent": run_agent_tool,
+        "run_block": run_block_tool,
+        "search_platform_docs": search_docs_tool,
+        "agent_output": agent_output_tool,
+    }
+    if tool_name not in tool_map:
         raise ValueError(f"Tool {tool_name} not found")
-    return await tool.execute(user_id, session, tool_call_id, **parameters)
+    return await tool_map[tool_name].execute(
+        user_id, session, tool_call_id, **parameters
+    )

autogpt_platform/backend/backend/api/features/chat/tools/_test_data.py

@@ -3,7 +3,6 @@ from datetime import UTC, datetime
 from os import getenv
 
 import pytest
-from prisma.types import ProfileCreateInput
 from pydantic import SecretStr
 
 from backend.api.features.chat.model import ChatSession
@@ -18,7 +17,7 @@ from backend.data.user import get_or_create_user
 from backend.integrations.credentials_store import IntegrationCredentialsStore
 
 
-def make_session(user_id: str):
+def make_session(user_id: str | None = None):
     return ChatSession(
         session_id=str(uuid.uuid4()),
         user_id=user_id,
@@ -50,13 +49,13 @@ async def setup_test_data():
     # 1b. Create a profile with username for the user (required for store agent lookup)
     username = user.email.split("@")[0]
     await prisma.profile.create(
-        data=ProfileCreateInput(
-            userId=user.id,
-            username=username,
-            name=f"Test User {username}",
-            description="Test user profile",
-            links=[],  # Required field - empty array for test profiles
-        )
+        data={
+            "userId": user.id,
+            "username": username,
+            "name": f"Test User {username}",
+            "description": "Test user profile",
+            "links": [],  # Required field - empty array for test profiles
+        }
     )
 
     # 2. Create a test graph with agent input -> agent output
@@ -173,13 +172,13 @@ async def setup_llm_test_data():
     # 1b. Create a profile with username for the user (required for store agent lookup)
     username = user.email.split("@")[0]
     await prisma.profile.create(
-        data=ProfileCreateInput(
-            userId=user.id,
-            username=username,
-            name=f"Test User {username}",
-            description="Test user profile for LLM tests",
-            links=[],  # Required field - empty array for test profiles
-        )
+        data={
+            "userId": user.id,
+            "username": username,
+            "name": f"Test User {username}",
+            "description": "Test user profile for LLM tests",
+            "links": [],  # Required field - empty array for test profiles
+        }
     )
 
     # 2. Create test OpenAI credentials for the user
@@ -333,13 +332,13 @@ async def setup_firecrawl_test_data():
     # 1b. Create a profile with username for the user (required for store agent lookup)
     username = user.email.split("@")[0]
     await prisma.profile.create(
-        data=ProfileCreateInput(
-            userId=user.id,
-            username=username,
-            name=f"Test User {username}",
-            description="Test user profile for Firecrawl tests",
-            links=[],  # Required field - empty array for test profiles
-        )
+        data={
+            "userId": user.id,
+            "username": username,
+            "name": f"Test User {username}",
+            "description": "Test user profile for Firecrawl tests",
+            "links": [],  # Required field - empty array for test profiles
+        }
     )
 
     # NOTE: We deliberately do NOT create Firecrawl credentials for this user

autogpt_platform/backend/backend/api/features/chat/tools/add_understanding.py

@@ -3,8 +3,6 @@
 import logging
 from typing import Any
 
-from langfuse import observe
-
 from backend.api.features.chat.model import ChatSession
 from backend.data.understanding import (
     BusinessUnderstandingInput,
@@ -12,7 +10,11 @@ from backend.data.understanding import (
 )
 
 from .base import BaseTool
-from .models import ErrorResponse, ToolResponseBase, UnderstandingUpdatedResponse
+from .models import (
+    ErrorResponse,
+    ToolResponseBase,
+    UnderstandingUpdatedResponse,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -36,32 +38,86 @@ and automations for the user's specific needs."""
 
     @property
     def parameters(self) -> dict[str, Any]:
-        # Auto-generate from Pydantic model schema
-        schema = BusinessUnderstandingInput.model_json_schema()
-        properties = {}
-        for field_name, field_schema in schema.get("properties", {}).items():
-            prop: dict[str, Any] = {"description": field_schema.get("description", "")}
-            # Handle anyOf for Optional types
-            if "anyOf" in field_schema:
-                for option in field_schema["anyOf"]:
-                    if option.get("type") != "null":
-                        prop["type"] = option.get("type", "string")
-                        if "items" in option:
-                            prop["items"] = option["items"]
-                        break
-            else:
-                prop["type"] = field_schema.get("type", "string")
-                if "items" in field_schema:
-                    prop["items"] = field_schema["items"]
-            properties[field_name] = prop
-        return {"type": "object", "properties": properties, "required": []}
+        return {
+            "type": "object",
+            "properties": {
+                "user_name": {
+                    "type": "string",
+                    "description": "The user's name",
+                },
+                "job_title": {
+                    "type": "string",
+                    "description": "The user's job title (e.g., 'Marketing Manager', 'CEO', 'Software Engineer')",
+                },
+                "business_name": {
+                    "type": "string",
+                    "description": "Name of the user's business or organization",
+                },
+                "industry": {
+                    "type": "string",
+                    "description": "Industry or sector (e.g., 'e-commerce', 'healthcare', 'finance')",
+                },
+                "business_size": {
+                    "type": "string",
+                    "description": "Company size: '1-10', '11-50', '51-200', '201-1000', or '1000+'",
+                },
+                "user_role": {
+                    "type": "string",
+                    "description": "User's role in organization context (e.g., 'decision maker', 'implementer', 'end user')",
+                },
+                "key_workflows": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Key business workflows (e.g., 'lead qualification', 'content publishing')",
+                },
+                "daily_activities": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Regular daily activities the user performs",
+                },
+                "pain_points": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Current pain points or challenges",
+                },
+                "bottlenecks": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Process bottlenecks slowing things down",
+                },
+                "manual_tasks": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Manual or repetitive tasks that could be automated",
+                },
+                "automation_goals": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Desired automation outcomes or goals",
+                },
+                "current_software": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Software and tools currently in use",
+                },
+                "existing_automation": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Any existing automations or integrations",
+                },
+                "additional_notes": {
+                    "type": "string",
+                    "description": "Any other relevant context or notes",
+                },
+            },
+            "required": [],
+        }
 
     @property
     def requires_auth(self) -> bool:
         """Requires authentication to store user-specific data."""
         return True
 
-    @observe(as_type="tool", name="add_understanding")
     async def _execute(
         self,
         user_id: str | None,
@@ -90,26 +146,54 @@ and automations for the user's specific needs."""
                 session_id=session_id,
             )
 
-        # Build input model from kwargs (only include fields defined in the model)
-        valid_fields = set(BusinessUnderstandingInput.model_fields.keys())
+        # Build input model
         input_data = BusinessUnderstandingInput(
-            **{k: v for k, v in kwargs.items() if k in valid_fields}
+            user_name=kwargs.get("user_name"),
+            job_title=kwargs.get("job_title"),
+            business_name=kwargs.get("business_name"),
+            industry=kwargs.get("industry"),
+            business_size=kwargs.get("business_size"),
+            user_role=kwargs.get("user_role"),
+            key_workflows=kwargs.get("key_workflows"),
+            daily_activities=kwargs.get("daily_activities"),
+            pain_points=kwargs.get("pain_points"),
+            bottlenecks=kwargs.get("bottlenecks"),
+            manual_tasks=kwargs.get("manual_tasks"),
+            automation_goals=kwargs.get("automation_goals"),
+            current_software=kwargs.get("current_software"),
+            existing_automation=kwargs.get("existing_automation"),
+            additional_notes=kwargs.get("additional_notes"),
         )
 
         # Track which fields were updated
-        updated_fields = [
-            k for k, v in kwargs.items() if k in valid_fields and v is not None
-        ]
+        updated_fields = [k for k, v in kwargs.items() if v is not None]
 
         # Upsert with merge
         understanding = await upsert_business_understanding(user_id, input_data)
 
-        # Build current understanding summary (filter out empty values)
+        # Build current understanding summary for the response
+        current_understanding = {
+            "user_name": understanding.user_name,
+            "job_title": understanding.job_title,
+            "business_name": understanding.business_name,
+            "industry": understanding.industry,
+            "business_size": understanding.business_size,
+            "user_role": understanding.user_role,
+            "key_workflows": understanding.key_workflows,
+            "daily_activities": understanding.daily_activities,
+            "pain_points": understanding.pain_points,
+            "bottlenecks": understanding.bottlenecks,
+            "manual_tasks": understanding.manual_tasks,
+            "automation_goals": understanding.automation_goals,
+            "current_software": understanding.current_software,
+            "existing_automation": understanding.existing_automation,
+            "additional_notes": understanding.additional_notes,
+        }
+
+        # Filter out empty values for cleaner response
         current_understanding = {
             k: v
-            for k, v in understanding.model_dump(
-                exclude={"id", "user_id", "created_at", "updated_at"}
-            ).items()
+            for k, v in current_understanding.items()
             if v is not None and v != [] and v != ""
         }
 

autogpt_platform/backend/backend/api/features/chat/tools/agent_generator/client.py

@@ -5,7 +5,7 @@ import os
 from openai import AsyncOpenAI
 
 # Configuration - use OPEN_ROUTER_API_KEY for consistency with chat/config.py
-OPENROUTER_API_KEY = os.getenv("OPEN_ROUTER_API_KEY")
+OPENROUTER_API_KEY = os.getenv("OPEN_ROUTER_API_KEY") or os.getenv("OPENROUTER_API_KEY")
 AGENT_GENERATOR_MODEL = os.getenv("AGENT_GENERATOR_MODEL", "anthropic/claude-opus-4.5")
 
 # OpenRouter client (OpenAI-compatible API)

autogpt_platform/backend/backend/api/features/chat/tools/agent_output.py

@@ -5,7 +5,6 @@ import re
 from datetime import datetime, timedelta, timezone
 from typing import Any
 
-from langfuse import observe
 from pydantic import BaseModel, field_validator
 
 from backend.api.features.chat.model import ChatSession
@@ -56,47 +55,56 @@ def parse_time_expression(
     """
     Parse time expression into datetime range (start, end).
 
-    Supports: "latest", "yesterday", "today", "last week", "last 7 days",
-    "last month", "last 30 days", ISO date "YYYY-MM-DD", ISO datetime.
+    Supports:
+    - "latest" or None -> returns (None, None) to get most recent
+    - "yesterday" -> 24h window for yesterday
+    - "today" -> Today from midnight
+    - "last week" / "last 7 days" -> 7 day window
+    - "last month" / "last 30 days" -> 30 day window
+    - ISO date "YYYY-MM-DD" -> 24h window for that date
     """
     if not time_expr or time_expr.lower() == "latest":
         return None, None
 
     now = datetime.now(timezone.utc)
-    today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
     expr = time_expr.lower().strip()
 
-    # Relative time expressions lookup
-    relative_times: dict[str, tuple[datetime, datetime]] = {
-        "yesterday": (today_start - timedelta(days=1), today_start),
-        "today": (today_start, now),
-        "last week": (now - timedelta(days=7), now),
-        "last 7 days": (now - timedelta(days=7), now),
-        "last month": (now - timedelta(days=30), now),
-        "last 30 days": (now - timedelta(days=30), now),
-    }
-    if expr in relative_times:
-        return relative_times[expr]
+    # Relative expressions
+    if expr == "yesterday":
+        end = now.replace(hour=0, minute=0, second=0, microsecond=0)
+        start = end - timedelta(days=1)
+        return start, end
+
+    if expr in ("last week", "last 7 days"):
+        return now - timedelta(days=7), now
+
+    if expr in ("last month", "last 30 days"):
+        return now - timedelta(days=30), now
+
+    if expr == "today":
+        start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+        return start, now
 
     # Try ISO date format (YYYY-MM-DD)
     date_match = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", expr)
     if date_match:
-        try:
-            year, month, day = map(int, date_match.groups())
-            start = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
-            return start, start + timedelta(days=1)
-        except ValueError:
-            # Invalid date components (e.g., month=13, day=32)
-            pass
+        year, month, day = map(int, date_match.groups())
+        start = datetime(year, month, day, 0, 0, 0, tzinfo=timezone.utc)
+        end = start + timedelta(days=1)
+        return start, end
 
     # Try ISO datetime
     try:
         parsed = datetime.fromisoformat(expr.replace("Z", "+00:00"))
         if parsed.tzinfo is None:
             parsed = parsed.replace(tzinfo=timezone.utc)
+        # Return +/- 1 hour window around the specified time
         return parsed - timedelta(hours=1), parsed + timedelta(hours=1)
     except ValueError:
-        return None, None
+        pass
+
+    # Fallback: treat as "latest"
+    return None, None
 
 
 class AgentOutputTool(BaseTool):
@@ -104,7 +112,7 @@ class AgentOutputTool(BaseTool):
 
     @property
     def name(self) -> str:
-        return "view_agent_output"
+        return "agent_output"
 
     @property
     def description(self) -> str:
@@ -329,7 +337,6 @@ class AgentOutputTool(BaseTool):
             total_executions=len(available_executions) if available_executions else 1,
         )
 
-    @observe(as_type="tool", name="view_agent_output")
     async def _execute(
         self,
         user_id: str | None,

autogpt_platform/backend/backend/api/features/chat/tools/agent_search.py

@@ -1,151 +0,0 @@
-"""Shared agent search functionality for find_agent and find_library_agent tools."""
-
-import logging
-from typing import Literal
-
-from backend.api.features.library import db as library_db
-from backend.api.features.store import db as store_db
-from backend.util.exceptions import DatabaseError, NotFoundError
-
-from .models import (
-    AgentInfo,
-    AgentsFoundResponse,
-    ErrorResponse,
-    NoResultsResponse,
-    ToolResponseBase,
-)
-
-logger = logging.getLogger(__name__)
-
-SearchSource = Literal["marketplace", "library"]
-
-
-async def search_agents(
-    query: str,
-    source: SearchSource,
-    session_id: str | None,
-    user_id: str | None = None,
-) -> ToolResponseBase:
-    """
-    Search for agents in marketplace or user library.
-
-    Args:
-        query: Search query string
-        source: "marketplace" or "library"
-        session_id: Chat session ID
-        user_id: User ID (required for library search)
-
-    Returns:
-        AgentsFoundResponse, NoResultsResponse, or ErrorResponse
-    """
-    if not query:
-        return ErrorResponse(
-            message="Please provide a search query", session_id=session_id
-        )
-
-    if source == "library" and not user_id:
-        return ErrorResponse(
-            message="User authentication required to search library",
-            session_id=session_id,
-        )
-
-    agents: list[AgentInfo] = []
-    try:
-        if source == "marketplace":
-            logger.info(f"Searching marketplace for: {query}")
-            results = await store_db.get_store_agents(search_query=query, page_size=5)
-            for agent in results.agents:
-                agents.append(
-                    AgentInfo(
-                        id=f"{agent.creator}/{agent.slug}",
-                        name=agent.agent_name,
-                        description=agent.description or "",
-                        source="marketplace",
-                        in_library=False,
-                        creator=agent.creator,
-                        category="general",
-                        rating=agent.rating,
-                        runs=agent.runs,
-                        is_featured=False,
-                    )
-                )
-        else:  # library
-            logger.info(f"Searching user library for: {query}")
-            results = await library_db.list_library_agents(
-                user_id=user_id,  # type: ignore[arg-type]
-                search_term=query,
-                page_size=10,
-            )
-            for agent in results.agents:
-                agents.append(
-                    AgentInfo(
-                        id=agent.id,
-                        name=agent.name,
-                        description=agent.description or "",
-                        source="library",
-                        in_library=True,
-                        creator=agent.creator_name,
-                        status=agent.status.value,
-                        can_access_graph=agent.can_access_graph,
-                        has_external_trigger=agent.has_external_trigger,
-                        new_output=agent.new_output,
-                        graph_id=agent.graph_id,
-                    )
-                )
-        logger.info(f"Found {len(agents)} agents in {source}")
-    except NotFoundError:
-        pass
-    except DatabaseError as e:
-        logger.error(f"Error searching {source}: {e}", exc_info=True)
-        return ErrorResponse(
-            message=f"Failed to search {source}. Please try again.",
-            error=str(e),
-            session_id=session_id,
-        )
-
-    if not agents:
-        suggestions = (
-            [
-                "Try more general terms",
-                "Browse categories in the marketplace",
-                "Check spelling",
-            ]
-            if source == "marketplace"
-            else [
-                "Try different keywords",
-                "Use find_agent to search the marketplace",
-                "Check your library at /library",
-            ]
-        )
-        no_results_msg = (
-            f"No agents found matching '{query}'. Try different keywords or browse the marketplace."
-            if source == "marketplace"
-            else f"No agents matching '{query}' found in your library."
-        )
-        return NoResultsResponse(
-            message=no_results_msg, session_id=session_id, suggestions=suggestions
-        )
-
-    title = f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} "
-    title += (
-        f"for '{query}'"
-        if source == "marketplace"
-        else f"in your library for '{query}'"
-    )
-
-    message = (
-        "Now you have found some options for the user to choose from. "
-        "You can add a link to a recommended agent at: /marketplace/agent/agent_id "
-        "Please ask the user if they would like to use any of these agents."
-        if source == "marketplace"
-        else "Found agents in the user's library. You can provide a link to view an agent at: "
-        "/library/agents/{agent_id}. Use agent_output to get execution results, or run_agent to execute."
-    )
-
-    return AgentsFoundResponse(
-        message=message,
-        title=title,
-        agents=agents,
-        count=len(agents),
-        session_id=session_id,
-    )

autogpt_platform/backend/backend/api/features/chat/tools/base.py

@@ -6,7 +6,7 @@ from typing import Any
 from openai.types.chat import ChatCompletionToolParam
 
 from backend.api.features.chat.model import ChatSession
-from backend.api.features.chat.response_model import StreamToolOutputAvailable
+from backend.api.features.chat.response_model import StreamToolExecutionResult
 
 from .models import ErrorResponse, NeedLoginResponse, ToolResponseBase
 
@@ -53,7 +53,7 @@ class BaseTool:
         session: ChatSession,
         tool_call_id: str,
         **kwargs,
-    ) -> StreamToolOutputAvailable:
+    ) -> StreamToolExecutionResult:
         """Execute the tool with authentication check.
 
         Args:
@@ -69,10 +69,10 @@ class BaseTool:
             logger.error(
                 f"Attempted tool call for {self.name} but user not authenticated"
             )
-            return StreamToolOutputAvailable(
-                toolCallId=tool_call_id,
-                toolName=self.name,
-                output=NeedLoginResponse(
+            return StreamToolExecutionResult(
+                tool_id=tool_call_id,
+                tool_name=self.name,
+                result=NeedLoginResponse(
                     message=f"Please sign in to use {self.name}",
                     session_id=session.session_id,
                 ).model_dump_json(),
@@ -81,17 +81,17 @@ class BaseTool:
 
         try:
             result = await self._execute(user_id, session, **kwargs)
-            return StreamToolOutputAvailable(
-                toolCallId=tool_call_id,
-                toolName=self.name,
-                output=result.model_dump_json(),
+            return StreamToolExecutionResult(
+                tool_id=tool_call_id,
+                tool_name=self.name,
+                result=result.model_dump_json(),
             )
         except Exception as e:
             logger.error(f"Error in {self.name}: {e}", exc_info=True)
-            return StreamToolOutputAvailable(
-                toolCallId=tool_call_id,
-                toolName=self.name,
-                output=ErrorResponse(
+            return StreamToolExecutionResult(
+                tool_id=tool_call_id,
+                tool_name=self.name,
+                result=ErrorResponse(
                     message=f"An error occurred while executing {self.name}",
                     error=str(e),
                     session_id=session.session_id,

autogpt_platform/backend/backend/api/features/chat/tools/create_agent.py

@@ -3,8 +3,6 @@
 import logging
 from typing import Any
 
-from langfuse import observe
-
 from backend.api.features.chat.model import ChatSession
 
 from .agent_generator import (
@@ -80,7 +78,6 @@ class CreateAgentTool(BaseTool):
             "required": ["description"],
         }
 
-    @observe(as_type="tool", name="create_agent")
     async def _execute(
         self,
         user_id: str | None,

autogpt_platform/backend/backend/api/features/chat/tools/edit_agent.py

@@ -3,8 +3,6 @@
 import logging
 from typing import Any
 
-from langfuse import observe
-
 from backend.api.features.chat.model import ChatSession
 
 from .agent_generator import (
@@ -87,7 +85,6 @@ class EditAgentTool(BaseTool):
             "required": ["agent_id", "changes"],
         }
 
-    @observe(as_type="tool", name="edit_agent")
     async def _execute(
         self,
         user_id: str | None,

autogpt_platform/backend/backend/api/features/chat/tools/find_agent.py

@@ -1,18 +1,26 @@
-"""Tool for discovering agents from marketplace."""
+"""Tool for discovering agents from marketplace and user library."""
 
+import logging
 from typing import Any
 
-from langfuse import observe
-
 from backend.api.features.chat.model import ChatSession
+from backend.api.features.store import db as store_db
+from backend.util.exceptions import DatabaseError, NotFoundError
 
-from .agent_search import search_agents
 from .base import BaseTool
-from .models import ToolResponseBase
+from .models import (
+    AgentCarouselResponse,
+    AgentInfo,
+    ErrorResponse,
+    NoResultsResponse,
+    ToolResponseBase,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class FindAgentTool(BaseTool):
-    """Tool for discovering agents from the marketplace."""
+    """Tool for discovering agents based on user needs."""
 
     @property
     def name(self) -> str:
@@ -37,13 +45,85 @@ class FindAgentTool(BaseTool):
             "required": ["query"],
         }
 
-    @observe(as_type="tool", name="find_agent")
     async def _execute(
-        self, user_id: str | None, session: ChatSession, **kwargs
+        self,
+        user_id: str | None,
+        session: ChatSession,
+        **kwargs,
     ) -> ToolResponseBase:
-        return await search_agents(
-            query=kwargs.get("query", "").strip(),
-            source="marketplace",
-            session_id=session.session_id,
-            user_id=user_id,
+        """Search for agents in the marketplace.
+
+        Args:
+            user_id: User ID (may be anonymous)
+            session_id: Chat session ID
+            query: Search query
+
+        Returns:
+            AgentCarouselResponse: List of agents found in the marketplace
+            NoResultsResponse: No agents found in the marketplace
+            ErrorResponse: Error message
+        """
+        query = kwargs.get("query", "").strip()
+        session_id = session.session_id
+        if not query:
+            return ErrorResponse(
+                message="Please provide a search query",
+                session_id=session_id,
+            )
+        agents = []
+        try:
+            logger.info(f"Searching marketplace for: {query}")
+            store_results = await store_db.get_store_agents(
+                search_query=query,
+                page_size=5,
+            )
+
+            logger.info(f"Find agents tool found {len(store_results.agents)} agents")
+            for agent in store_results.agents:
+                agent_id = f"{agent.creator}/{agent.slug}"
+                logger.info(f"Building agent ID = {agent_id}")
+                agents.append(
+                    AgentInfo(
+                        id=agent_id,
+                        name=agent.agent_name,
+                        description=agent.description or "",
+                        source="marketplace",
+                        in_library=False,
+                        creator=agent.creator,
+                        category="general",
+                        rating=agent.rating,
+                        runs=agent.runs,
+                        is_featured=False,
+                    ),
+                )
+        except NotFoundError:
+            pass
+        except DatabaseError as e:
+            logger.error(f"Error searching agents: {e}", exc_info=True)
+            return ErrorResponse(
+                message="Failed to search for agents. Please try again.",
+                error=str(e),
+                session_id=session_id,
+            )
+        if not agents:
+            return NoResultsResponse(
+                message=f"No agents found matching '{query}'. Try different keywords or browse the marketplace. If you have 3 consecutive find_agent tool calls results and found no agents. Please stop trying and ask the user if there is anything else you can help with.",
+                session_id=session_id,
+                suggestions=[
+                    "Try more general terms",
+                    "Browse categories in the marketplace",
+                    "Check spelling",
+                ],
+            )
+
+        # Return formatted carousel
+        title = (
+            f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} for '{query}'"
+        )
+        return AgentCarouselResponse(
+            message="Now you have found some options for the user to choose from. You can add a link to a recommended agent at: /marketplace/agent/agent_id Please ask the user if they would like to use any of these agents. If they do, please call the get_agent_details tool for this agent.",
+            title=title,
+            agents=agents,
+            count=len(agents),
+            session_id=session_id,
         )

autogpt_platform/backend/backend/api/features/chat/tools/find_block.py

@@ -1,20 +1,20 @@
+"""Tool for searching available blocks using hybrid search."""
+
 import logging
 from typing import Any
 
-from langfuse import observe
-from prisma.enums import ContentType
-
 from backend.api.features.chat.model import ChatSession
-from backend.api.features.chat.tools.base import BaseTool, ToolResponseBase
-from backend.api.features.chat.tools.models import (
+from backend.blocks import load_all_blocks
+
+from .base import BaseTool
+from .models import (
     BlockInfoSummary,
-    BlockInputFieldInfo,
     BlockListResponse,
     ErrorResponse,
     NoResultsResponse,
+    ToolResponseBase,
 )
-from backend.api.features.store.hybrid_search import unified_hybrid_search
-from backend.data.block import get_block
+from .search_blocks import get_block_search_index
 
 logger = logging.getLogger(__name__)
 
@@ -32,8 +32,7 @@ class FindBlockTool(BaseTool):
             "Search for available blocks by name or description. "
             "Blocks are reusable components that perform specific tasks like "
             "sending emails, making API calls, processing text, etc. "
-            "IMPORTANT: Use this tool FIRST to get the block's 'id' before calling run_block. "
-            "The response includes each block's id, required_inputs, and input_schema."
+            "Use this to find blocks that can be executed directly."
         )
 
     @property
@@ -56,7 +55,39 @@ class FindBlockTool(BaseTool):
     def requires_auth(self) -> bool:
         return True
 
-    @observe(as_type="tool", name="find_block")
+    def _matches_query(self, block, query: str) -> tuple[int, bool]:
+        """
+        Check if a block matches the query and return a priority score.
+
+        Returns (priority, matches) where:
+        - priority 0: exact name match
+        - priority 1: name contains query
+        - priority 2: description contains query
+        - priority 3: category contains query
+        """
+        query_lower = query.lower()
+        name_lower = block.name.lower()
+        desc_lower = block.description.lower()
+
+        # Exact name match
+        if query_lower == name_lower:
+            return 0, True
+
+        # Name contains query
+        if query_lower in name_lower:
+            return 1, True
+
+        # Description contains query
+        if query_lower in desc_lower:
+            return 2, True
+
+        # Category contains query
+        for category in block.categories:
+            if query_lower in category.name.lower():
+                return 3, True
+
+        return 4, False
+
     async def _execute(
         self,
         user_id: str | None,
@@ -85,110 +116,138 @@ class FindBlockTool(BaseTool):
             )
 
         try:
-            # Search for blocks using hybrid search
-            results, total = await unified_hybrid_search(
-                query=query,
-                content_types=[ContentType.BLOCK],
-                page=1,
-                page_size=10,
-            )
+            # Try hybrid search first
+            search_results = self._hybrid_search(query)
 
-            if not results:
-                return NoResultsResponse(
-                    message=f"No blocks found for '{query}'",
-                    suggestions=[
-                        "Try broader keywords like 'email', 'http', 'text', 'ai'",
-                        "Check spelling of technical terms",
-                    ],
-                    session_id=session_id,
-                )
-
-            # Enrich results with full block information
-            blocks: list[BlockInfoSummary] = []
-            for result in results:
-                block_id = result["content_id"]
-                block = get_block(block_id)
-
-                if block:
-                    # Get input/output schemas
-                    input_schema = {}
-                    output_schema = {}
-                    try:
-                        input_schema = block.input_schema.jsonschema()
-                    except Exception:
-                        pass
-                    try:
-                        output_schema = block.output_schema.jsonschema()
-                    except Exception:
-                        pass
-
-                    # Get categories from block instance
-                    categories = []
-                    if hasattr(block, "categories") and block.categories:
-                        categories = [cat.value for cat in block.categories]
-
-                    # Extract required inputs for easier use
-                    required_inputs: list[BlockInputFieldInfo] = []
-                    if input_schema:
-                        properties = input_schema.get("properties", {})
-                        required_fields = set(input_schema.get("required", []))
-                        # Get credential field names to exclude from required inputs
-                        credentials_fields = set(
-                            block.input_schema.get_credentials_fields().keys()
-                        )
-
-                        for field_name, field_schema in properties.items():
-                            # Skip credential fields - they're handled separately
-                            if field_name in credentials_fields:
-                                continue
-
-                            required_inputs.append(
-                                BlockInputFieldInfo(
-                                    name=field_name,
-                                    type=field_schema.get("type", "string"),
-                                    description=field_schema.get("description", ""),
-                                    required=field_name in required_fields,
-                                    default=field_schema.get("default"),
-                                )
-                            )
-
-                    blocks.append(
-                        BlockInfoSummary(
-                            id=block_id,
-                            name=block.name,
-                            description=block.description or "",
-                            categories=categories,
-                            input_schema=input_schema,
-                            output_schema=output_schema,
-                            required_inputs=required_inputs,
-                        )
+            if search_results is not None:
+                # Hybrid search succeeded
+                if not search_results:
+                    return NoResultsResponse(
+                        message=f"No blocks found matching '{query}'",
+                        session_id=session_id,
+                        suggestions=[
+                            "Try more general terms",
+                            "Search by category: ai, text, social, search, etc.",
+                            "Check block names like 'SendEmail', 'HttpRequest', etc.",
+                        ],
                     )
 
-            if not blocks:
-                return NoResultsResponse(
-                    message=f"No blocks found for '{query}'",
-                    suggestions=[
-                        "Try broader keywords like 'email', 'http', 'text', 'ai'",
-                    ],
+                # Get full block info for each result
+                all_blocks = load_all_blocks()
+                blocks = []
+                for result in search_results:
+                    block_cls = all_blocks.get(result.block_id)
+                    if block_cls:
+                        block = block_cls()
+                        blocks.append(
+                            BlockInfoSummary(
+                                id=block.id,
+                                name=block.name,
+                                description=block.description,
+                                categories=[cat.name for cat in block.categories],
+                                input_schema=block.input_schema.jsonschema(),
+                                output_schema=block.output_schema.jsonschema(),
+                            )
+                        )
+
+                return BlockListResponse(
+                    message=(
+                        f"Found {len(blocks)} block{'s' if len(blocks) != 1 else ''} "
+                        f"matching '{query}'. Use run_block to execute a block with "
+                        "the required inputs."
+                    ),
+                    blocks=blocks,
+                    count=len(blocks),
+                    query=query,
                     session_id=session_id,
                 )
 
-            return BlockListResponse(
-                message=(
-                    f"Found {len(blocks)} block(s) matching '{query}'. "
-                    "To execute a block, use run_block with the block's 'id' field "
-                    "and provide 'input_data' matching the block's input_schema."
-                ),
-                blocks=blocks,
-                count=len(blocks),
-                query=query,
-                session_id=session_id,
-            )
+            # Fallback to simple search if hybrid search failed
+            return self._simple_search(query, session_id)
 
         except Exception as e:
             logger.error(f"Error searching blocks: {e}", exc_info=True)
             return ErrorResponse(
-                message="Failed to search blocks",
+                message="Failed to search blocks. Please try again.",
                 error=str(e),
                 session_id=session_id,
             )
+
+    def _hybrid_search(self, query: str) -> list | None:
+        """
+        Perform hybrid search using embeddings and BM25.
+
+        Returns:
+            List of BlockSearchResult if successful, None if index not available
+        """
+        try:
+            index = get_block_search_index()
+            if not index.load():
+                logger.info(
+                    "Block search index not available, falling back to simple search"
+                )
+                return None
+
+            results = index.search(query, top_k=10)
+            logger.info(f"Hybrid search found {len(results)} blocks for: {query}")
+            return results
+
+        except Exception as e:
+            logger.warning(f"Hybrid search failed, falling back to simple: {e}")
+            return None
+
+    def _simple_search(self, query: str, session_id: str) -> ToolResponseBase:
+        """Fallback simple search using substring matching."""
+        all_blocks = load_all_blocks()
+        logger.info(f"Simple searching {len(all_blocks)} blocks for: {query}")
+
+        # Find matching blocks with priority scores
+        matches: list[tuple[int, Any]] = []
+        for block_id, block_cls in all_blocks.items():
+            block = block_cls()
+            priority, is_match = self._matches_query(block, query)
+            if is_match:
+                matches.append((priority, block))
+
+        # Sort by priority (lower is better)
+        matches.sort(key=lambda x: x[0])
+
+        # Take top 10 results
+        top_matches = [block for _, block in matches[:10]]
+
+        if not top_matches:
+            return NoResultsResponse(
+                message=f"No blocks found matching '{query}'",
+                session_id=session_id,
+                suggestions=[
+                    "Try more general terms",
+                    "Search by category: ai, text, social, search, etc.",
+                    "Check block names like 'SendEmail', 'HttpRequest', etc.",
+                ],
+            )
+
+        # Build response
+        blocks = []
+        for block in top_matches:
+            blocks.append(
+                BlockInfoSummary(
+                    id=block.id,
+                    name=block.name,
+                    description=block.description,
+                    categories=[cat.name for cat in block.categories],
+                    input_schema=block.input_schema.jsonschema(),
+                    output_schema=block.output_schema.jsonschema(),
+                )
+            )
+
+        return BlockListResponse(
+            message=(
+                f"Found {len(blocks)} block{'s' if len(blocks) != 1 else ''} "
+                f"matching '{query}'. Use run_block to execute a block with "
+                "the required inputs."
+            ),
+            blocks=blocks,
+            count=len(blocks),
+            query=query,
+            session_id=session_id,
+        )

autogpt_platform/backend/backend/api/features/chat/tools/find_library_agent.py

@@ -1,14 +1,22 @@
 """Tool for searching agents in the user's library."""
 
+import logging
 from typing import Any
 
-from langfuse import observe
-
 from backend.api.features.chat.model import ChatSession
+from backend.api.features.library import db as library_db
+from backend.util.exceptions import DatabaseError
 
-from .agent_search import search_agents
 from .base import BaseTool
-from .models import ToolResponseBase
+from .models import (
+    AgentCarouselResponse,
+    AgentInfo,
+    ErrorResponse,
+    NoResultsResponse,
+    ToolResponseBase,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class FindLibraryAgentTool(BaseTool):
@@ -33,7 +41,10 @@ class FindLibraryAgentTool(BaseTool):
             "properties": {
                 "query": {
                     "type": "string",
-                    "description": "Search query to find agents by name or description.",
+                    "description": (
+                        "Search query to find agents by name or description. "
+                        "Use keywords for best results."
+                    ),
                 },
             },
             "required": ["query"],
@@ -43,13 +54,104 @@ class FindLibraryAgentTool(BaseTool):
     def requires_auth(self) -> bool:
         return True
 
-    @observe(as_type="tool", name="find_library_agent")
     async def _execute(
-        self, user_id: str | None, session: ChatSession, **kwargs
+        self,
+        user_id: str | None,
+        session: ChatSession,
+        **kwargs,
     ) -> ToolResponseBase:
-        return await search_agents(
-            query=kwargs.get("query", "").strip(),
-            source="library",
-            session_id=session.session_id,
-            user_id=user_id,
+        """Search for agents in the user's library.
+
+        Args:
+            user_id: User ID (required)
+            session: Chat session
+            query: Search query
+
+        Returns:
+            AgentCarouselResponse: List of agents found in the library
+            NoResultsResponse: No agents found
+            ErrorResponse: Error message
+        """
+        query = kwargs.get("query", "").strip()
+        session_id = session.session_id
+
+        if not query:
+            return ErrorResponse(
+                message="Please provide a search query",
+                session_id=session_id,
+            )
+
+        if not user_id:
+            return ErrorResponse(
+                message="User authentication required to search library",
+                session_id=session_id,
+            )
+
+        agents = []
+        try:
+            logger.info(f"Searching user library for: {query}")
+            library_results = await library_db.list_library_agents(
+                user_id=user_id,
+                search_term=query,
+                page_size=10,
+            )
+
+            logger.info(
+                f"Find library agents tool found {len(library_results.agents)} agents"
+            )
+
+            for agent in library_results.agents:
+                agents.append(
+                    AgentInfo(
+                        id=agent.id,
+                        name=agent.name,
+                        description=agent.description or "",
+                        source="library",
+                        in_library=True,
+                        creator=agent.creator_name,
+                        status=agent.status.value,
+                        can_access_graph=agent.can_access_graph,
+                        has_external_trigger=agent.has_external_trigger,
+                        new_output=agent.new_output,
+                        graph_id=agent.graph_id,
+                    ),
+                )
+
+        except DatabaseError as e:
+            logger.error(f"Error searching library agents: {e}", exc_info=True)
+            return ErrorResponse(
+                message="Failed to search library. Please try again.",
+                error=str(e),
+                session_id=session_id,
+            )
+
+        if not agents:
+            return NoResultsResponse(
+                message=(
+                    f"No agents found matching '{query}' in your library. "
+                    "Try different keywords or use find_agent to search the marketplace."
+                ),
+                session_id=session_id,
+                suggestions=[
+                    "Try more general terms",
+                    "Use find_agent to search the marketplace",
+                    "Check your library at /library",
+                ],
+            )
+
+        title = (
+            f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} "
+            f"in your library for '{query}'"
+        )
+
+        return AgentCarouselResponse(
+            message=(
+                "Found agents in the user's library. You can provide a link to "
+                "view an agent at: /library/agents/{agent_id}. "
+                "Use agent_output to get execution results, or run_agent to execute."
+            ),
+            title=title,
+            agents=agents,
+            count=len(agents),
+            session_id=session_id,
         )

autogpt_platform/backend/backend/api/features/chat/tools/get_doc_page.py

@@ -1,151 +0,0 @@
-"""GetDocPageTool - Fetch full content of a documentation page."""
-
-import logging
-from pathlib import Path
-from typing import Any
-
-from langfuse import observe
-
-from backend.api.features.chat.model import ChatSession
-from backend.api.features.chat.tools.base import BaseTool
-from backend.api.features.chat.tools.models import (
-    DocPageResponse,
-    ErrorResponse,
-    ToolResponseBase,
-)
-
-logger = logging.getLogger(__name__)
-
-# Base URL for documentation (can be configured)
-DOCS_BASE_URL = "https://docs.agpt.co"
-
-
-class GetDocPageTool(BaseTool):
-    """Tool for fetching full content of a documentation page."""
-
-    @property
-    def name(self) -> str:
-        return "get_doc_page"
-
-    @property
-    def description(self) -> str:
-        return (
-            "Get the full content of a documentation page by its path. "
-            "Use this after search_docs to read the complete content of a relevant page."
-        )
-
-    @property
-    def parameters(self) -> dict[str, Any]:
-        return {
-            "type": "object",
-            "properties": {
-                "path": {
-                    "type": "string",
-                    "description": (
-                        "The path to the documentation file, as returned by search_docs. "
-                        "Example: 'platform/block-sdk-guide.md'"
-                    ),
-                },
-            },
-            "required": ["path"],
-        }
-
-    @property
-    def requires_auth(self) -> bool:
-        return False  # Documentation is public
-
-    def _get_docs_root(self) -> Path:
-        """Get the documentation root directory."""
-        this_file = Path(__file__)
-        project_root = this_file.parent.parent.parent.parent.parent.parent.parent.parent
-        return project_root / "docs"
-
-    def _extract_title(self, content: str, fallback: str) -> str:
-        """Extract title from markdown content."""
-        lines = content.split("\n")
-        for line in lines:
-            if line.startswith("# "):
-                return line[2:].strip()
-        return fallback
-
-    def _make_doc_url(self, path: str) -> str:
-        """Create a URL for a documentation page."""
-        url_path = path.rsplit(".", 1)[0] if "." in path else path
-        return f"{DOCS_BASE_URL}/{url_path}"
-
-    @observe(as_type="tool", name="get_doc_page")
-    async def _execute(
-        self,
-        user_id: str | None,
-        session: ChatSession,
-        **kwargs,
-    ) -> ToolResponseBase:
-        """Fetch full content of a documentation page.
-
-        Args:
-            user_id: User ID (not required for docs)
-            session: Chat session
-            path: Path to the documentation file
-
-        Returns:
-            DocPageResponse: Full document content
-            ErrorResponse: Error message
-        """
-        path = kwargs.get("path", "").strip()
-        session_id = session.session_id if session else None
-
-        if not path:
-            return ErrorResponse(
-                message="Please provide a documentation path.",
-                error="Missing path parameter",
-                session_id=session_id,
-            )
-
-        # Sanitize path to prevent directory traversal
-        if ".." in path or path.startswith("/"):
-            return ErrorResponse(
-                message="Invalid documentation path.",
-                error="invalid_path",
-                session_id=session_id,
-            )
-
-        docs_root = self._get_docs_root()
-        full_path = docs_root / path
-
-        if not full_path.exists():
-            return ErrorResponse(
-                message=f"Documentation page not found: {path}",
-                error="not_found",
-                session_id=session_id,
-            )
-
-        # Ensure the path is within docs root
-        try:
-            full_path.resolve().relative_to(docs_root.resolve())
-        except ValueError:
-            return ErrorResponse(
-                message="Invalid documentation path.",
-                error="invalid_path",
-                session_id=session_id,
-            )
-
-        try:
-            content = full_path.read_text(encoding="utf-8")
-            title = self._extract_title(content, path)
-
-            return DocPageResponse(
-                message=f"Retrieved documentation page: {title}",
-                title=title,
-                path=path,
-                content=content,
-                doc_url=self._make_doc_url(path),
-                session_id=session_id,
-            )
-
-        except Exception as e:
-            logger.error(f"Failed to read documentation page {path}: {e}")
-            return ErrorResponse(
-                message=f"Failed to read documentation page: {str(e)}",
-                error="read_failed",
-                session_id=session_id,
-            )

autogpt_platform/backend/backend/api/features/chat/tools/index_blocks.py

@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+"""
+Block Indexer for Hybrid Search
+
+Creates a hybrid search index from blocks:
+- OpenAI embeddings (text-embedding-3-small)
+- BM25 index for lexical search
+- Name index for title matching boost
+
+Supports incremental updates by tracking content hashes.
+
+Usage:
+    python -m backend.server.v2.chat.tools.index_blocks [--force]
+"""
+
+import argparse
+import base64
+import hashlib
+import json
+import logging
+import os
+import re
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+# Check for OpenAI availability
+try:
+    import openai  # noqa: F401
+
+    HAS_OPENAI = True
+except ImportError:
+    HAS_OPENAI = False
+    print("Warning: openai not installed. Run: pip install openai")
+
+# Default embedding model (OpenAI)
+DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
+DEFAULT_EMBEDDING_DIM = 1536
+
+# Output path (relative to this file)
+INDEX_PATH = Path(__file__).parent / "blocks_index.json"
+
+# Stopwords for tokenization
+STOPWORDS = {
+    "the",
+    "a",
+    "an",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "could",
+    "should",
+    "may",
+    "might",
+    "must",
+    "shall",
+    "can",
+    "need",
+    "dare",
+    "ought",
+    "used",
+    "to",
+    "of",
+    "in",
+    "for",
+    "on",
+    "with",
+    "at",
+    "by",
+    "from",
+    "as",
+    "into",
+    "through",
+    "during",
+    "before",
+    "after",
+    "above",
+    "below",
+    "between",
+    "under",
+    "again",
+    "further",
+    "then",
+    "once",
+    "and",
+    "but",
+    "or",
+    "nor",
+    "so",
+    "yet",
+    "both",
+    "either",
+    "neither",
+    "not",
+    "only",
+    "own",
+    "same",
+    "than",
+    "too",
+    "very",
+    "just",
+    "also",
+    "now",
+    "here",
+    "there",
+    "when",
+    "where",
+    "why",
+    "how",
+    "all",
+    "each",
+    "every",
+    "few",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "no",
+    "any",
+    "this",
+    "that",
+    "these",
+    "those",
+    "it",
+    "its",
+    "block",  # Too common in block context
+}
+
+
+def tokenize(text: str) -> list[str]:
+    """Simple tokenizer for BM25."""
+    text = text.lower()
+    # Remove code blocks if any
+    text = re.sub(r"```[\s\S]*?```", "", text)
+    text = re.sub(r"`[^`]+`", "", text)
+    # Extract words (including camelCase split)
+    # First, split camelCase
+    text = re.sub(r"([a-z])([A-Z])", r"\1 \2", text)
+    # Extract words
+    words = re.findall(r"\b[a-z][a-z0-9_-]*\b", text)
+    # Remove very short words and stopwords
+    return [w for w in words if len(w) > 2 and w not in STOPWORDS]
+
+
+def build_searchable_text(block: Any) -> str:
+    """Build searchable text from block attributes."""
+    parts = []
+
+    # Block name (split camelCase for better tokenization)
+    name = block.name
+    # Split camelCase: GetCurrentTimeBlock -> Get Current Time Block
+    name_split = re.sub(r"([a-z])([A-Z])", r"\1 \2", name)
+    parts.append(name_split)
+
+    # Description
+    if block.description:
+        parts.append(block.description)
+
+    # Categories
+    for category in block.categories:
+        parts.append(category.name)
+
+    # Input schema field names and descriptions
+    try:
+        input_schema = block.input_schema.jsonschema()
+        if "properties" in input_schema:
+            for field_name, field_info in input_schema["properties"].items():
+                parts.append(field_name)
+                if "description" in field_info:
+                    parts.append(field_info["description"])
+    except Exception:
+        pass
+
+    # Output schema field names
+    try:
+        output_schema = block.output_schema.jsonschema()
+        if "properties" in output_schema:
+            for field_name in output_schema["properties"]:
+                parts.append(field_name)
+    except Exception:
+        pass
+
+    return " ".join(parts)
+
+
+def compute_content_hash(text: str) -> str:
+    """Compute MD5 hash of text for change detection."""
+    return hashlib.md5(text.encode()).hexdigest()
+
+
+def load_existing_index(index_path: Path) -> dict[str, Any] | None:
+    """Load existing index if present."""
+    if not index_path.exists():
+        return None
+
+    try:
+        with open(index_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        logger.warning(f"Failed to load existing index: {e}")
+        return None
+
+
+def create_embeddings(
+    texts: list[str],
+    model_name: str = DEFAULT_EMBEDDING_MODEL,
+    batch_size: int = 100,
+) -> np.ndarray:
+    """Create embeddings using OpenAI API."""
+    if not HAS_OPENAI:
+        raise RuntimeError("openai not installed. Run: pip install openai")
+
+    # Import here to satisfy type checker
+    from openai import OpenAI
+
+    # Check for API key
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("OPENAI_API_KEY environment variable not set")
+
+    client = OpenAI(api_key=api_key)
+    embeddings = []
+
+    print(f"Creating embeddings for {len(texts)} texts using {model_name}...")
+
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i : i + batch_size]
+        # Truncate texts to max token limit (8191 tokens for text-embedding-3-small)
+        # Roughly 4 chars per token, so ~32000 chars max
+        batch = [text[:32000] for text in batch]
+
+        response = client.embeddings.create(
+            model=model_name,
+            input=batch,
+        )
+
+        for embedding_data in response.data:
+            embeddings.append(embedding_data.embedding)
+
+        print(f"  Processed {min(i + batch_size, len(texts))}/{len(texts)} texts")
+
+    return np.array(embeddings, dtype=np.float32)
+
+
+def build_bm25_data(
+    blocks_data: list[dict[str, Any]],
+) -> dict[str, Any]:
+    """Build BM25 metadata from block data."""
+    # Tokenize all searchable texts
+    tokenized_docs = []
+    for block in blocks_data:
+        tokens = tokenize(block["searchable_text"])
+        tokenized_docs.append(tokens)
+
+    # Calculate document frequencies
+    doc_freq: dict[str, int] = {}
+    for tokens in tokenized_docs:
+        seen = set()
+        for token in tokens:
+            if token not in seen:
+                doc_freq[token] = doc_freq.get(token, 0) + 1
+                seen.add(token)
+
+    n_docs = len(tokenized_docs)
+    doc_lens = [len(d) for d in tokenized_docs]
+    avgdl = sum(doc_lens) / max(n_docs, 1)
+
+    return {
+        "n_docs": n_docs,
+        "avgdl": avgdl,
+        "df": doc_freq,
+        "doc_lens": doc_lens,
+    }
+
+
+def build_name_index(
+    blocks_data: list[dict[str, Any]],
+) -> dict[str, list[list[int | float]]]:
+    """Build inverted index for name search boost."""
+    index: dict[str, list[list[int | float]]] = defaultdict(list)
+
+    for idx, block in enumerate(blocks_data):
+        # Tokenize block name
+        name_tokens = tokenize(block["name"])
+        seen = set()
+
+        for i, token in enumerate(name_tokens):
+            if token in seen:
+                continue
+            seen.add(token)
+
+            # Score: first token gets higher weight
+            score = 1.5 if i == 0 else 1.0
+            index[token].append([idx, score])
+
+    return dict(index)
+
+
+def build_block_index(
+    force_rebuild: bool = False,
+    output_path: Path = INDEX_PATH,
+) -> dict[str, Any]:
+    """
+    Build the block search index.
+
+    Args:
+        force_rebuild: If True, rebuild all embeddings even if unchanged
+        output_path: Path to save the index
+
+    Returns:
+        The generated index dictionary
+    """
+    # Import here to avoid circular imports
+    from backend.blocks import load_all_blocks
+
+    print("Loading all blocks...")
+    all_blocks = load_all_blocks()
+    print(f"Found {len(all_blocks)} blocks")
+
+    # Load existing index for incremental updates
+    existing_index = None if force_rebuild else load_existing_index(output_path)
+    existing_blocks: dict[str, dict[str, Any]] = {}
+
+    if existing_index:
+        print(
+            f"Loaded existing index with {len(existing_index.get('blocks', []))} blocks"
+        )
+        for block in existing_index.get("blocks", []):
+            existing_blocks[block["id"]] = block
+
+    # Process each block
+    blocks_data: list[dict[str, Any]] = []
+    blocks_needing_embedding: list[tuple[int, str]] = []  # (index, searchable_text)
+
+    for block_id, block_cls in all_blocks.items():
+        try:
+            block = block_cls()
+
+            # Skip disabled blocks
+            if block.disabled:
+                continue
+
+            searchable_text = build_searchable_text(block)
+            content_hash = compute_content_hash(searchable_text)
+
+            block_data = {
+                "id": block.id,
+                "name": block.name,
+                "description": block.description,
+                "categories": [cat.name for cat in block.categories],
+                "searchable_text": searchable_text,
+                "content_hash": content_hash,
+                "emb": None,  # Will be filled later
+            }
+
+            # Check if we can reuse existing embedding
+            if (
+                block.id in existing_blocks
+                and existing_blocks[block.id].get("content_hash") == content_hash
+                and existing_blocks[block.id].get("emb")
+            ):
+                # Reuse existing embedding
+                block_data["emb"] = existing_blocks[block.id]["emb"]
+            else:
+                # Need new embedding
+                blocks_needing_embedding.append((len(blocks_data), searchable_text))
+
+            blocks_data.append(block_data)
+
+        except Exception as e:
+            logger.warning(f"Failed to process block {block_id}: {e}")
+            continue
+
+    print(f"Processed {len(blocks_data)} blocks")
+    print(f"Blocks needing new embeddings: {len(blocks_needing_embedding)}")
+
+    # Create embeddings for new/changed blocks
+    if blocks_needing_embedding and HAS_OPENAI:
+        texts_to_embed = [text for _, text in blocks_needing_embedding]
+        try:
+            embeddings = create_embeddings(texts_to_embed)
+
+            # Assign embeddings to blocks
+            for i, (block_idx, _) in enumerate(blocks_needing_embedding):
+                emb = embeddings[i].astype(np.float32)
+                # Encode as base64
+                blocks_data[block_idx]["emb"] = base64.b64encode(emb.tobytes()).decode(
+                    "ascii"
+                )
+        except Exception as e:
+            print(f"Warning: Failed to create embeddings: {e}")
+    elif blocks_needing_embedding:
+        print(
+            "Warning: Cannot create embeddings (openai not installed or OPENAI_API_KEY not set)"
+        )
+
+    # Build BM25 data
+    print("Building BM25 index...")
+    bm25_data = build_bm25_data(blocks_data)
+
+    # Build name index
+    print("Building name index...")
+    name_index = build_name_index(blocks_data)
+
+    # Build final index
+    index = {
+        "version": "1.0.0",
+        "embedding_model": DEFAULT_EMBEDDING_MODEL,
+        "embedding_dim": DEFAULT_EMBEDDING_DIM,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "blocks": blocks_data,
+        "bm25": bm25_data,
+        "name_index": name_index,
+    }
+
+    # Save index
+    print(f"Saving index to {output_path}...")
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(index, f, separators=(",", ":"))
+
+    size_kb = output_path.stat().st_size / 1024
+    print(f"Index saved ({size_kb:.1f} KB)")
+
+    # Print statistics
+    print("\nIndex Statistics:")
+    print(f"  Blocks indexed: {len(blocks_data)}")
+    print(f"  BM25 vocabulary size: {len(bm25_data['df'])}")
+    print(f"  Name index terms: {len(name_index)}")
+    print(f"  Embeddings: {'Yes' if any(b.get('emb') for b in blocks_data) else 'No'}")
+
+    return index
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build hybrid search index for blocks")
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force rebuild all embeddings even if unchanged",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=INDEX_PATH,
+        help=f"Output index file path (default: {INDEX_PATH})",
+    )
+
+    args = parser.parse_args()
+
+    try:
+        build_block_index(
+            force_rebuild=args.force,
+            output_path=args.output,
+        )
+    except Exception as e:
+        print(f"Error building index: {e}")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()