chore: clean ups

- MarkdownRenderer: add null guard for empty image src
- GenericTool: use composite key for todo items
- ViewAgentOutput/BlockOutputCard: use parent key + index instead of bare index
- SidebarItemCard: extract onKeyDown to named function declaration

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/platform-frontend-ci.yml

@@ -83,6 +83,65 @@ jobs:
       - name: Run lint
         run: pnpm lint
 
+  react-doctor:
+    runs-on: ubuntu-latest
+    needs: setup
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: "22.18.0"
+          cache: "pnpm"
+          cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run React Doctor
+        id: react-doctor
+        continue-on-error: true
+        run: |
+          OUTPUT=$(pnpm react-doctor:diff 2>&1) || true
+          echo "$OUTPUT"
+          SCORE=$(echo "$OUTPUT" | grep -oP '\d+(?= / 100)' | head -1)
+          echo "score=${SCORE:-0}" >> "$GITHUB_OUTPUT"
+
+      - name: Check React Doctor score
+        env:
+          RD_SCORE: ${{ steps.react-doctor.outputs.score }}
+          MIN_SCORE: "90"
+        run: |
+          echo "React Doctor score: ${RD_SCORE}/100 (minimum: ${MIN_SCORE})"
+          if [ "${RD_SCORE}" -lt "${MIN_SCORE}" ]; then
+            echo "::error::React Doctor score ${RD_SCORE} is below the minimum threshold of ${MIN_SCORE}."
+            echo ""
+            echo "=========================================="
+            echo "  React Doctor score too low!"
+            echo "=========================================="
+            echo ""
+            echo "To fix these issues, run Claude Code locally:"
+            echo ""
+            echo "  cd autogpt_platform/frontend"
+            echo "  claude"
+            echo ""
+            echo "Then ask Claude to run react-doctor and fix the issues."
+            echo "You can also run it manually:"
+            echo ""
+            echo "  pnpm react-doctor       # scan all files"
+            echo "  pnpm react-doctor:diff  # scan only changed files"
+            echo ""
+            exit 1
+          fi
+
   chromatic:
     runs-on: ubuntu-latest
     needs: setup

.gitignore

@@ -180,6 +180,4 @@ autogpt_platform/backend/settings.py
 .claude/settings.local.json
 CLAUDE.local.md
 /autogpt_platform/backend/logs
-.next
-# Implementation plans (generated by AI agents)
-plans/
+.next

.pre-commit-config.yaml

@@ -1,10 +1,3 @@
-default_install_hook_types:
-  - pre-commit
-  - pre-push
-  - post-checkout
-
-default_stages: [pre-commit]
-
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
@@ -24,7 +17,6 @@ repos:
         name: Detect secrets
         description: Detects high entropy strings that are likely to be passwords.
         files: ^autogpt_platform/
-        exclude: pnpm-lock\.yaml$
         stages: [pre-push]
 
   - repo: local
@@ -34,106 +26,49 @@ repos:
       - id: poetry-install
         name: Check & Install dependencies - AutoGPT Platform - Backend
         alias: poetry-install-platform-backend
+        entry: poetry -C autogpt_platform/backend install
         # include autogpt_libs source (since it's a path dependency)
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^autogpt_platform/(backend|autogpt_libs)/poetry\.lock$" || exit 0;
-          poetry -C autogpt_platform/backend install
-          '
-        always_run: true
+        files: ^autogpt_platform/(backend|autogpt_libs)/poetry\.lock$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
       - id: poetry-install
         name: Check & Install dependencies - AutoGPT Platform - Libs
         alias: poetry-install-platform-libs
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^autogpt_platform/autogpt_libs/poetry\.lock$" || exit 0;
-          poetry -C autogpt_platform/autogpt_libs install
-          '
-        always_run: true
+        entry: poetry -C autogpt_platform/autogpt_libs install
+        files: ^autogpt_platform/autogpt_libs/poetry\.lock$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
-
-      - id: pnpm-install
-        name: Check & Install dependencies - AutoGPT Platform - Frontend
-        alias: pnpm-install-platform-frontend
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^autogpt_platform/frontend/pnpm-lock\.yaml$" || exit 0;
-          pnpm --prefix autogpt_platform/frontend install
-          '
-        always_run: true
-        language: system
-        pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
       - id: poetry-install
         name: Check & Install dependencies - Classic - AutoGPT
         alias: poetry-install-classic-autogpt
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^classic/(original_autogpt|forge)/poetry\.lock$" || exit 0;
-          poetry -C classic/original_autogpt install
-          '
+        entry: poetry -C classic/original_autogpt install
         # include forge source (since it's a path dependency)
-        always_run: true
+        files: ^classic/(original_autogpt|forge)/poetry\.lock$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
       - id: poetry-install
         name: Check & Install dependencies - Classic - Forge
         alias: poetry-install-classic-forge
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^classic/forge/poetry\.lock$" || exit 0;
-          poetry -C classic/forge install
-          '
-        always_run: true
+        entry: poetry -C classic/forge install
+        files: ^classic/forge/poetry\.lock$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
       - id: poetry-install
         name: Check & Install dependencies - Classic - Benchmark
         alias: poetry-install-classic-benchmark
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^classic/benchmark/poetry\.lock$" || exit 0;
-          poetry -C classic/benchmark install
-          '
-        always_run: true
+        entry: poetry -C classic/benchmark install
+        files: ^classic/benchmark/poetry\.lock$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
   - repo: local
     # For proper type checking, Prisma client must be up-to-date.
@@ -141,54 +76,12 @@ repos:
       - id: prisma-generate
         name: Prisma Generate - AutoGPT Platform - Backend
         alias: prisma-generate-platform-backend
-        entry: >
-          bash -c '
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --name-only "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF"
-          else
-            git diff --cached --name-only
-          fi | grep -qE "^autogpt_platform/((backend|autogpt_libs)/poetry\.lock|backend/schema\.prisma)$" || exit 0;
-          cd autogpt_platform/backend
-          && poetry run prisma generate
-          && poetry run gen-prisma-stub
-          '
+        entry: bash -c 'cd autogpt_platform/backend && poetry run prisma generate'
         # include everything that triggers poetry install + the prisma schema
-        always_run: true
+        files: ^autogpt_platform/((backend|autogpt_libs)/poetry\.lock|backend/schema.prisma)$
+        types: [file]
         language: system
         pass_filenames: false
-        stages: [pre-commit, post-checkout]
-
-      - id: export-api-schema
-        name: Export API schema - AutoGPT Platform - Backend -> Frontend
-        alias: export-api-schema-platform
-        entry: >
-          bash -c '
-          cd autogpt_platform/backend
-          && poetry run export-api-schema --output ../frontend/src/app/api/openapi.json
-          && cd ../frontend
-          && pnpm prettier --write ./src/app/api/openapi.json
-          '
-        files: ^autogpt_platform/backend/
-        language: system
-        pass_filenames: false
-
-      - id: generate-api-client
-        name: Generate API client - AutoGPT Platform - Frontend
-        alias: generate-api-client-platform-frontend
-        entry: >
-          bash -c '
-          SCHEMA=autogpt_platform/frontend/src/app/api/openapi.json;
-          if [ -n "$PRE_COMMIT_FROM_REF" ]; then
-            git diff --quiet "$PRE_COMMIT_FROM_REF" "$PRE_COMMIT_TO_REF" -- "$SCHEMA" && exit 0
-          else
-            git diff --quiet HEAD -- "$SCHEMA" && exit 0
-          fi;
-          cd autogpt_platform/frontend && pnpm generate:api
-          '
-        always_run: true
-        language: system
-        pass_filenames: false
-        stages: [pre-commit, post-checkout]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.7.2

autogpt_platform/backend/.env.default

@@ -190,8 +190,5 @@ ZEROBOUNCE_API_KEY=
 POSTHOG_API_KEY=
 POSTHOG_HOST=https://eu.i.posthog.com
 
-# Tally Form Integration (pre-populate business understanding on signup)
-TALLY_API_KEY=
-
 # Other Services
 AUTOMOD_API_KEY=

autogpt_platform/backend/backend/api/external/middleware.py

@@ -88,23 +88,20 @@ async def require_auth(
     )
 
 
-def require_permission(*permissions: APIKeyPermission):
+def require_permission(permission: APIKeyPermission):
     """
-    Dependency function for checking required permissions.
-    All listed permissions must be present.
+    Dependency function for checking specific permissions
     (works with API keys and OAuth tokens)
     """
 
-    async def check_permissions(
+    async def check_permission(
         auth: APIAuthorizationInfo = Security(require_auth),
     ) -> APIAuthorizationInfo:
-        missing = [p for p in permissions if p not in auth.scopes]
-        if missing:
+        if permission not in auth.scopes:
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
-                detail=f"Missing required permission(s): "
-                f"{', '.join(p.value for p in missing)}",
+                detail=f"Missing required permission: {permission.value}",
             )
         return auth
 
-    return check_permissions
+    return check_permission

autogpt_platform/backend/backend/api/external/v1/routes.py

@@ -18,7 +18,6 @@ from backend.data import user as user_db
 from backend.data.auth.base import APIAuthorizationInfo
 from backend.data.block import BlockInput, CompletedBlockOutput
 from backend.executor.utils import add_graph_execution
-from backend.integrations.webhooks.graph_lifecycle_hooks import on_graph_activate
 from backend.util.settings import Settings
 
 from .integrations import integrations_router
@@ -96,43 +95,6 @@ async def execute_graph_block(
     return output
 
 
-@v1_router.post(
-    path="/graphs",
-    tags=["graphs"],
-    status_code=201,
-    dependencies=[
-        Security(
-            require_permission(
-                APIKeyPermission.WRITE_GRAPH, APIKeyPermission.WRITE_LIBRARY
-            )
-        )
-    ],
-)
-async def create_graph(
-    graph: graph_db.Graph,
-    auth: APIAuthorizationInfo = Security(
-        require_permission(APIKeyPermission.WRITE_GRAPH, APIKeyPermission.WRITE_LIBRARY)
-    ),
-) -> graph_db.GraphModel:
-    """
-    Create a new agent graph.
-
-    The graph will be validated and assigned a new ID.
-    It is automatically added to the user's library.
-    """
-    from backend.api.features.library import db as library_db
-
-    graph_model = graph_db.make_graph_model(graph, auth.user_id)
-    graph_model.reassign_ids(user_id=auth.user_id, reassign_graph_id=True)
-    graph_model.validate_graph(for_run=False)
-
-    await graph_db.create_graph(graph_model, user_id=auth.user_id)
-    await library_db.create_library_agent(graph_model, auth.user_id)
-    activated_graph = await on_graph_activate(graph_model, user_id=auth.user_id)
-
-    return activated_graph
-
-
 @v1_router.post(
     path="/graphs/{graph_id}/execute/{graph_version}",
     tags=["graphs"],

autogpt_platform/backend/backend/api/features/builder/db.py

@@ -1,17 +1,15 @@
 import logging
 from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
 from difflib import SequenceMatcher
-from typing import Any, Sequence, get_args, get_origin
+from typing import Sequence
 
 import prisma
-from prisma.enums import ContentType
-from prisma.models import mv_suggested_blocks
 
 import backend.api.features.library.db as library_db
 import backend.api.features.library.model as library_model
 import backend.api.features.store.db as store_db
 import backend.api.features.store.model as store_model
-from backend.api.features.store.hybrid_search import unified_hybrid_search
 from backend.blocks import load_all_blocks
 from backend.blocks._base import (
     AnyBlockSchema,
@@ -21,6 +19,7 @@ from backend.blocks._base import (
     BlockType,
 )
 from backend.blocks.llm import LlmModel
+from backend.data.db import query_raw_with_schema
 from backend.integrations.providers import ProviderName
 from backend.util.cache import cached
 from backend.util.models import Pagination
@@ -43,16 +42,6 @@ MAX_LIBRARY_AGENT_RESULTS = 100
 MAX_MARKETPLACE_AGENT_RESULTS = 100
 MIN_SCORE_FOR_FILTERED_RESULTS = 10.0
 
-# Boost blocks over marketplace agents in search results
-BLOCK_SCORE_BOOST = 50.0
-
-# Block IDs to exclude from search results
-EXCLUDED_BLOCK_IDS = frozenset(
-    {
-        "e189baac-8c20-45a1-94a7-55177ea42565",  # AgentExecutorBlock
-    }
-)
-
 SearchResultItem = BlockInfo | library_model.LibraryAgent | store_model.StoreAgent
 
 
@@ -75,8 +64,8 @@ def get_block_categories(category_blocks: int = 3) -> list[BlockCategoryResponse
 
     for block_type in load_all_blocks().values():
         block: AnyBlockSchema = block_type()
-        # Skip disabled and excluded blocks
-        if block.disabled or block.id in EXCLUDED_BLOCK_IDS:
+        # Skip disabled blocks
+        if block.disabled:
             continue
         # Skip blocks that don't have categories (all should have at least one)
         if not block.categories:
@@ -127,9 +116,6 @@ def get_blocks(
         # Skip disabled blocks
         if block.disabled:
             continue
-        # Skip excluded blocks
-        if block.id in EXCLUDED_BLOCK_IDS:
-            continue
         # Skip blocks that don't match the category
         if category and category not in {c.name.lower() for c in block.categories}:
             continue
@@ -269,25 +255,14 @@ async def _build_cached_search_results(
         "my_agents": 0,
     }
 
-    # Use hybrid search when query is present, otherwise list all blocks
-    if (include_blocks or include_integrations) and normalized_query:
-        block_results, block_total, integration_total = await _hybrid_search_blocks(
-            query=search_query,
-            include_blocks=include_blocks,
-            include_integrations=include_integrations,
-        )
-        scored_items.extend(block_results)
-        total_items["blocks"] = block_total
-        total_items["integrations"] = integration_total
-    elif include_blocks or include_integrations:
-        # No query - list all blocks using in-memory approach
-        block_results, block_total, integration_total = _collect_block_results(
-            include_blocks=include_blocks,
-            include_integrations=include_integrations,
-        )
-        scored_items.extend(block_results)
-        total_items["blocks"] = block_total
-        total_items["integrations"] = integration_total
+    block_results, block_total, integration_total = _collect_block_results(
+        normalized_query=normalized_query,
+        include_blocks=include_blocks,
+        include_integrations=include_integrations,
+    )
+    scored_items.extend(block_results)
+    total_items["blocks"] = block_total
+    total_items["integrations"] = integration_total
 
     if include_library_agents:
         library_response = await library_db.list_library_agents(
@@ -332,14 +307,10 @@ async def _build_cached_search_results(
 
 def _collect_block_results(
     *,
+    normalized_query: str,
     include_blocks: bool,
     include_integrations: bool,
 ) -> tuple[list[_ScoredItem], int, int]:
-    """
-    Collect all blocks for listing (no search query).
-
-    All blocks get BLOCK_SCORE_BOOST to prioritize them over marketplace agents.
-    """
     results: list[_ScoredItem] = []
     block_count = 0
     integration_count = 0
@@ -352,10 +323,6 @@ def _collect_block_results(
         if block.disabled:
             continue
 
-        # Skip excluded blocks
-        if block.id in EXCLUDED_BLOCK_IDS:
-            continue
-
         block_info = block.get_info()
         credentials = list(block.input_schema.get_credentials_fields().values())
         is_integration = len(credentials) > 0
@@ -365,6 +332,10 @@ def _collect_block_results(
         if not is_integration and not include_blocks:
             continue
 
+        score = _score_block(block, block_info, normalized_query)
+        if not _should_include_item(score, normalized_query):
+            continue
+
         filter_type: FilterType = "integrations" if is_integration else "blocks"
         if is_integration:
             integration_count += 1
@@ -375,122 +346,8 @@ def _collect_block_results(
             _ScoredItem(
                 item=block_info,
                 filter_type=filter_type,
-                score=BLOCK_SCORE_BOOST,
-                sort_key=block_info.name.lower(),
-            )
-        )
-
-    return results, block_count, integration_count
-
-
-async def _hybrid_search_blocks(
-    *,
-    query: str,
-    include_blocks: bool,
-    include_integrations: bool,
-) -> tuple[list[_ScoredItem], int, int]:
-    """
-    Search blocks using hybrid search with builder-specific filtering.
-
-    Uses unified_hybrid_search for semantic + lexical search, then applies
-    post-filtering for block/integration types and scoring adjustments.
-
-    Scoring:
-        - Base: hybrid relevance score (0-1) scaled to 0-100, plus BLOCK_SCORE_BOOST
-          to prioritize blocks over marketplace agents in combined results
-        - +30 for exact name match, +15 for prefix name match
-        - +20 if the block has an LlmModel field and the query matches an LLM model name
-
-    Args:
-        query: The search query string
-        include_blocks: Whether to include regular blocks
-        include_integrations: Whether to include integration blocks
-
-    Returns:
-        Tuple of (scored_items, block_count, integration_count)
-    """
-    results: list[_ScoredItem] = []
-    block_count = 0
-    integration_count = 0
-
-    if not include_blocks and not include_integrations:
-        return results, block_count, integration_count
-
-    normalized_query = query.strip().lower()
-
-    # Fetch more results to account for post-filtering
-    search_results, _ = await unified_hybrid_search(
-        query=query,
-        content_types=[ContentType.BLOCK],
-        page=1,
-        page_size=150,
-        min_score=0.10,
-    )
-
-    # Load all blocks for getting BlockInfo
-    all_blocks = load_all_blocks()
-
-    for result in search_results:
-        block_id = result["content_id"]
-
-        # Skip excluded blocks
-        if block_id in EXCLUDED_BLOCK_IDS:
-            continue
-
-        metadata = result.get("metadata", {})
-        hybrid_score = result.get("relevance", 0.0)
-
-        # Get the actual block class
-        if block_id not in all_blocks:
-            continue
-
-        block_cls = all_blocks[block_id]
-        block: AnyBlockSchema = block_cls()
-
-        if block.disabled:
-            continue
-
-        # Check block/integration filter using metadata
-        is_integration = metadata.get("is_integration", False)
-
-        if is_integration and not include_integrations:
-            continue
-        if not is_integration and not include_blocks:
-            continue
-
-        # Get block info
-        block_info = block.get_info()
-
-        # Calculate final score: scale hybrid score and add builder-specific bonuses
-        # Hybrid scores are 0-1, builder scores were 0-200+
-        # Add BLOCK_SCORE_BOOST to prioritize blocks over marketplace agents
-        final_score = hybrid_score * 100 + BLOCK_SCORE_BOOST
-
-        # Add LLM model match bonus
-        has_llm_field = metadata.get("has_llm_model_field", False)
-        if has_llm_field and _matches_llm_model(block.input_schema, normalized_query):
-            final_score += 20
-
-        # Add exact/prefix match bonus for deterministic tie-breaking
-        name = block_info.name.lower()
-        if name == normalized_query:
-            final_score += 30
-        elif name.startswith(normalized_query):
-            final_score += 15
-
-        # Track counts
-        filter_type: FilterType = "integrations" if is_integration else "blocks"
-        if is_integration:
-            integration_count += 1
-        else:
-            block_count += 1
-
-        results.append(
-            _ScoredItem(
-                item=block_info,
-                filter_type=filter_type,
-                score=final_score,
-                sort_key=name,
+                score=score,
+                sort_key=_get_item_name(block_info),
             )
         )
 
@@ -615,8 +472,6 @@ async def _get_static_counts():
         block: AnyBlockSchema = block_type()
         if block.disabled:
             continue
-        if block.id in EXCLUDED_BLOCK_IDS:
-            continue
 
         all_blocks += 1
 
@@ -643,25 +498,47 @@ async def _get_static_counts():
     }
 
 
-def _contains_type(annotation: Any, target: type) -> bool:
-    """Check if an annotation is or contains the target type (handles Optional/Union/Annotated)."""
-    if annotation is target:
-        return True
-    origin = get_origin(annotation)
-    if origin is None:
-        return False
-    return any(_contains_type(arg, target) for arg in get_args(annotation))
-
-
 def _matches_llm_model(schema_cls: type[BlockSchema], query: str) -> bool:
     for field in schema_cls.model_fields.values():
-        if _contains_type(field.annotation, LlmModel):
+        if field.annotation == LlmModel:
             # Check if query matches any value in llm_models
             if any(query in name for name in llm_models):
                 return True
     return False
 
 
+def _score_block(
+    block: AnyBlockSchema,
+    block_info: BlockInfo,
+    normalized_query: str,
+) -> float:
+    if not normalized_query:
+        return 0.0
+
+    name = block_info.name.lower()
+    description = block_info.description.lower()
+    score = _score_primary_fields(name, description, normalized_query)
+
+    category_text = " ".join(
+        category.get("category", "").lower() for category in block_info.categories
+    )
+    score += _score_additional_field(category_text, normalized_query, 12, 6)
+
+    credentials_info = block.input_schema.get_credentials_fields_info().values()
+    provider_names = [
+        provider.value.lower()
+        for info in credentials_info
+        for provider in info.provider
+    ]
+    provider_text = " ".join(provider_names)
+    score += _score_additional_field(provider_text, normalized_query, 15, 6)
+
+    if _matches_llm_model(block.input_schema, normalized_query):
+        score += 20
+
+    return score
+
+
 def _score_library_agent(
     agent: library_model.LibraryAgent,
     normalized_query: str,
@@ -768,20 +645,31 @@ def _get_all_providers() -> dict[ProviderName, Provider]:
     return providers
 
 
-@cached(ttl_seconds=3600, shared_cache=True)
+@cached(ttl_seconds=3600)
 async def get_suggested_blocks(count: int = 5) -> list[BlockInfo]:
-    """Return the most-executed blocks from the last 14 days.
+    suggested_blocks = []
+    # Sum the number of executions for each block type
+    # Prisma cannot group by nested relations, so we do a raw query
+    # Calculate the cutoff timestamp
+    timestamp_threshold = datetime.now(timezone.utc) - timedelta(days=30)
 
-    Queries the mv_suggested_blocks materialized view (refreshed hourly via pg_cron)
-    and returns the top `count` blocks sorted by execution count, excluding
-    Input/Output/Agent block types and blocks in EXCLUDED_BLOCK_IDS.
-    """
-    results = await mv_suggested_blocks.prisma().find_many()
+    results = await query_raw_with_schema(
+        """
+        SELECT
+            agent_node."agentBlockId" AS block_id,
+            COUNT(execution.id) AS execution_count
+        FROM {schema_prefix}"AgentNodeExecution" execution
+        JOIN {schema_prefix}"AgentNode" agent_node ON execution."agentNodeId" = agent_node.id
+        WHERE execution."endedTime" >= $1::timestamp
+        GROUP BY agent_node."agentBlockId"
+        ORDER BY execution_count DESC;
+        """,
+        timestamp_threshold,
+    )
 
     # Get the top blocks based on execution count
-    # But ignore Input, Output, Agent, and excluded blocks
+    # But ignore Input and Output blocks
     blocks: list[tuple[BlockInfo, int]] = []
-    execution_counts = {row.block_id: row.execution_count for row in results}
 
     for block_type in load_all_blocks().values():
         block: AnyBlockSchema = block_type()
@@ -791,9 +679,11 @@ async def get_suggested_blocks(count: int = 5) -> list[BlockInfo]:
             BlockType.AGENT,
         ):
             continue
-        if block.id in EXCLUDED_BLOCK_IDS:
-            continue
-        execution_count = execution_counts.get(block.id, 0)
+        # Find the execution count for this block
+        execution_count = next(
+            (row["execution_count"] for row in results if row["block_id"] == block.id),
+            0,
+        )
         blocks.append((block.get_info(), execution_count))
     # Sort blocks by execution count
     blocks.sort(key=lambda x: x[1], reverse=True)

autogpt_platform/backend/backend/api/features/builder/model.py

@@ -27,6 +27,7 @@ class SearchEntry(BaseModel):
 
 # Suggestions
 class SuggestionsResponse(BaseModel):
+    otto_suggestions: list[str]
     recent_searches: list[SearchEntry]
     providers: list[ProviderName]
     top_blocks: list[BlockInfo]

autogpt_platform/backend/backend/api/features/builder/routes.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Annotated, Sequence, cast, get_args
+from typing import Annotated, Sequence
 
 import fastapi
 from autogpt_libs.auth.dependencies import get_user_id, requires_user
@@ -10,8 +10,6 @@ from backend.util.models import Pagination
 from . import db as builder_db
 from . import model as builder_model
 
-VALID_FILTER_VALUES = get_args(builder_model.FilterType)
-
 logger = logging.getLogger(__name__)
 
 router = fastapi.APIRouter(
@@ -51,6 +49,11 @@ async def get_suggestions(
     Get all suggestions for the Blocks Menu.
     """
     return builder_model.SuggestionsResponse(
+        otto_suggestions=[
+            "What blocks do I need to get started?",
+            "Help me create a list",
+            "Help me feed my data to Google Maps",
+        ],
         recent_searches=await builder_db.get_recent_searches(user_id),
         providers=[
             ProviderName.TWITTER,
@@ -148,7 +151,7 @@ async def get_providers(
 async def search(
     user_id: Annotated[str, fastapi.Security(get_user_id)],
     search_query: Annotated[str | None, fastapi.Query()] = None,
-    filter: Annotated[str | None, fastapi.Query()] = None,
+    filter: Annotated[list[builder_model.FilterType] | None, fastapi.Query()] = None,
     search_id: Annotated[str | None, fastapi.Query()] = None,
     by_creator: Annotated[list[str] | None, fastapi.Query()] = None,
     page: Annotated[int, fastapi.Query()] = 1,
@@ -157,20 +160,9 @@ async def search(
     """
     Search for blocks (including integrations), marketplace agents, and user library agents.
     """
-    # Parse and validate filter parameter
-    filters: list[builder_model.FilterType]
-    if filter:
-        filter_values = [f.strip() for f in filter.split(",")]
-        invalid_filters = [f for f in filter_values if f not in VALID_FILTER_VALUES]
-        if invalid_filters:
-            raise fastapi.HTTPException(
-                status_code=400,
-                detail=f"Invalid filter value(s): {', '.join(invalid_filters)}. "
-                f"Valid values are: {', '.join(VALID_FILTER_VALUES)}",
-            )
-        filters = cast(list[builder_model.FilterType], filter_values)
-    else:
-        filters = [
+    # If no filters are provided, then we will return all types
+    if not filter:
+        filter = [
             "blocks",
             "integrations",
             "marketplace_agents",
@@ -182,7 +174,7 @@ async def search(
     cached_results = await builder_db.get_sorted_search_results(
         user_id=user_id,
         search_query=search_query,
-        filters=filters,
+        filters=filter,
         by_creator=by_creator,
     )
 
@@ -204,7 +196,7 @@ async def search(
         user_id,
         builder_model.SearchEntry(
             search_query=search_query,
-            filter=filters,
+            filter=filter,
             by_creator=by_creator,
             search_id=search_id,
         ),

autogpt_platform/backend/backend/api/features/chat/routes.py

@@ -2,19 +2,23 @@
 
 import asyncio
 import logging
+import uuid as uuid_module
 from collections.abc import AsyncGenerator
 from typing import Annotated
-from uuid import uuid4
 
 from autogpt_libs import auth
-from fastapi import APIRouter, Depends, HTTPException, Query, Response, Security
+from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, Security
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
 
 from backend.copilot import service as chat_service
 from backend.copilot import stream_registry
+from backend.copilot.completion_handler import (
+    process_operation_failure,
+    process_operation_success,
+)
 from backend.copilot.config import ChatConfig
-from backend.copilot.executor.utils import enqueue_cancel_task, enqueue_copilot_turn
+from backend.copilot.executor.utils import enqueue_cancel_task, enqueue_copilot_task
 from backend.copilot.model import (
     ChatMessage,
     ChatSession,
@@ -42,6 +46,9 @@ from backend.copilot.tools.models import (
     InputValidationErrorResponse,
     NeedLoginResponse,
     NoResultsResponse,
+    OperationInProgressResponse,
+    OperationPendingResponse,
+    OperationStartedResponse,
     SetupRequirementsResponse,
     SuggestedGoalResponse,
     UnderstandingUpdatedResponse,
@@ -92,8 +99,10 @@ class CreateSessionResponse(BaseModel):
 class ActiveStreamInfo(BaseModel):
     """Information about an active stream for reconnection."""
 
-    turn_id: str
+    task_id: str
     last_message_id: str  # Redis Stream message ID for resumption
+    operation_id: str  # Operation ID for completion tracking
+    tool_name: str  # Name of the tool being executed
 
 
 class SessionDetailResponse(BaseModel):
@@ -123,13 +132,22 @@ class ListSessionsResponse(BaseModel):
     total: int
 
 
-class CancelSessionResponse(BaseModel):
-    """Response model for the cancel session endpoint."""
+class CancelTaskResponse(BaseModel):
+    """Response model for the cancel task endpoint."""
 
     cancelled: bool
+    task_id: str | None = None
     reason: str | None = None
 
 
+class OperationCompleteRequest(BaseModel):
+    """Request model for external completion webhook."""
+
+    success: bool
+    result: dict | str | None = None
+    error: str | None = None
+
+
 # ========== Routes ==========
 
 
@@ -252,7 +270,7 @@ async def get_session(
     Retrieve the details of a specific chat session.
 
     Looks up a chat session by ID for the given user (if authenticated) and returns all session data including messages.
-    If there's an active stream for this session, returns active_stream info for reconnection.
+    If there's an active stream for this session, returns the task_id for reconnection.
 
     Args:
         session_id: The unique identifier for the desired chat session.
@@ -270,21 +288,28 @@ async def get_session(
 
     # Check if there's an active stream for this session
     active_stream_info = None
-    active_session, last_message_id = await stream_registry.get_active_session(
+    active_task, last_message_id = await stream_registry.get_active_task_for_session(
         session_id, user_id
     )
     logger.info(
-        f"[GET_SESSION] session={session_id}, active_session={active_session is not None}, "
+        f"[GET_SESSION] session={session_id}, active_task={active_task is not None}, "
         f"msg_count={len(messages)}, last_role={messages[-1].get('role') if messages else 'none'}"
     )
-    if active_session:
-        # Keep the assistant message (including tool_calls) so the frontend can
-        # render the correct tool UI (e.g. CreateAgent with mini game).
-        # convertChatSessionToUiMessages handles isComplete=false by setting
-        # tool parts without output to state "input-available".
+    if active_task:
+        # Filter out the in-progress assistant message from the session response.
+        # The client will receive the complete assistant response through the SSE
+        # stream replay instead, preventing duplicate content.
+        if messages and messages[-1].get("role") == "assistant":
+            messages = messages[:-1]
+
+        # Use "0-0" as last_message_id to replay the stream from the beginning.
+        # Since we filtered out the cached assistant message, the client needs
+        # the full stream to reconstruct the response.
         active_stream_info = ActiveStreamInfo(
-            turn_id=active_session.turn_id,
-            last_message_id=last_message_id,
+            task_id=active_task.task_id,
+            last_message_id="0-0",
+            operation_id=active_task.operation_id,
+            tool_name=active_task.tool_name,
         )
 
     return SessionDetailResponse(
@@ -304,7 +329,7 @@ async def get_session(
 async def cancel_session_task(
     session_id: str,
     user_id: Annotated[str | None, Depends(auth.get_user_id)],
-) -> CancelSessionResponse:
+) -> CancelTaskResponse:
     """Cancel the active streaming task for a session.
 
     Publishes a cancel event to the executor via RabbitMQ FANOUT, then
@@ -313,33 +338,39 @@ async def cancel_session_task(
     """
     await _validate_and_get_session(session_id, user_id)
 
-    active_session, _ = await stream_registry.get_active_session(session_id, user_id)
-    if not active_session:
-        return CancelSessionResponse(cancelled=True, reason="no_active_session")
+    active_task, _ = await stream_registry.get_active_task_for_session(
+        session_id, user_id
+    )
+    if not active_task:
+        return CancelTaskResponse(cancelled=False, reason="no_active_task")
 
-    await enqueue_cancel_task(session_id)
-    logger.info(f"[CANCEL] Published cancel for session ...{session_id[-8:]}")
+    task_id = active_task.task_id
+    await enqueue_cancel_task(task_id)
+    logger.info(
+        f"[CANCEL] Published cancel for task ...{task_id[-8:]} "
+        f"session ...{session_id[-8:]}"
+    )
 
     # Poll until the executor confirms the task is no longer running.
+    # Keep max_wait below typical reverse-proxy read timeouts.
     poll_interval = 0.5
     max_wait = 5.0
     waited = 0.0
     while waited < max_wait:
         await asyncio.sleep(poll_interval)
         waited += poll_interval
-        session_state = await stream_registry.get_session(session_id)
-        if session_state is None or session_state.status != "running":
+        task = await stream_registry.get_task(task_id)
+        if task is None or task.status != "running":
             logger.info(
-                f"[CANCEL] Session ...{session_id[-8:]} confirmed stopped "
-                f"(status={session_state.status if session_state else 'gone'}) after {waited:.1f}s"
+                f"[CANCEL] Task ...{task_id[-8:]} confirmed stopped "
+                f"(status={task.status if task else 'gone'}) after {waited:.1f}s"
             )
-            return CancelSessionResponse(cancelled=True)
+            return CancelTaskResponse(cancelled=True, task_id=task_id)
 
-    logger.warning(
-        f"[CANCEL] Session ...{session_id[-8:]} not confirmed after {max_wait}s, force-completing"
+    logger.warning(f"[CANCEL] Task ...{task_id[-8:]} not confirmed after {max_wait}s")
+    return CancelTaskResponse(
+        cancelled=True, task_id=task_id, reason="cancel_published_not_confirmed"
     )
-    await stream_registry.mark_session_completed(session_id, error_message="Cancelled")
-    return CancelSessionResponse(cancelled=True)
 
 
 @router.post(
@@ -359,15 +390,16 @@ async def stream_chat_post(
       - Tool execution results
 
     The AI generation runs in a background task that continues even if the client disconnects.
-    All chunks are written to a per-turn Redis stream for reconnection support. If the client
-    disconnects, they can reconnect using GET /sessions/{session_id}/stream to resume.
+    All chunks are written to Redis for reconnection support. If the client disconnects,
+    they can reconnect using GET /tasks/{task_id}/stream to resume from where they left off.
 
     Args:
         session_id: The chat session identifier to associate with the streamed messages.
         request: Request body containing message, is_user_message, and optional context.
         user_id: Optional authenticated user ID.
     Returns:
-        StreamingResponse: SSE-formatted response chunks.
+        StreamingResponse: SSE-formatted response chunks. First chunk is a "start" event
+        containing the task_id for reconnection.
 
     """
     import asyncio
@@ -414,35 +446,35 @@ async def stream_chat_post(
         logger.info(f"[STREAM] User message saved for session {session_id}")
 
     # Create a task in the stream registry for reconnection support
-    turn_id = str(uuid4())
-    log_meta["turn_id"] = turn_id
+    task_id = str(uuid_module.uuid4())
+    operation_id = str(uuid_module.uuid4())
+    log_meta["task_id"] = task_id
 
-    session_create_start = time.perf_counter()
-    await stream_registry.create_session(
+    task_create_start = time.perf_counter()
+    await stream_registry.create_task(
+        task_id=task_id,
         session_id=session_id,
         user_id=user_id,
-        tool_call_id="chat_stream",
+        tool_call_id="chat_stream",  # Not a tool call, but needed for the model
         tool_name="chat",
-        turn_id=turn_id,
+        operation_id=operation_id,
     )
     logger.info(
-        f"[TIMING] create_session completed in {(time.perf_counter() - session_create_start) * 1000:.1f}ms",
+        f"[TIMING] create_task completed in {(time.perf_counter() - task_create_start) * 1000:.1f}ms",
         extra={
             "json_fields": {
                 **log_meta,
-                "duration_ms": (time.perf_counter() - session_create_start) * 1000,
+                "duration_ms": (time.perf_counter() - task_create_start) * 1000,
             }
         },
     )
 
-    # Per-turn stream is always fresh (unique turn_id), subscribe from beginning
-    subscribe_from_id = "0-0"
-
-    await enqueue_copilot_turn(
+    await enqueue_copilot_task(
+        task_id=task_id,
         session_id=session_id,
         user_id=user_id,
+        operation_id=operation_id,
         message=request.message,
-        turn_id=turn_id,
         is_user_message=request.is_user_message,
         context=request.context,
     )
@@ -459,7 +491,7 @@ async def stream_chat_post(
 
         event_gen_start = time_module.perf_counter()
         logger.info(
-            f"[TIMING] event_generator STARTED, turn={turn_id}, session={session_id}, "
+            f"[TIMING] event_generator STARTED, task={task_id}, session={session_id}, "
             f"user={user_id}",
             extra={"json_fields": log_meta},
         )
@@ -467,12 +499,11 @@ async def stream_chat_post(
         first_chunk_yielded = False
         chunks_yielded = 0
         try:
-            # Subscribe from the position we captured before enqueuing
-            # This avoids replaying old messages while catching all new ones
-            subscriber_queue = await stream_registry.subscribe_to_session(
-                session_id=session_id,
+            # Subscribe to the task stream (this replays existing messages + live updates)
+            subscriber_queue = await stream_registry.subscribe_to_task(
+                task_id=task_id,
                 user_id=user_id,
-                last_message_id=subscribe_from_id,
+                last_message_id="0-0",  # Get all messages from the beginning
             )
 
             if subscriber_queue is None:
@@ -555,19 +586,19 @@ async def stream_chat_post(
             # Unsubscribe when client disconnects or stream ends
             if subscriber_queue is not None:
                 try:
-                    await stream_registry.unsubscribe_from_session(
-                        session_id, subscriber_queue
+                    await stream_registry.unsubscribe_from_task(
+                        task_id, subscriber_queue
                     )
                 except Exception as unsub_err:
                     logger.error(
-                        f"Error unsubscribing from session {session_id}: {unsub_err}",
+                        f"Error unsubscribing from task {task_id}: {unsub_err}",
                         exc_info=True,
                     )
             # AI SDK protocol termination - always yield even if unsubscribe fails
             total_time = time_module.perf_counter() - event_gen_start
             logger.info(
                 f"[TIMING] event_generator FINISHED in {total_time:.2f}s; "
-                f"turn={turn_id}, session={session_id}, n_chunks={chunks_yielded}",
+                f"task={task_id}, session={session_id}, n_chunks={chunks_yielded}",
                 extra={
                     "json_fields": {
                         **log_meta,
@@ -614,21 +645,17 @@ async def resume_session_stream(
     """
     import asyncio
 
-    active_session, last_message_id = await stream_registry.get_active_session(
+    active_task, _last_id = await stream_registry.get_active_task_for_session(
         session_id, user_id
     )
 
-    if not active_session:
+    if not active_task:
         return Response(status_code=204)
 
-    # Always replay from the beginning ("0-0") on resume.
-    # We can't use last_message_id because it's the latest ID in the backend
-    # stream, not the latest the frontend received — the gap causes lost
-    # messages. The frontend deduplicates replayed content.
-    subscriber_queue = await stream_registry.subscribe_to_session(
-        session_id=session_id,
+    subscriber_queue = await stream_registry.subscribe_to_task(
+        task_id=active_task.task_id,
         user_id=user_id,
-        last_message_id="0-0",
+        last_message_id="0-0",  # Full replay so useChat rebuilds the message
     )
 
     if subscriber_queue is None:
@@ -664,12 +691,12 @@ async def resume_session_stream(
             logger.error(f"Error in resume stream for session {session_id}: {e}")
         finally:
             try:
-                await stream_registry.unsubscribe_from_session(
-                    session_id, subscriber_queue
+                await stream_registry.unsubscribe_from_task(
+                    active_task.task_id, subscriber_queue
                 )
             except Exception as unsub_err:
                 logger.error(
-                    f"Error unsubscribing from session {active_session.session_id}: {unsub_err}",
+                    f"Error unsubscribing from task {active_task.task_id}: {unsub_err}",
                     exc_info=True,
                 )
             logger.info(
@@ -720,6 +747,229 @@ async def session_assign_user(
     return {"status": "ok"}
 
 
+# ========== Task Streaming (SSE Reconnection) ==========
+
+
+@router.get(
+    "/tasks/{task_id}/stream",
+)
+async def stream_task(
+    task_id: str,
+    user_id: str | None = Depends(auth.get_user_id),
+    last_message_id: str = Query(
+        default="0-0",
+        description="Last Redis Stream message ID received (e.g., '1706540123456-0'). Use '0-0' for full replay.",
+    ),
+):
+    """
+    Reconnect to a long-running task's SSE stream.
+
+    When a long-running operation (like agent generation) starts, the client
+    receives a task_id. If the connection drops, the client can reconnect
+    using this endpoint to resume receiving updates.
+
+    Args:
+        task_id: The task ID from the operation_started response.
+        user_id: Authenticated user ID for ownership validation.
+        last_message_id: Last Redis Stream message ID received ("0-0" for full replay).
+
+    Returns:
+        StreamingResponse: SSE-formatted response chunks starting after last_message_id.
+
+    Raises:
+        HTTPException: 404 if task not found, 410 if task expired, 403 if access denied.
+    """
+    # Check task existence and expiry before subscribing
+    task, error_code = await stream_registry.get_task_with_expiry_info(task_id)
+
+    if error_code == "TASK_EXPIRED":
+        raise HTTPException(
+            status_code=410,
+            detail={
+                "code": "TASK_EXPIRED",
+                "message": "This operation has expired. Please try again.",
+            },
+        )
+
+    if error_code == "TASK_NOT_FOUND":
+        raise HTTPException(
+            status_code=404,
+            detail={
+                "code": "TASK_NOT_FOUND",
+                "message": f"Task {task_id} not found.",
+            },
+        )
+
+    # Validate ownership if task has an owner
+    if task and task.user_id and user_id != task.user_id:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "code": "ACCESS_DENIED",
+                "message": "You do not have access to this task.",
+            },
+        )
+
+    # Get subscriber queue from stream registry
+    subscriber_queue = await stream_registry.subscribe_to_task(
+        task_id=task_id,
+        user_id=user_id,
+        last_message_id=last_message_id,
+    )
+
+    if subscriber_queue is None:
+        raise HTTPException(
+            status_code=404,
+            detail={
+                "code": "TASK_NOT_FOUND",
+                "message": f"Task {task_id} not found or access denied.",
+            },
+        )
+
+    async def event_generator() -> AsyncGenerator[str, None]:
+        heartbeat_interval = 15.0  # Send heartbeat every 15 seconds
+        try:
+            while True:
+                try:
+                    # Wait for next chunk with timeout for heartbeats
+                    chunk = await asyncio.wait_for(
+                        subscriber_queue.get(), timeout=heartbeat_interval
+                    )
+                    yield chunk.to_sse()
+
+                    # Check for finish signal
+                    if isinstance(chunk, StreamFinish):
+                        break
+                except asyncio.TimeoutError:
+                    # Send heartbeat to keep connection alive
+                    yield StreamHeartbeat().to_sse()
+        except Exception as e:
+            logger.error(f"Error in task stream {task_id}: {e}", exc_info=True)
+        finally:
+            # Unsubscribe when client disconnects or stream ends
+            try:
+                await stream_registry.unsubscribe_from_task(task_id, subscriber_queue)
+            except Exception as unsub_err:
+                logger.error(
+                    f"Error unsubscribing from task {task_id}: {unsub_err}",
+                    exc_info=True,
+                )
+            # AI SDK protocol termination - always yield even if unsubscribe fails
+            yield "data: [DONE]\n\n"
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+            "x-vercel-ai-ui-message-stream": "v1",
+        },
+    )
+
+
+@router.get(
+    "/tasks/{task_id}",
+)
+async def get_task_status(
+    task_id: str,
+    user_id: str | None = Depends(auth.get_user_id),
+) -> dict:
+    """
+    Get the status of a long-running task.
+
+    Args:
+        task_id: The task ID to check.
+        user_id: Authenticated user ID for ownership validation.
+
+    Returns:
+        dict: Task status including task_id, status, tool_name, and operation_id.
+
+    Raises:
+        NotFoundError: If task_id is not found or user doesn't have access.
+    """
+    task = await stream_registry.get_task(task_id)
+
+    if task is None:
+        raise NotFoundError(f"Task {task_id} not found.")
+
+    # Validate ownership - if task has an owner, requester must match
+    if task.user_id and user_id != task.user_id:
+        raise NotFoundError(f"Task {task_id} not found.")
+
+    return {
+        "task_id": task.task_id,
+        "session_id": task.session_id,
+        "status": task.status,
+        "tool_name": task.tool_name,
+        "operation_id": task.operation_id,
+        "created_at": task.created_at.isoformat(),
+    }
+
+
+# ========== External Completion Webhook ==========
+
+
+@router.post(
+    "/operations/{operation_id}/complete",
+    status_code=200,
+)
+async def complete_operation(
+    operation_id: str,
+    request: OperationCompleteRequest,
+    x_api_key: str | None = Header(default=None),
+) -> dict:
+    """
+    External completion webhook for long-running operations.
+
+    Called by Agent Generator (or other services) when an operation completes.
+    This triggers the stream registry to publish completion and continue LLM generation.
+
+    Args:
+        operation_id: The operation ID to complete.
+        request: Completion payload with success status and result/error.
+        x_api_key: Internal API key for authentication.
+
+    Returns:
+        dict: Status of the completion.
+
+    Raises:
+        HTTPException: If API key is invalid or operation not found.
+    """
+    # Validate internal API key - reject if not configured or invalid
+    if not config.internal_api_key:
+        logger.error(
+            "Operation complete webhook rejected: CHAT_INTERNAL_API_KEY not configured"
+        )
+        raise HTTPException(
+            status_code=503,
+            detail="Webhook not available: internal API key not configured",
+        )
+    if x_api_key != config.internal_api_key:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    # Find task by operation_id
+    task = await stream_registry.find_task_by_operation_id(operation_id)
+    if task is None:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Operation {operation_id} not found",
+        )
+
+    logger.info(
+        f"Received completion webhook for operation {operation_id} "
+        f"(task_id={task.task_id}, success={request.success})"
+    )
+
+    if request.success:
+        await process_operation_success(task, request.result)
+    else:
+        await process_operation_failure(task, request.error)
+
+    return {"status": "ok", "task_id": task.task_id}
+
+
 # ========== Configuration ==========
 
 
@@ -800,6 +1050,9 @@ ToolResponseUnion = (
     | BlockOutputResponse
     | DocSearchResultsResponse
     | DocPageResponse
+    | OperationStartedResponse
+    | OperationPendingResponse
+    | OperationInProgressResponse
 )
 
 

autogpt_platform/backend/backend/api/features/library/db_test.py

@@ -144,7 +144,6 @@ async def test_add_agent_to_library(mocker):
     )
 
     mock_library_agent = mocker.patch("prisma.models.LibraryAgent.prisma")
-    mock_library_agent.return_value.find_first = mocker.AsyncMock(return_value=None)
     mock_library_agent.return_value.find_unique = mocker.AsyncMock(return_value=None)
     mock_library_agent.return_value.create = mocker.AsyncMock(
         return_value=mock_library_agent_data
@@ -179,6 +178,7 @@ async def test_add_agent_to_library(mocker):
                 "agentGraphVersion": 1,
             }
         },
+        include={"AgentGraph": True},
     )
     # Check that create was called with the expected data including settings
     create_call_args = mock_library_agent.return_value.create.call_args

autogpt_platform/backend/backend/api/features/library/exceptions.py

@@ -1,10 +0,0 @@
-class FolderValidationError(Exception):
-    """Raised when folder operations fail validation."""
-
-    pass
-
-
-class FolderAlreadyExistsError(FolderValidationError):
-    """Raised when a folder with the same name already exists in the location."""
-
-    pass

autogpt_platform/backend/backend/api/features/library/model.py

@@ -26,95 +26,6 @@ class LibraryAgentStatus(str, Enum):
     ERROR = "ERROR"
 
 
-# === Folder Models ===
-
-
-class LibraryFolder(pydantic.BaseModel):
-    """Represents a folder for organizing library agents."""
-
-    id: str
-    user_id: str
-    name: str
-    icon: str | None = None
-    color: str | None = None
-    parent_id: str | None = None
-    created_at: datetime.datetime
-    updated_at: datetime.datetime
-    agent_count: int = 0  # Direct agents in folder
-    subfolder_count: int = 0  # Direct child folders
-
-    @staticmethod
-    def from_db(
-        folder: prisma.models.LibraryFolder,
-        agent_count: int = 0,
-        subfolder_count: int = 0,
-    ) -> "LibraryFolder":
-        """Factory method that constructs a LibraryFolder from a Prisma model."""
-        return LibraryFolder(
-            id=folder.id,
-            user_id=folder.userId,
-            name=folder.name,
-            icon=folder.icon,
-            color=folder.color,
-            parent_id=folder.parentId,
-            created_at=folder.createdAt,
-            updated_at=folder.updatedAt,
-            agent_count=agent_count,
-            subfolder_count=subfolder_count,
-        )
-
-
-class LibraryFolderTree(LibraryFolder):
-    """Folder with nested children for tree view."""
-
-    children: list["LibraryFolderTree"] = []
-
-
-class FolderCreateRequest(pydantic.BaseModel):
-    """Request model for creating a folder."""
-
-    name: str = pydantic.Field(..., min_length=1, max_length=100)
-    icon: str | None = None
-    color: str | None = pydantic.Field(
-        None, pattern=r"^#[0-9A-Fa-f]{6}$", description="Hex color code (#RRGGBB)"
-    )
-    parent_id: str | None = None
-
-
-class FolderUpdateRequest(pydantic.BaseModel):
-    """Request model for updating a folder."""
-
-    name: str | None = pydantic.Field(None, min_length=1, max_length=100)
-    icon: str | None = None
-    color: str | None = None
-
-
-class FolderMoveRequest(pydantic.BaseModel):
-    """Request model for moving a folder to a new parent."""
-
-    target_parent_id: str | None = None  # None = move to root
-
-
-class BulkMoveAgentsRequest(pydantic.BaseModel):
-    """Request model for moving multiple agents to a folder."""
-
-    agent_ids: list[str]
-    folder_id: str | None = None  # None = move to root
-
-
-class FolderListResponse(pydantic.BaseModel):
-    """Response schema for a list of folders."""
-
-    folders: list[LibraryFolder]
-    pagination: Pagination
-
-
-class FolderTreeResponse(pydantic.BaseModel):
-    """Response schema for folder tree structure."""
-
-    tree: list[LibraryFolderTree]
-
-
 class MarketplaceListingCreator(pydantic.BaseModel):
     """Creator information for a marketplace listing."""
 
@@ -209,9 +120,6 @@ class LibraryAgent(pydantic.BaseModel):
     can_access_graph: bool
     is_latest_version: bool
     is_favorite: bool
-    folder_id: str | None = None
-    folder_name: str | None = None  # Denormalized for display
-
     recommended_schedule_cron: str | None = None
     settings: GraphSettings = pydantic.Field(default_factory=GraphSettings)
     marketplace_listing: Optional["MarketplaceListing"] = None
@@ -351,8 +259,6 @@ class LibraryAgent(pydantic.BaseModel):
             can_access_graph=can_access_graph,
             is_latest_version=is_latest_version,
             is_favorite=agent.isFavorite,
-            folder_id=agent.folderId,
-            folder_name=agent.Folder.name if agent.Folder else None,
             recommended_schedule_cron=agent.AgentGraph.recommendedScheduleCron,
             settings=_parse_settings(agent.settings),
             marketplace_listing=marketplace_listing_data,
@@ -564,7 +470,3 @@ class LibraryAgentUpdateRequest(pydantic.BaseModel):
     settings: Optional[GraphSettings] = pydantic.Field(
         default=None, description="User-specific settings for this library agent"
     )
-    folder_id: Optional[str] = pydantic.Field(
-        default=None,
-        description="Folder ID to move agent to (None to move to root)",
-    )

autogpt_platform/backend/backend/api/features/library/routes/__init__.py

@@ -1,11 +1,9 @@
 import fastapi
 
 from .agents import router as agents_router
-from .folders import router as folders_router
 from .presets import router as presets_router
 
 router = fastapi.APIRouter()
 
 router.include_router(presets_router)
-router.include_router(folders_router)
 router.include_router(agents_router)

autogpt_platform/backend/backend/api/features/library/routes/agents.py

@@ -41,14 +41,6 @@ async def list_library_agents(
         ge=1,
         description="Number of agents per page (must be >= 1)",
     ),
-    folder_id: Optional[str] = Query(
-        None,
-        description="Filter by folder ID",
-    ),
-    include_root_only: bool = Query(
-        False,
-        description="Only return agents without a folder (root-level agents)",
-    ),
 ) -> library_model.LibraryAgentResponse:
     """
     Get all agents in the user's library (both created and saved).
@@ -59,8 +51,6 @@ async def list_library_agents(
         sort_by=sort_by,
         page=page,
         page_size=page_size,
-        folder_id=folder_id,
-        include_root_only=include_root_only,
     )
 
 
@@ -178,7 +168,6 @@ async def update_library_agent(
         is_favorite=payload.is_favorite,
         is_archived=payload.is_archived,
         settings=payload.settings,
-        folder_id=payload.folder_id,
     )
 
 

autogpt_platform/backend/backend/api/features/library/routes/folders.py

@@ -1,287 +0,0 @@
-from typing import Optional
-
-import autogpt_libs.auth as autogpt_auth_lib
-from fastapi import APIRouter, Query, Security, status
-from fastapi.responses import Response
-
-from .. import db as library_db
-from .. import model as library_model
-
-router = APIRouter(
-    prefix="/folders",
-    tags=["library", "folders", "private"],
-    dependencies=[Security(autogpt_auth_lib.requires_user)],
-)
-
-
-@router.get(
-    "",
-    summary="List Library Folders",
-    response_model=library_model.FolderListResponse,
-    responses={
-        200: {"description": "List of folders"},
-        500: {"description": "Server error"},
-    },
-)
-async def list_folders(
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-    parent_id: Optional[str] = Query(
-        None,
-        description="Filter by parent folder ID. If not provided, returns root-level folders.",
-    ),
-    include_relations: bool = Query(
-        True,
-        description="Include agent and subfolder relations (for counts)",
-    ),
-) -> library_model.FolderListResponse:
-    """
-    List folders for the authenticated user.
-
-    Args:
-        user_id: ID of the authenticated user.
-        parent_id: Optional parent folder ID to filter by.
-        include_relations: Whether to include agent and subfolder relations for counts.
-
-    Returns:
-        A FolderListResponse containing folders.
-    """
-    folders = await library_db.list_folders(
-        user_id=user_id,
-        parent_id=parent_id,
-        include_relations=include_relations,
-    )
-    return library_model.FolderListResponse(
-        folders=folders,
-        pagination=library_model.Pagination(
-            total_items=len(folders),
-            total_pages=1,
-            current_page=1,
-            page_size=len(folders),
-        ),
-    )
-
-
-@router.get(
-    "/tree",
-    summary="Get Folder Tree",
-    response_model=library_model.FolderTreeResponse,
-    responses={
-        200: {"description": "Folder tree structure"},
-        500: {"description": "Server error"},
-    },
-)
-async def get_folder_tree(
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> library_model.FolderTreeResponse:
-    """
-    Get the full folder tree for the authenticated user.
-
-    Args:
-        user_id: ID of the authenticated user.
-
-    Returns:
-        A FolderTreeResponse containing the nested folder structure.
-    """
-    tree = await library_db.get_folder_tree(user_id=user_id)
-    return library_model.FolderTreeResponse(tree=tree)
-
-
-@router.get(
-    "/{folder_id}",
-    summary="Get Folder",
-    response_model=library_model.LibraryFolder,
-    responses={
-        200: {"description": "Folder details"},
-        404: {"description": "Folder not found"},
-        500: {"description": "Server error"},
-    },
-)
-async def get_folder(
-    folder_id: str,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> library_model.LibraryFolder:
-    """
-    Get a specific folder.
-
-    Args:
-        folder_id: ID of the folder to retrieve.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        The requested LibraryFolder.
-    """
-    return await library_db.get_folder(folder_id=folder_id, user_id=user_id)
-
-
-@router.post(
-    "",
-    summary="Create Folder",
-    status_code=status.HTTP_201_CREATED,
-    response_model=library_model.LibraryFolder,
-    responses={
-        201: {"description": "Folder created successfully"},
-        400: {"description": "Validation error"},
-        404: {"description": "Parent folder not found"},
-        409: {"description": "Folder name conflict"},
-        500: {"description": "Server error"},
-    },
-)
-async def create_folder(
-    payload: library_model.FolderCreateRequest,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> library_model.LibraryFolder:
-    """
-    Create a new folder.
-
-    Args:
-        payload: The folder creation request.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        The created LibraryFolder.
-    """
-    return await library_db.create_folder(
-        user_id=user_id,
-        name=payload.name,
-        parent_id=payload.parent_id,
-        icon=payload.icon,
-        color=payload.color,
-    )
-
-
-@router.patch(
-    "/{folder_id}",
-    summary="Update Folder",
-    response_model=library_model.LibraryFolder,
-    responses={
-        200: {"description": "Folder updated successfully"},
-        400: {"description": "Validation error"},
-        404: {"description": "Folder not found"},
-        409: {"description": "Folder name conflict"},
-        500: {"description": "Server error"},
-    },
-)
-async def update_folder(
-    folder_id: str,
-    payload: library_model.FolderUpdateRequest,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> library_model.LibraryFolder:
-    """
-    Update a folder's properties.
-
-    Args:
-        folder_id: ID of the folder to update.
-        payload: The folder update request.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        The updated LibraryFolder.
-    """
-    return await library_db.update_folder(
-        folder_id=folder_id,
-        user_id=user_id,
-        name=payload.name,
-        icon=payload.icon,
-        color=payload.color,
-    )
-
-
-@router.post(
-    "/{folder_id}/move",
-    summary="Move Folder",
-    response_model=library_model.LibraryFolder,
-    responses={
-        200: {"description": "Folder moved successfully"},
-        400: {"description": "Validation error (circular reference)"},
-        404: {"description": "Folder or target parent not found"},
-        409: {"description": "Folder name conflict in target location"},
-        500: {"description": "Server error"},
-    },
-)
-async def move_folder(
-    folder_id: str,
-    payload: library_model.FolderMoveRequest,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> library_model.LibraryFolder:
-    """
-    Move a folder to a new parent.
-
-    Args:
-        folder_id: ID of the folder to move.
-        payload: The move request with target parent.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        The moved LibraryFolder.
-    """
-    return await library_db.move_folder(
-        folder_id=folder_id,
-        user_id=user_id,
-        target_parent_id=payload.target_parent_id,
-    )
-
-
-@router.delete(
-    "/{folder_id}",
-    summary="Delete Folder",
-    status_code=status.HTTP_204_NO_CONTENT,
-    responses={
-        204: {"description": "Folder deleted successfully"},
-        404: {"description": "Folder not found"},
-        500: {"description": "Server error"},
-    },
-)
-async def delete_folder(
-    folder_id: str,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> Response:
-    """
-    Soft-delete a folder and all its contents.
-
-    Args:
-        folder_id: ID of the folder to delete.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        204 No Content if successful.
-    """
-    await library_db.delete_folder(
-        folder_id=folder_id,
-        user_id=user_id,
-        soft_delete=True,
-    )
-    return Response(status_code=status.HTTP_204_NO_CONTENT)
-
-
-# === Bulk Agent Operations ===
-
-
-@router.post(
-    "/agents/bulk-move",
-    summary="Bulk Move Agents",
-    response_model=list[library_model.LibraryAgent],
-    responses={
-        200: {"description": "Agents moved successfully"},
-        404: {"description": "Folder not found"},
-        500: {"description": "Server error"},
-    },
-)
-async def bulk_move_agents(
-    payload: library_model.BulkMoveAgentsRequest,
-    user_id: str = Security(autogpt_auth_lib.get_user_id),
-) -> list[library_model.LibraryAgent]:
-    """
-    Move multiple agents to a folder.
-
-    Args:
-        payload: The bulk move request with agent IDs and target folder.
-        user_id: ID of the authenticated user.
-
-    Returns:
-        The updated LibraryAgents.
-    """
-    return await library_db.bulk_move_agents_to_folder(
-        agent_ids=payload.agent_ids,
-        folder_id=payload.folder_id,
-        user_id=user_id,
-    )

autogpt_platform/backend/backend/api/features/library/routes_test.py

@@ -115,8 +115,6 @@ async def test_get_library_agents_success(
         sort_by=library_model.LibraryAgentSort.UPDATED_AT,
         page=1,
         page_size=15,
-        folder_id=None,
-        include_root_only=False,
     )
 
 

autogpt_platform/backend/backend/api/features/store/content_handlers.py

@@ -9,26 +9,15 @@ import logging
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, get_args, get_origin
+from typing import Any
 
 from prisma.enums import ContentType
 
-from backend.blocks.llm import LlmModel
 from backend.data.db import query_raw_with_schema
 
 logger = logging.getLogger(__name__)
 
 
-def _contains_type(annotation: Any, target: type) -> bool:
-    """Check if an annotation is or contains the target type (handles Optional/Union/Annotated)."""
-    if annotation is target:
-        return True
-    origin = get_origin(annotation)
-    if origin is None:
-        return False
-    return any(_contains_type(arg, target) for arg in get_args(annotation))
-
-
 @dataclass
 class ContentItem:
     """Represents a piece of content to be embedded."""
@@ -199,51 +188,45 @@ class BlockHandler(ContentHandler):
             try:
                 block_instance = block_cls()
 
+                # Skip disabled blocks - they shouldn't be indexed
                 if block_instance.disabled:
                     continue
 
                 # Build searchable text from block metadata
                 parts = []
-                if block_instance.name:
+                if hasattr(block_instance, "name") and block_instance.name:
                     parts.append(block_instance.name)
-                if block_instance.description:
+                if (
+                    hasattr(block_instance, "description")
+                    and block_instance.description
+                ):
                     parts.append(block_instance.description)
-                if block_instance.categories:
+                if hasattr(block_instance, "categories") and block_instance.categories:
+                    # Convert BlockCategory enum to strings
                     parts.append(
                         " ".join(str(cat.value) for cat in block_instance.categories)
                     )
 
-                # Add input schema field descriptions
-                block_input_fields = block_instance.input_schema.model_fields
-                parts += [
-                    f"{field_name}: {field_info.description}"
-                    for field_name, field_info in block_input_fields.items()
-                    if field_info.description
-                ]
+                # Add input/output schema info
+                if hasattr(block_instance, "input_schema"):
+                    schema = block_instance.input_schema
+                    if hasattr(schema, "model_json_schema"):
+                        schema_dict = schema.model_json_schema()
+                        if "properties" in schema_dict:
+                            for prop_name, prop_info in schema_dict[
+                                "properties"
+                            ].items():
+                                if "description" in prop_info:
+                                    parts.append(
+                                        f"{prop_name}: {prop_info['description']}"
+                                    )
 
                 searchable_text = " ".join(parts)
 
+                # Convert categories set of enums to list of strings for JSON serialization
+                categories = getattr(block_instance, "categories", set())
                 categories_list = (
-                    [cat.value for cat in block_instance.categories]
-                    if block_instance.categories
-                    else []
-                )
-
-                # Extract provider names from credentials fields
-                credentials_info = (
-                    block_instance.input_schema.get_credentials_fields_info()
-                )
-                is_integration = len(credentials_info) > 0
-                provider_names = [
-                    provider.value.lower()
-                    for info in credentials_info.values()
-                    for provider in info.provider
-                ]
-
-                # Check if block has LlmModel field in input schema
-                has_llm_model_field = any(
-                    _contains_type(field.annotation, LlmModel)
-                    for field in block_instance.input_schema.model_fields.values()
+                    [cat.value for cat in categories] if categories else []
                 )
 
                 items.append(
@@ -252,11 +235,8 @@ class BlockHandler(ContentHandler):
                         content_type=ContentType.BLOCK,
                         searchable_text=searchable_text,
                         metadata={
-                            "name": block_instance.name,
+                            "name": getattr(block_instance, "name", ""),
                             "categories": categories_list,
-                            "providers": provider_names,
-                            "has_llm_model_field": has_llm_model_field,
-                            "is_integration": is_integration,
                         },
                         user_id=None,  # Blocks are public
                     )

autogpt_platform/backend/backend/api/features/store/content_handlers_test.py

@@ -82,10 +82,9 @@ async def test_block_handler_get_missing_items(mocker):
     mock_block_instance.description = "Performs calculations"
     mock_block_instance.categories = [MagicMock(value="MATH")]
     mock_block_instance.disabled = False
-    mock_field = MagicMock()
-    mock_field.description = "Math expression to evaluate"
-    mock_block_instance.input_schema.model_fields = {"expression": mock_field}
-    mock_block_instance.input_schema.get_credentials_fields_info.return_value = {}
+    mock_block_instance.input_schema.model_json_schema.return_value = {
+        "properties": {"expression": {"description": "Math expression to evaluate"}}
+    }
     mock_block_class.return_value = mock_block_instance
 
     mock_blocks = {"block-uuid-1": mock_block_class}
@@ -310,19 +309,19 @@ async def test_content_handlers_registry():
 
 
 @pytest.mark.asyncio(loop_scope="session")
-async def test_block_handler_handles_empty_attributes():
-    """Test BlockHandler handles blocks with empty/falsy attribute values."""
+async def test_block_handler_handles_missing_attributes():
+    """Test BlockHandler gracefully handles blocks with missing attributes."""
     handler = BlockHandler()
 
-    # Mock block with empty values (all attributes exist but are falsy)
+    # Mock block with minimal attributes
     mock_block_class = MagicMock()
     mock_block_instance = MagicMock()
     mock_block_instance.name = "Minimal Block"
     mock_block_instance.disabled = False
-    mock_block_instance.description = ""
-    mock_block_instance.categories = set()
-    mock_block_instance.input_schema.model_fields = {}
-    mock_block_instance.input_schema.get_credentials_fields_info.return_value = {}
+    # No description, categories, or schema
+    del mock_block_instance.description
+    del mock_block_instance.categories
+    del mock_block_instance.input_schema
     mock_block_class.return_value = mock_block_instance
 
     mock_blocks = {"block-minimal": mock_block_class}
@@ -353,8 +352,6 @@ async def test_block_handler_skips_failed_blocks():
     good_instance.description = "Works fine"
     good_instance.categories = []
     good_instance.disabled = False
-    good_instance.input_schema.model_fields = {}
-    good_instance.input_schema.get_credentials_fields_info.return_value = {}
     good_block.return_value = good_instance
 
     bad_block = MagicMock()

autogpt_platform/backend/backend/api/features/v1.py

@@ -126,9 +126,6 @@ v1_router = APIRouter()
 ########################################################
 
 
-_tally_background_tasks: set[asyncio.Task] = set()
-
-
 @v1_router.post(
     "/auth/user",
     summary="Get or create user",
@@ -137,24 +134,6 @@ _tally_background_tasks: set[asyncio.Task] = set()
 )
 async def get_or_create_user_route(user_data: dict = Security(get_jwt_payload)):
     user = await get_or_create_user(user_data)
-
-    # Fire-and-forget: populate business understanding from Tally form.
-    # We use created_at proximity instead of an is_new flag because
-    # get_or_create_user is cached — a separate is_new return value would be
-    # unreliable on repeated calls within the cache TTL.
-    age_seconds = (datetime.now(timezone.utc) - user.created_at).total_seconds()
-    if age_seconds < 30:
-        try:
-            from backend.data.tally import populate_understanding_from_tally
-
-            task = asyncio.create_task(
-                populate_understanding_from_tally(user.id, user.email)
-            )
-            _tally_background_tasks.add(task)
-            task.add_done_callback(_tally_background_tasks.discard)
-        except Exception:
-            logger.debug("Failed to start Tally population task", exc_info=True)
-
     return user.model_dump()
 
 

autogpt_platform/backend/backend/api/features/v1_test.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 from io import BytesIO
 from unittest.mock import AsyncMock, Mock, patch
 
@@ -43,7 +43,6 @@ def test_get_or_create_user_route(
 ) -> None:
     """Test get or create user endpoint"""
     mock_user = Mock()
-    mock_user.created_at = datetime.now(timezone.utc)
     mock_user.model_dump.return_value = {
         "id": test_user_id,
         "email": "test@example.com",

autogpt_platform/backend/backend/api/rest_api.py

@@ -41,11 +41,11 @@ import backend.data.user
 import backend.integrations.webhooks.utils
 import backend.util.service
 import backend.util.settings
-from backend.api.features.library.exceptions import (
-    FolderAlreadyExistsError,
-    FolderValidationError,
-)
 from backend.blocks.llm import DEFAULT_LLM_MODEL
+from backend.copilot.completion_consumer import (
+    start_completion_consumer,
+    stop_completion_consumer,
+)
 from backend.data.model import Credentials
 from backend.integrations.providers import ProviderName
 from backend.monitoring.instrumentation import instrument_fastapi
@@ -123,9 +123,21 @@ async def lifespan_context(app: fastapi.FastAPI):
     await backend.data.graph.migrate_llm_models(DEFAULT_LLM_MODEL)
     await backend.integrations.webhooks.utils.migrate_legacy_triggered_graphs()
 
+    # Start chat completion consumer for Redis Streams notifications
+    try:
+        await start_completion_consumer()
+    except Exception as e:
+        logger.warning(f"Could not start chat completion consumer: {e}")
+
     with launch_darkly_context():
         yield
 
+    # Stop chat completion consumer
+    try:
+        await stop_completion_consumer()
+    except Exception as e:
+        logger.warning(f"Error stopping chat completion consumer: {e}")
+
     try:
         await shutdown_cloud_storage_handler()
     except Exception as e:
@@ -265,10 +277,6 @@ async def validation_error_handler(
 
 
 app.add_exception_handler(PrismaError, handle_internal_http_error(500))
-app.add_exception_handler(
-    FolderAlreadyExistsError, handle_internal_http_error(409, False)
-)
-app.add_exception_handler(FolderValidationError, handle_internal_http_error(400, False))
 app.add_exception_handler(NotFoundError, handle_internal_http_error(404, False))
 app.add_exception_handler(NotAuthorizedError, handle_internal_http_error(403, False))
 app.add_exception_handler(RequestValidationError, validation_error_handler)

autogpt_platform/backend/backend/app.py

@@ -24,7 +24,7 @@ def run_processes(*processes: "AppProcess", **kwargs):
         # Run the last process in the foreground.
         processes[-1].start(background=False, **kwargs)
     finally:
-        for process in reversed(processes):
+        for process in processes:
             try:
                 process.stop()
             except Exception as e:

autogpt_platform/backend/backend/blocks/telegram/_api.py

@@ -1,182 +0,0 @@
-"""
-Telegram Bot API helper functions.
-
-Provides utilities for making authenticated requests to the Telegram Bot API.
-"""
-
-import logging
-from io import BytesIO
-from typing import Any, Optional
-
-from pydantic import BaseModel
-
-from backend.data.model import APIKeyCredentials
-from backend.util.request import Requests
-
-logger = logging.getLogger(__name__)
-
-TELEGRAM_API_BASE = "https://api.telegram.org"
-
-
-class TelegramMessageResult(BaseModel, extra="allow"):
-    """Result from Telegram send/edit message API calls."""
-
-    message_id: int = 0
-    chat: dict[str, Any] = {}
-    date: int = 0
-    text: str = ""
-
-
-class TelegramFileResult(BaseModel, extra="allow"):
-    """Result from Telegram getFile API call."""
-
-    file_id: str = ""
-    file_unique_id: str = ""
-    file_size: int = 0
-    file_path: str = ""
-
-
-class TelegramAPIException(ValueError):
-    """Exception raised for Telegram API errors."""
-
-    def __init__(self, message: str, error_code: int = 0):
-        super().__init__(message)
-        self.error_code = error_code
-
-
-def get_bot_api_url(bot_token: str, method: str) -> str:
-    """Construct Telegram Bot API URL for a method."""
-    return f"{TELEGRAM_API_BASE}/bot{bot_token}/{method}"
-
-
-def get_file_url(bot_token: str, file_path: str) -> str:
-    """Construct Telegram file download URL."""
-    return f"{TELEGRAM_API_BASE}/file/bot{bot_token}/{file_path}"
-
-
-async def call_telegram_api(
-    credentials: APIKeyCredentials,
-    method: str,
-    data: Optional[dict[str, Any]] = None,
-) -> TelegramMessageResult:
-    """
-    Make a request to the Telegram Bot API.
-
-    Args:
-        credentials: Bot token credentials
-        method: API method name (e.g., "sendMessage", "getFile")
-        data: Request parameters
-
-    Returns:
-        API response result
-
-    Raises:
-        TelegramAPIException: If the API returns an error
-    """
-    token = credentials.api_key.get_secret_value()
-    url = get_bot_api_url(token, method)
-
-    response = await Requests().post(url, json=data or {})
-    result = response.json()
-
-    if not result.get("ok"):
-        error_code = result.get("error_code", 0)
-        description = result.get("description", "Unknown error")
-        raise TelegramAPIException(description, error_code)
-
-    return TelegramMessageResult(**result.get("result", {}))
-
-
-async def call_telegram_api_with_file(
-    credentials: APIKeyCredentials,
-    method: str,
-    file_field: str,
-    file_data: bytes,
-    filename: str,
-    content_type: str,
-    data: Optional[dict[str, Any]] = None,
-) -> TelegramMessageResult:
-    """
-    Make a multipart/form-data request to the Telegram Bot API with a file upload.
-
-    Args:
-        credentials: Bot token credentials
-        method: API method name (e.g., "sendPhoto", "sendVoice")
-        file_field: Form field name for the file (e.g., "photo", "voice")
-        file_data: Raw file bytes
-        filename: Filename for the upload
-        content_type: MIME type of the file
-        data: Additional form parameters
-
-    Returns:
-        API response result
-
-    Raises:
-        TelegramAPIException: If the API returns an error
-    """
-    token = credentials.api_key.get_secret_value()
-    url = get_bot_api_url(token, method)
-
-    files = [(file_field, (filename, BytesIO(file_data), content_type))]
-
-    response = await Requests().post(url, files=files, data=data or {})
-    result = response.json()
-
-    if not result.get("ok"):
-        error_code = result.get("error_code", 0)
-        description = result.get("description", "Unknown error")
-        raise TelegramAPIException(description, error_code)
-
-    return TelegramMessageResult(**result.get("result", {}))
-
-
-async def get_file_info(
-    credentials: APIKeyCredentials, file_id: str
-) -> TelegramFileResult:
-    """
-    Get file information from Telegram.
-
-    Args:
-        credentials: Bot token credentials
-        file_id: Telegram file_id from message
-
-    Returns:
-        File info dict containing file_id, file_unique_id, file_size, file_path
-    """
-    result = await call_telegram_api(credentials, "getFile", {"file_id": file_id})
-    return TelegramFileResult(**result.model_dump())
-
-
-async def get_file_download_url(credentials: APIKeyCredentials, file_id: str) -> str:
-    """
-    Get the download URL for a Telegram file.
-
-    Args:
-        credentials: Bot token credentials
-        file_id: Telegram file_id from message
-
-    Returns:
-        Full download URL
-    """
-    token = credentials.api_key.get_secret_value()
-    result = await get_file_info(credentials, file_id)
-    file_path = result.file_path
-    if not file_path:
-        raise TelegramAPIException("No file_path returned from getFile")
-    return get_file_url(token, file_path)
-
-
-async def download_telegram_file(credentials: APIKeyCredentials, file_id: str) -> bytes:
-    """
-    Download a file from Telegram servers.
-
-    Args:
-        credentials: Bot token credentials
-        file_id: Telegram file_id
-
-    Returns:
-        File content as bytes
-    """
-    url = await get_file_download_url(credentials, file_id)
-    response = await Requests().get(url)
-    return response.content

autogpt_platform/backend/backend/blocks/telegram/_auth.py

@@ -1,43 +0,0 @@
-"""
-Telegram Bot credentials handling.
-
-Telegram bots use an API key (bot token) obtained from @BotFather.
-"""
-
-from typing import Literal
-
-from pydantic import SecretStr
-
-from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput
-from backend.integrations.providers import ProviderName
-
-# Bot token credentials (API key style)
-TelegramCredentials = APIKeyCredentials
-TelegramCredentialsInput = CredentialsMetaInput[
-    Literal[ProviderName.TELEGRAM], Literal["api_key"]
-]
-
-
-def TelegramCredentialsField() -> TelegramCredentialsInput:
-    """Creates a Telegram bot token credentials field."""
-    return CredentialsField(
-        description="Telegram Bot API token from @BotFather. "
-        "Create a bot at https://t.me/BotFather to get your token."
-    )
-
-
-# Test credentials for unit tests
-TEST_CREDENTIALS = APIKeyCredentials(
-    id="01234567-89ab-cdef-0123-456789abcdef",
-    provider="telegram",
-    api_key=SecretStr("test_telegram_bot_token"),
-    title="Mock Telegram Bot Token",
-    expires_at=None,
-)
-
-TEST_CREDENTIALS_INPUT = {
-    "provider": TEST_CREDENTIALS.provider,
-    "id": TEST_CREDENTIALS.id,
-    "type": TEST_CREDENTIALS.type,
-    "title": TEST_CREDENTIALS.title,
-}

autogpt_platform/backend/backend/blocks/telegram/triggers.py

@@ -1,377 +0,0 @@
-"""
-Telegram trigger blocks for receiving messages via webhooks.
-"""
-
-import logging
-
-from pydantic import BaseModel
-
-from backend.blocks._base import (
-    Block,
-    BlockCategory,
-    BlockOutput,
-    BlockSchemaInput,
-    BlockSchemaOutput,
-    BlockWebhookConfig,
-)
-from backend.data.model import SchemaField
-from backend.integrations.providers import ProviderName
-from backend.integrations.webhooks.telegram import TelegramWebhookType
-
-from ._auth import (
-    TEST_CREDENTIALS,
-    TEST_CREDENTIALS_INPUT,
-    TelegramCredentialsField,
-    TelegramCredentialsInput,
-)
-
-logger = logging.getLogger(__name__)
-
-
-# Example payload for testing
-EXAMPLE_MESSAGE_PAYLOAD = {
-    "update_id": 123456789,
-    "message": {
-        "message_id": 1,
-        "from": {
-            "id": 12345678,
-            "is_bot": False,
-            "first_name": "John",
-            "last_name": "Doe",
-            "username": "johndoe",
-            "language_code": "en",
-        },
-        "chat": {
-            "id": 12345678,
-            "first_name": "John",
-            "last_name": "Doe",
-            "username": "johndoe",
-            "type": "private",
-        },
-        "date": 1234567890,
-        "text": "Hello, bot!",
-    },
-}
-
-
-class TelegramTriggerBase:
-    """Base class for Telegram trigger blocks."""
-
-    class Input(BlockSchemaInput):
-        credentials: TelegramCredentialsInput = TelegramCredentialsField()
-        payload: dict = SchemaField(hidden=True, default_factory=dict)
-
-
-class TelegramMessageTriggerBlock(TelegramTriggerBase, Block):
-    """
-    Triggers when a message is received or edited in your Telegram bot.
-
-    Supports text, photos, voice messages, audio files, documents, and videos.
-    Connect the outputs to other blocks to process messages and send responses.
-    """
-
-    class Input(TelegramTriggerBase.Input):
-        class EventsFilter(BaseModel):
-            """Filter for message types to receive."""
-
-            text: bool = True
-            photo: bool = False
-            voice: bool = False
-            audio: bool = False
-            document: bool = False
-            video: bool = False
-            edited_message: bool = False
-
-        events: EventsFilter = SchemaField(
-            title="Message Types", description="Types of messages to receive"
-        )
-
-    class Output(BlockSchemaOutput):
-        payload: dict = SchemaField(
-            description="The complete webhook payload from Telegram"
-        )
-        chat_id: int = SchemaField(
-            description="The chat ID where the message was received. "
-            "Use this to send replies."
-        )
-        message_id: int = SchemaField(description="The unique message ID")
-        user_id: int = SchemaField(description="The user ID who sent the message")
-        username: str = SchemaField(description="Username of the sender (may be empty)")
-        first_name: str = SchemaField(description="First name of the sender")
-        event: str = SchemaField(
-            description="The message type (text, photo, voice, audio, etc.)"
-        )
-        text: str = SchemaField(
-            description="Text content of the message (for text messages)"
-        )
-        photo_file_id: str = SchemaField(
-            description="File ID of the photo (for photo messages). "
-            "Use GetTelegramFileBlock to download."
-        )
-        voice_file_id: str = SchemaField(
-            description="File ID of the voice message (for voice messages). "
-            "Use GetTelegramFileBlock to download."
-        )
-        audio_file_id: str = SchemaField(
-            description="File ID of the audio file (for audio messages). "
-            "Use GetTelegramFileBlock to download."
-        )
-        file_id: str = SchemaField(
-            description="File ID for document/video messages. "
-            "Use GetTelegramFileBlock to download."
-        )
-        file_name: str = SchemaField(
-            description="Original filename (for document/audio messages)"
-        )
-        caption: str = SchemaField(description="Caption for media messages")
-        is_edited: bool = SchemaField(
-            description="Whether this is an edit of a previously sent message"
-        )
-
-    def __init__(self):
-        super().__init__(
-            id="4435e4e0-df6e-4301-8f35-ad70b12fc9ec",
-            description="Triggers when a message is received or edited in your Telegram bot. "
-            "Supports text, photos, voice messages, audio files, documents, and videos.",
-            categories={BlockCategory.SOCIAL},
-            input_schema=TelegramMessageTriggerBlock.Input,
-            output_schema=TelegramMessageTriggerBlock.Output,
-            webhook_config=BlockWebhookConfig(
-                provider=ProviderName.TELEGRAM,
-                webhook_type=TelegramWebhookType.BOT,
-                resource_format="bot",
-                event_filter_input="events",
-                event_format="message.{event}",
-            ),
-            test_input={
-                "events": {"text": True, "photo": True},
-                "credentials": TEST_CREDENTIALS_INPUT,
-                "payload": EXAMPLE_MESSAGE_PAYLOAD,
-            },
-            test_credentials=TEST_CREDENTIALS,
-            test_output=[
-                ("payload", EXAMPLE_MESSAGE_PAYLOAD),
-                ("chat_id", 12345678),
-                ("message_id", 1),
-                ("user_id", 12345678),
-                ("username", "johndoe"),
-                ("first_name", "John"),
-                ("is_edited", False),
-                ("event", "text"),
-                ("text", "Hello, bot!"),
-                ("photo_file_id", ""),
-                ("voice_file_id", ""),
-                ("audio_file_id", ""),
-                ("file_id", ""),
-                ("file_name", ""),
-                ("caption", ""),
-            ],
-        )
-
-    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        payload = input_data.payload
-        is_edited = "edited_message" in payload
-        message = payload.get("message") or payload.get("edited_message", {})
-
-        # Extract common fields
-        chat = message.get("chat", {})
-        sender = message.get("from", {})
-
-        yield "payload", payload
-        yield "chat_id", chat.get("id", 0)
-        yield "message_id", message.get("message_id", 0)
-        yield "user_id", sender.get("id", 0)
-        yield "username", sender.get("username", "")
-        yield "first_name", sender.get("first_name", "")
-        yield "is_edited", is_edited
-
-        # For edited messages, yield event as "edited_message" and extract
-        # all content fields from the edited message body
-        if is_edited:
-            yield "event", "edited_message"
-            yield "text", message.get("text", "")
-            photos = message.get("photo", [])
-            yield "photo_file_id", photos[-1].get("file_id", "") if photos else ""
-            voice = message.get("voice", {})
-            yield "voice_file_id", voice.get("file_id", "")
-            audio = message.get("audio", {})
-            yield "audio_file_id", audio.get("file_id", "")
-            document = message.get("document", {})
-            video = message.get("video", {})
-            yield "file_id", (document.get("file_id", "") or video.get("file_id", ""))
-            yield "file_name", (
-                document.get("file_name", "") or audio.get("file_name", "")
-            )
-            yield "caption", message.get("caption", "")
-        # Determine message type and extract content
-        elif "text" in message:
-            yield "event", "text"
-            yield "text", message.get("text", "")
-            yield "photo_file_id", ""
-            yield "voice_file_id", ""
-            yield "audio_file_id", ""
-            yield "file_id", ""
-            yield "file_name", ""
-            yield "caption", ""
-        elif "photo" in message:
-            # Get the largest photo (last in array)
-            photos = message.get("photo", [])
-            photo_fid = photos[-1].get("file_id", "") if photos else ""
-            yield "event", "photo"
-            yield "text", ""
-            yield "photo_file_id", photo_fid
-            yield "voice_file_id", ""
-            yield "audio_file_id", ""
-            yield "file_id", ""
-            yield "file_name", ""
-            yield "caption", message.get("caption", "")
-        elif "voice" in message:
-            voice = message.get("voice", {})
-            yield "event", "voice"
-            yield "text", ""
-            yield "photo_file_id", ""
-            yield "voice_file_id", voice.get("file_id", "")
-            yield "audio_file_id", ""
-            yield "file_id", ""
-            yield "file_name", ""
-            yield "caption", message.get("caption", "")
-        elif "audio" in message:
-            audio = message.get("audio", {})
-            yield "event", "audio"
-            yield "text", ""
-            yield "photo_file_id", ""
-            yield "voice_file_id", ""
-            yield "audio_file_id", audio.get("file_id", "")
-            yield "file_id", ""
-            yield "file_name", audio.get("file_name", "")
-            yield "caption", message.get("caption", "")
-        elif "document" in message:
-            document = message.get("document", {})
-            yield "event", "document"
-            yield "text", ""
-            yield "photo_file_id", ""
-            yield "voice_file_id", ""
-            yield "audio_file_id", ""
-            yield "file_id", document.get("file_id", "")
-            yield "file_name", document.get("file_name", "")
-            yield "caption", message.get("caption", "")
-        elif "video" in message:
-            video = message.get("video", {})
-            yield "event", "video"
-            yield "text", ""
-            yield "photo_file_id", ""
-            yield "voice_file_id", ""
-            yield "audio_file_id", ""
-            yield "file_id", video.get("file_id", "")
-            yield "file_name", video.get("file_name", "")
-            yield "caption", message.get("caption", "")
-        else:
-            yield "event", "other"
-            yield "text", ""
-            yield "photo_file_id", ""
-            yield "voice_file_id", ""
-            yield "audio_file_id", ""
-            yield "file_id", ""
-            yield "file_name", ""
-            yield "caption", ""
-
-
-# Example payload for reaction trigger testing
-EXAMPLE_REACTION_PAYLOAD = {
-    "update_id": 123456790,
-    "message_reaction": {
-        "chat": {
-            "id": 12345678,
-            "first_name": "John",
-            "last_name": "Doe",
-            "username": "johndoe",
-            "type": "private",
-        },
-        "message_id": 42,
-        "user": {
-            "id": 12345678,
-            "is_bot": False,
-            "first_name": "John",
-            "username": "johndoe",
-        },
-        "date": 1234567890,
-        "new_reaction": [{"type": "emoji", "emoji": "👍"}],
-        "old_reaction": [],
-    },
-}
-
-
-class TelegramMessageReactionTriggerBlock(TelegramTriggerBase, Block):
-    """
-    Triggers when a reaction to a message is changed.
-
-    Works automatically in private chats. In group chats, the bot must be
-    an administrator to receive reaction updates.
-    """
-
-    class Input(TelegramTriggerBase.Input):
-        pass
-
-    class Output(BlockSchemaOutput):
-        payload: dict = SchemaField(
-            description="The complete webhook payload from Telegram"
-        )
-        chat_id: int = SchemaField(
-            description="The chat ID where the reaction occurred"
-        )
-        message_id: int = SchemaField(description="The message ID that was reacted to")
-        user_id: int = SchemaField(description="The user ID who changed the reaction")
-        username: str = SchemaField(description="Username of the user (may be empty)")
-        new_reactions: list = SchemaField(
-            description="List of new reactions on the message"
-        )
-        old_reactions: list = SchemaField(
-            description="List of previous reactions on the message"
-        )
-
-    def __init__(self):
-        super().__init__(
-            id="82525328-9368-4966-8f0c-cd78e80181fd",
-            description="Triggers when a reaction to a message is changed. "
-            "Works in private chats automatically. "
-            "In groups, the bot must be an administrator.",
-            categories={BlockCategory.SOCIAL},
-            input_schema=TelegramMessageReactionTriggerBlock.Input,
-            output_schema=TelegramMessageReactionTriggerBlock.Output,
-            webhook_config=BlockWebhookConfig(
-                provider=ProviderName.TELEGRAM,
-                webhook_type=TelegramWebhookType.BOT,
-                resource_format="bot",
-                event_filter_input="",
-                event_format="message_reaction",
-            ),
-            test_input={
-                "credentials": TEST_CREDENTIALS_INPUT,
-                "payload": EXAMPLE_REACTION_PAYLOAD,
-            },
-            test_credentials=TEST_CREDENTIALS,
-            test_output=[
-                ("payload", EXAMPLE_REACTION_PAYLOAD),
-                ("chat_id", 12345678),
-                ("message_id", 42),
-                ("user_id", 12345678),
-                ("username", "johndoe"),
-                ("new_reactions", [{"type": "emoji", "emoji": "👍"}]),
-                ("old_reactions", []),
-            ],
-        )
-
-    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        payload = input_data.payload
-        reaction = payload.get("message_reaction", {})
-
-        chat = reaction.get("chat", {})
-        user = reaction.get("user", {})
-
-        yield "payload", payload
-        yield "chat_id", chat.get("id", 0)
-        yield "message_id", reaction.get("message_id", 0)
-        yield "user_id", user.get("id", 0)
-        yield "username", user.get("username", "")
-        yield "new_reactions", reaction.get("new_reaction", [])
-        yield "old_reactions", reaction.get("old_reaction", [])

autogpt_platform/backend/backend/cli/generate_openapi_json.py

@@ -34,12 +34,10 @@ def main(output: Path, pretty: bool):
     """Generate and output the OpenAPI JSON specification."""
     openapi_schema = get_openapi_schema()
 
-    json_output = json.dumps(
-        openapi_schema, indent=2 if pretty else None, ensure_ascii=False
-    )
+    json_output = json.dumps(openapi_schema, indent=2 if pretty else None)
 
     if output:
-        output.write_text(json_output, encoding="utf-8")
+        output.write_text(json_output)
         click.echo(f"✅ OpenAPI specification written to {output}\n\nPreview:")
         click.echo(f"\n{json_output[:500]} ...")
     else:

autogpt_platform/backend/backend/copilot/completion_consumer.py

@@ -0,0 +1,349 @@
+"""Redis Streams consumer for operation completion messages.
+
+This module provides a consumer (ChatCompletionConsumer) that listens for
+completion notifications (OperationCompleteMessage) from external services
+(like Agent Generator) and triggers the appropriate stream registry and
+chat service updates via process_operation_success/process_operation_failure.
+
+Why Redis Streams instead of RabbitMQ?
+--------------------------------------
+While the project typically uses RabbitMQ for async task queues (e.g., execution
+queue), Redis Streams was chosen for chat completion notifications because:
+
+1. **Unified Infrastructure**: The SSE reconnection feature already uses Redis
+   Streams (via stream_registry) for message persistence and replay. Using Redis
+   Streams for completion notifications keeps all chat streaming infrastructure
+   in one system, simplifying operations and reducing cross-system coordination.
+
+2. **Message Replay**: Redis Streams support XREAD with arbitrary message IDs,
+   allowing consumers to replay missed messages after reconnection. This aligns
+   with the SSE reconnection pattern where clients can resume from last_message_id.
+
+3. **Consumer Groups with XAUTOCLAIM**: Redis consumer groups provide automatic
+   load balancing across pods with explicit message claiming (XAUTOCLAIM) for
+   recovering from dead consumers - ideal for the completion callback pattern.
+
+4. **Lower Latency**: For real-time SSE updates, Redis (already in-memory for
+   stream_registry) provides lower latency than an additional RabbitMQ hop.
+
+5. **Atomicity with Task State**: Completion processing often needs to update
+   task metadata stored in Redis. Keeping both in Redis enables simpler
+   transactional semantics without distributed coordination.
+
+The consumer uses Redis Streams with consumer groups for reliable message
+processing across multiple platform pods, with XAUTOCLAIM for reclaiming
+stale pending messages from dead consumers.
+"""
+
+import asyncio
+import logging
+import uuid
+from typing import Any
+
+import orjson
+from pydantic import BaseModel
+from redis.exceptions import ResponseError
+
+from backend.data.redis_client import get_redis_async
+
+from . import stream_registry
+from .completion_handler import process_operation_failure, process_operation_success
+from .config import ChatConfig
+
+logger = logging.getLogger(__name__)
+config = ChatConfig()
+
+
+class OperationCompleteMessage(BaseModel):
+    """Message format for operation completion notifications."""
+
+    operation_id: str
+    task_id: str
+    success: bool
+    result: dict | str | None = None
+    error: str | None = None
+
+
+class ChatCompletionConsumer:
+    """Consumer for chat operation completion messages from Redis Streams.
+
+    Database operations are handled through the chat_db() accessor, which
+    routes through DatabaseManager RPC when Prisma is not directly connected.
+
+    Uses Redis consumer groups to allow multiple platform pods to consume
+    messages reliably with automatic redelivery on failure.
+    """
+
+    def __init__(self):
+        self._consumer_task: asyncio.Task | None = None
+        self._running = False
+        self._consumer_name = f"consumer-{uuid.uuid4().hex[:8]}"
+
+    async def start(self) -> None:
+        """Start the completion consumer."""
+        if self._running:
+            logger.warning("Completion consumer already running")
+            return
+
+        # Create consumer group if it doesn't exist
+        try:
+            redis = await get_redis_async()
+            await redis.xgroup_create(
+                config.stream_completion_name,
+                config.stream_consumer_group,
+                id="0",
+                mkstream=True,
+            )
+            logger.info(
+                f"Created consumer group '{config.stream_consumer_group}' "
+                f"on stream '{config.stream_completion_name}'"
+            )
+        except ResponseError as e:
+            if "BUSYGROUP" in str(e):
+                logger.debug(
+                    f"Consumer group '{config.stream_consumer_group}' already exists"
+                )
+            else:
+                raise
+
+        self._running = True
+        self._consumer_task = asyncio.create_task(self._consume_messages())
+        logger.info(
+            f"Chat completion consumer started (consumer: {self._consumer_name})"
+        )
+
+    async def stop(self) -> None:
+        """Stop the completion consumer."""
+        self._running = False
+
+        if self._consumer_task:
+            self._consumer_task.cancel()
+            try:
+                await self._consumer_task
+            except asyncio.CancelledError:
+                pass
+            self._consumer_task = None
+
+        logger.info("Chat completion consumer stopped")
+
+    async def _consume_messages(self) -> None:
+        """Main message consumption loop with retry logic."""
+        max_retries = 10
+        retry_delay = 5  # seconds
+        retry_count = 0
+        block_timeout = 5000  # milliseconds
+
+        while self._running and retry_count < max_retries:
+            try:
+                redis = await get_redis_async()
+
+                # Reset retry count on successful connection
+                retry_count = 0
+
+                while self._running:
+                    # First, claim any stale pending messages from dead consumers
+                    # Redis does NOT auto-redeliver pending messages; we must explicitly
+                    # claim them using XAUTOCLAIM
+                    try:
+                        claimed_result = await redis.xautoclaim(
+                            name=config.stream_completion_name,
+                            groupname=config.stream_consumer_group,
+                            consumername=self._consumer_name,
+                            min_idle_time=config.stream_claim_min_idle_ms,
+                            start_id="0-0",
+                            count=10,
+                        )
+                        # xautoclaim returns: (next_start_id, [(id, data), ...], [deleted_ids])
+                        if claimed_result and len(claimed_result) >= 2:
+                            claimed_entries = claimed_result[1]
+                            if claimed_entries:
+                                logger.info(
+                                    f"Claimed {len(claimed_entries)} stale pending messages"
+                                )
+                                for entry_id, data in claimed_entries:
+                                    if not self._running:
+                                        return
+                                    await self._process_entry(redis, entry_id, data)
+                    except Exception as e:
+                        logger.warning(f"XAUTOCLAIM failed (non-fatal): {e}")
+
+                    # Read new messages from the stream
+                    messages = await redis.xreadgroup(
+                        groupname=config.stream_consumer_group,
+                        consumername=self._consumer_name,
+                        streams={config.stream_completion_name: ">"},
+                        block=block_timeout,
+                        count=10,
+                    )
+
+                    if not messages:
+                        continue
+
+                    for stream_name, entries in messages:
+                        for entry_id, data in entries:
+                            if not self._running:
+                                return
+                            await self._process_entry(redis, entry_id, data)
+
+            except asyncio.CancelledError:
+                logger.info("Consumer cancelled")
+                return
+            except Exception as e:
+                retry_count += 1
+                logger.error(
+                    f"Consumer error (retry {retry_count}/{max_retries}): {e}",
+                    exc_info=True,
+                )
+                if self._running and retry_count < max_retries:
+                    await asyncio.sleep(retry_delay)
+                else:
+                    logger.error("Max retries reached, stopping consumer")
+                    return
+
+    async def _process_entry(
+        self, redis: Any, entry_id: str, data: dict[str, Any]
+    ) -> None:
+        """Process a single stream entry and acknowledge it on success.
+
+        Args:
+            redis: Redis client connection
+            entry_id: The stream entry ID
+            data: The entry data dict
+        """
+        try:
+            # Handle the message
+            message_data = data.get("data")
+            if message_data:
+                await self._handle_message(
+                    message_data.encode()
+                    if isinstance(message_data, str)
+                    else message_data
+                )
+
+            # Acknowledge the message after successful processing
+            await redis.xack(
+                config.stream_completion_name,
+                config.stream_consumer_group,
+                entry_id,
+            )
+        except Exception as e:
+            logger.error(
+                f"Error processing completion message {entry_id}: {e}",
+                exc_info=True,
+            )
+            # Message remains in pending state and will be claimed by
+            # XAUTOCLAIM after min_idle_time expires
+
+    async def _handle_message(self, body: bytes) -> None:
+        """Handle a completion message."""
+        try:
+            data = orjson.loads(body)
+            message = OperationCompleteMessage(**data)
+        except Exception as e:
+            logger.error(f"Failed to parse completion message: {e}")
+            return
+
+        logger.info(
+            f"[COMPLETION] Received completion for operation {message.operation_id} "
+            f"(task_id={message.task_id}, success={message.success})"
+        )
+
+        # Find task in registry
+        task = await stream_registry.find_task_by_operation_id(message.operation_id)
+        if task is None:
+            task = await stream_registry.get_task(message.task_id)
+
+        if task is None:
+            logger.warning(
+                f"[COMPLETION] Task not found for operation {message.operation_id} "
+                f"(task_id={message.task_id})"
+            )
+            return
+
+        logger.info(
+            f"[COMPLETION] Found task: task_id={task.task_id}, "
+            f"session_id={task.session_id}, tool_call_id={task.tool_call_id}"
+        )
+
+        # Guard against empty task fields
+        if not task.task_id or not task.session_id or not task.tool_call_id:
+            logger.error(
+                f"[COMPLETION] Task has empty critical fields! "
+                f"task_id={task.task_id!r}, session_id={task.session_id!r}, "
+                f"tool_call_id={task.tool_call_id!r}"
+            )
+            return
+
+        if message.success:
+            await self._handle_success(task, message)
+        else:
+            await self._handle_failure(task, message)
+
+    async def _handle_success(
+        self,
+        task: stream_registry.ActiveTask,
+        message: OperationCompleteMessage,
+    ) -> None:
+        """Handle successful operation completion."""
+        await process_operation_success(task, message.result)
+
+    async def _handle_failure(
+        self,
+        task: stream_registry.ActiveTask,
+        message: OperationCompleteMessage,
+    ) -> None:
+        """Handle failed operation completion."""
+        await process_operation_failure(task, message.error)
+
+
+# Module-level consumer instance
+_consumer: ChatCompletionConsumer | None = None
+
+
+async def start_completion_consumer() -> None:
+    """Start the global completion consumer."""
+    global _consumer
+    if _consumer is None:
+        _consumer = ChatCompletionConsumer()
+    await _consumer.start()
+
+
+async def stop_completion_consumer() -> None:
+    """Stop the global completion consumer."""
+    global _consumer
+    if _consumer:
+        await _consumer.stop()
+        _consumer = None
+
+
+async def publish_operation_complete(
+    operation_id: str,
+    task_id: str,
+    success: bool,
+    result: dict | str | None = None,
+    error: str | None = None,
+) -> None:
+    """Publish an operation completion message to Redis Streams.
+
+    Args:
+        operation_id: The operation ID that completed.
+        task_id: The task ID associated with the operation.
+        success: Whether the operation succeeded.
+        result: The result data (for success).
+        error: The error message (for failure).
+    """
+    message = OperationCompleteMessage(
+        operation_id=operation_id,
+        task_id=task_id,
+        success=success,
+        result=result,
+        error=error,
+    )
+
+    redis = await get_redis_async()
+    await redis.xadd(
+        config.stream_completion_name,
+        {"data": message.model_dump_json()},
+        maxlen=config.stream_max_length,
+    )
+    logger.info(f"Published completion for operation {operation_id}")

autogpt_platform/backend/backend/copilot/completion_handler.py

@@ -0,0 +1,329 @@
+"""Shared completion handling for operation success and failure.
+
+This module provides common logic for handling operation completion from both:
+- The Redis Streams consumer (completion_consumer.py)
+- The HTTP webhook endpoint (routes.py)
+"""
+
+import logging
+from typing import Any
+
+import orjson
+
+from backend.data.db_accessors import chat_db
+
+from . import service as chat_service
+from . import stream_registry
+from .response_model import StreamError, StreamToolOutputAvailable
+from .tools.models import ErrorResponse
+
+logger = logging.getLogger(__name__)
+
+# Tools that produce agent_json that needs to be saved to library
+AGENT_GENERATION_TOOLS = {"create_agent", "edit_agent"}
+
+# Keys that should be stripped from agent_json when returning in error responses
+SENSITIVE_KEYS = frozenset(
+    {
+        "api_key",
+        "apikey",
+        "api_secret",
+        "password",
+        "secret",
+        "credentials",
+        "credential",
+        "token",
+        "access_token",
+        "refresh_token",
+        "private_key",
+        "privatekey",
+        "auth",
+        "authorization",
+    }
+)
+
+
+def _sanitize_agent_json(obj: Any) -> Any:
+    """Recursively sanitize agent_json by removing sensitive keys.
+
+    Args:
+        obj: The object to sanitize (dict, list, or primitive)
+
+    Returns:
+        Sanitized copy with sensitive keys removed/redacted
+    """
+    if isinstance(obj, dict):
+        return {
+            k: "[REDACTED]" if k.lower() in SENSITIVE_KEYS else _sanitize_agent_json(v)
+            for k, v in obj.items()
+        }
+    elif isinstance(obj, list):
+        return [_sanitize_agent_json(item) for item in obj]
+    else:
+        return obj
+
+
+class ToolMessageUpdateError(Exception):
+    """Raised when updating a tool message in the database fails."""
+
+    pass
+
+
+async def _update_tool_message(
+    session_id: str,
+    tool_call_id: str,
+    content: str,
+) -> None:
+    """Update tool message in database using the chat_db accessor.
+
+    Routes through DatabaseManager RPC when Prisma is not directly
+    connected (e.g. in the CoPilot Executor microservice).
+
+    Args:
+        session_id: The session ID
+        tool_call_id: The tool call ID to update
+        content: The new content for the message
+
+    Raises:
+        ToolMessageUpdateError: If the database update fails.
+    """
+    try:
+        updated = await chat_db().update_tool_message_content(
+            session_id=session_id,
+            tool_call_id=tool_call_id,
+            new_content=content,
+        )
+        if not updated:
+            raise ToolMessageUpdateError(
+                f"No message found with tool_call_id="
+                f"{tool_call_id} in session {session_id}"
+            )
+    except ToolMessageUpdateError:
+        raise
+    except Exception as e:
+        logger.error(
+            f"[COMPLETION] Failed to update tool message: {e}",
+            exc_info=True,
+        )
+        raise ToolMessageUpdateError(
+            f"Failed to update tool message for tool call #{tool_call_id}: {e}"
+        ) from e
+
+
+def serialize_result(result: dict | list | str | int | float | bool | None) -> str:
+    """Serialize result to JSON string with sensible defaults.
+
+    Args:
+        result: The result to serialize. Can be a dict, list, string,
+            number, boolean, or None.
+
+    Returns:
+        JSON string representation of the result. Returns '{"status": "completed"}'
+        only when result is explicitly None.
+    """
+    if isinstance(result, str):
+        return result
+    if result is None:
+        return '{"status": "completed"}'
+    return orjson.dumps(result).decode("utf-8")
+
+
+async def _save_agent_from_result(
+    result: dict[str, Any],
+    user_id: str | None,
+    tool_name: str,
+) -> dict[str, Any]:
+    """Save agent to library if result contains agent_json.
+
+    Args:
+        result: The result dict that may contain agent_json
+        user_id: The user ID to save the agent for
+        tool_name: The tool name (create_agent or edit_agent)
+
+    Returns:
+        Updated result dict with saved agent details, or original result if no agent_json
+    """
+    if not user_id:
+        logger.warning("[COMPLETION] Cannot save agent: no user_id in task")
+        return result
+
+    agent_json = result.get("agent_json")
+    if not agent_json:
+        logger.warning(
+            f"[COMPLETION] {tool_name} completed but no agent_json in result"
+        )
+        return result
+
+    try:
+        from .tools.agent_generator import save_agent_to_library
+
+        is_update = tool_name == "edit_agent"
+        created_graph, library_agent = await save_agent_to_library(
+            agent_json, user_id, is_update=is_update
+        )
+
+        logger.info(
+            f"[COMPLETION] Saved agent '{created_graph.name}' to library "
+            f"(graph_id={created_graph.id}, library_agent_id={library_agent.id})"
+        )
+
+        # Return a response similar to AgentSavedResponse
+        return {
+            "type": "agent_saved",
+            "message": f"Agent '{created_graph.name}' has been saved to your library!",
+            "agent_id": created_graph.id,
+            "agent_name": created_graph.name,
+            "library_agent_id": library_agent.id,
+            "library_agent_link": f"/library/agents/{library_agent.id}",
+            "agent_page_link": f"/build?flowID={created_graph.id}",
+        }
+    except Exception as e:
+        logger.error(
+            f"[COMPLETION] Failed to save agent to library: {e}",
+            exc_info=True,
+        )
+        # Return error but don't fail the whole operation
+        # Sanitize agent_json to remove sensitive keys before returning
+        return {
+            "type": "error",
+            "message": f"Agent was generated but failed to save: {str(e)}",
+            "error": str(e),
+            "agent_json": _sanitize_agent_json(agent_json),
+        }
+
+
+async def process_operation_success(
+    task: stream_registry.ActiveTask,
+    result: dict | str | None,
+) -> None:
+    """Handle successful operation completion.
+
+    Publishes the result to the stream registry, updates the database,
+    generates LLM continuation, and marks the task as completed.
+
+    Args:
+        task: The active task that completed
+        result: The result data from the operation
+
+    Raises:
+        ToolMessageUpdateError: If the database update fails. The task
+            will be marked as failed instead of completed.
+    """
+    # For agent generation tools, save the agent to library
+    if task.tool_name in AGENT_GENERATION_TOOLS and isinstance(result, dict):
+        result = await _save_agent_from_result(result, task.user_id, task.tool_name)
+
+    # Serialize result for output (only substitute default when result is exactly None)
+    result_output = result if result is not None else {"status": "completed"}
+    output_str = (
+        result_output
+        if isinstance(result_output, str)
+        else orjson.dumps(result_output).decode("utf-8")
+    )
+
+    # Publish result to stream registry
+    await stream_registry.publish_chunk(
+        task.task_id,
+        StreamToolOutputAvailable(
+            toolCallId=task.tool_call_id,
+            toolName=task.tool_name,
+            output=output_str,
+            success=True,
+        ),
+    )
+
+    # Update pending operation in database
+    # If this fails, we must not continue to mark the task as completed
+    result_str = serialize_result(result)
+    try:
+        await _update_tool_message(
+            session_id=task.session_id,
+            tool_call_id=task.tool_call_id,
+            content=result_str,
+        )
+    except ToolMessageUpdateError:
+        # DB update failed - mark task as failed to avoid inconsistent state
+        logger.error(
+            f"[COMPLETION] DB update failed for task {task.task_id}, "
+            "marking as failed instead of completed"
+        )
+        await stream_registry.publish_chunk(
+            task.task_id,
+            StreamError(errorText="Failed to save operation result to database"),
+        )
+        await stream_registry.mark_task_completed(task.task_id, status="failed")
+        raise
+
+    # Generate LLM continuation with streaming
+    try:
+        await chat_service._generate_llm_continuation_with_streaming(
+            session_id=task.session_id,
+            user_id=task.user_id,
+            task_id=task.task_id,
+        )
+    except Exception as e:
+        logger.error(
+            f"[COMPLETION] Failed to generate LLM continuation: {e}",
+            exc_info=True,
+        )
+
+    # Mark task as completed and release Redis lock
+    await stream_registry.mark_task_completed(task.task_id, status="completed")
+    try:
+        await chat_service._mark_operation_completed(task.tool_call_id)
+    except Exception as e:
+        logger.error(f"[COMPLETION] Failed to mark operation completed: {e}")
+
+    logger.info(
+        f"[COMPLETION] Successfully processed completion for task {task.task_id}"
+    )
+
+
+async def process_operation_failure(
+    task: stream_registry.ActiveTask,
+    error: str | None,
+) -> None:
+    """Handle failed operation completion.
+
+    Publishes the error to the stream registry, updates the database
+    with the error response, and marks the task as failed.
+
+    Args:
+        task: The active task that failed
+        error: The error message from the operation
+    """
+    error_msg = error or "Operation failed"
+
+    # Publish error to stream registry
+    await stream_registry.publish_chunk(
+        task.task_id,
+        StreamError(errorText=error_msg),
+    )
+
+    # Update pending operation with error
+    # If this fails, we still continue to mark the task as failed
+    error_response = ErrorResponse(
+        message=error_msg,
+        error=error,
+    )
+    try:
+        await _update_tool_message(
+            session_id=task.session_id,
+            tool_call_id=task.tool_call_id,
+            content=error_response.model_dump_json(),
+        )
+    except ToolMessageUpdateError:
+        # DB update failed - log but continue with cleanup
+        logger.error(
+            f"[COMPLETION] DB update failed while processing failure for task {task.task_id}, "
+            "continuing with cleanup"
+        )
+
+    # Mark task as failed and release Redis lock
+    await stream_registry.mark_task_completed(task.task_id, status="failed")
+    try:
+        await chat_service._mark_operation_completed(task.tool_call_id)
+    except Exception as e:
+        logger.error(f"[COMPLETION] Failed to mark operation completed: {e}")
+
+    logger.info(f"[COMPLETION] Processed failure for task {task.task_id}: {error_msg}")

autogpt_platform/backend/backend/copilot/config.py

@@ -36,6 +36,14 @@ class ChatConfig(BaseSettings):
         default=30, description="Maximum number of agent schedules"
     )
 
+    # Long-running operation configuration
+    long_running_operation_ttl: int = Field(
+        default=3600,
+        description="TTL in seconds for long-running operation deduplication lock "
+        "(1 hour, matches stream_ttl). Prevents duplicate operations if pod dies. "
+        "For longer operations, the stream_registry heartbeat keeps them alive.",
+    )
+
     # Stream registry configuration for SSE reconnection
     stream_ttl: int = Field(
         default=3600,
@@ -51,14 +59,36 @@ class ChatConfig(BaseSettings):
         description="Maximum number of messages to store per stream",
     )
 
-    # Redis key prefixes for stream registry
-    session_meta_prefix: str = Field(
-        default="chat:task:meta:",
-        description="Prefix for session metadata hash keys",
+    # Redis Streams configuration for completion consumer
+    stream_completion_name: str = Field(
+        default="chat:completions",
+        description="Redis Stream name for operation completions",
     )
-    turn_stream_prefix: str = Field(
+    stream_consumer_group: str = Field(
+        default="chat_consumers",
+        description="Consumer group name for completion stream",
+    )
+    stream_claim_min_idle_ms: int = Field(
+        default=60000,
+        description="Minimum idle time in milliseconds before claiming pending messages from dead consumers",
+    )
+
+    # Redis key prefixes for stream registry
+    task_meta_prefix: str = Field(
+        default="chat:task:meta:",
+        description="Prefix for task metadata hash keys",
+    )
+    task_stream_prefix: str = Field(
         default="chat:stream:",
-        description="Prefix for turn message stream keys",
+        description="Prefix for task message stream keys",
+    )
+    task_op_prefix: str = Field(
+        default="chat:task:op:",
+        description="Prefix for operation ID to task ID mapping keys",
+    )
+    internal_api_key: str | None = Field(
+        default=None,
+        description="API key for internal webhook callbacks (env: CHAT_INTERNAL_API_KEY)",
     )
 
     # Langfuse Prompt Management Configuration
@@ -85,7 +115,7 @@ class ChatConfig(BaseSettings):
     )
     claude_agent_max_subtasks: int = Field(
         default=10,
-        description="Max number of concurrent sub-agent Tasks the SDK can run per session.",
+        description="Max number of sub-agent Tasks the SDK can spawn per session.",
     )
     claude_agent_use_resume: bool = Field(
         default=True,
@@ -130,6 +160,14 @@ class ChatConfig(BaseSettings):
                 v = "https://openrouter.ai/api/v1"
         return v
 
+    @field_validator("internal_api_key", mode="before")
+    @classmethod
+    def get_internal_api_key(cls, v):
+        """Get internal API key from environment if not provided."""
+        if v is None:
+            v = os.getenv("CHAT_INTERNAL_API_KEY")
+        return v
+
     @field_validator("use_claude_agent_sdk", mode="before")
     @classmethod
     def get_use_claude_agent_sdk(cls, v):

autogpt_platform/backend/backend/copilot/executor/manager.py

@@ -4,7 +4,6 @@ This module contains the CoPilotExecutor class that consumes chat tasks from
 RabbitMQ and processes them using a thread pool, following the graph executor pattern.
 """
 
-import asyncio
 import logging
 import os
 import threading
@@ -26,7 +25,7 @@ from backend.util.process import AppProcess
 from backend.util.retry import continuous_retry
 from backend.util.settings import Settings
 
-from .processor import execute_copilot_turn, init_worker
+from .processor import execute_copilot_task, init_worker
 from .utils import (
     COPILOT_CANCEL_QUEUE_NAME,
     COPILOT_EXECUTION_QUEUE_NAME,
@@ -182,13 +181,13 @@ class CoPilotExecutor(AppProcess):
             self._executor.shutdown(wait=False)
 
         # Release any remaining locks
-        for session_id, lock in list(self._task_locks.items()):
+        for task_id, lock in list(self._task_locks.items()):
             try:
                 lock.release()
-                logger.info(f"[cleanup {pid}] Released lock for {session_id}")
+                logger.info(f"[cleanup {pid}] Released lock for {task_id}")
             except Exception as e:
                 logger.error(
-                    f"[cleanup {pid}] Failed to release lock for {session_id}: {e}"
+                    f"[cleanup {pid}] Failed to release lock for {task_id}: {e}"
                 )
 
         logger.info(f"[cleanup {pid}] Graceful shutdown completed")
@@ -268,20 +267,20 @@ class CoPilotExecutor(AppProcess):
     ):
         """Handle cancel message from FANOUT exchange."""
         request = CancelCoPilotEvent.model_validate_json(body)
-        session_id = request.session_id
-        if not session_id:
-            logger.warning("Cancel message missing 'session_id'")
+        task_id = request.task_id
+        if not task_id:
+            logger.warning("Cancel message missing 'task_id'")
             return
-        if session_id not in self.active_tasks:
-            logger.debug(f"Cancel received for {session_id} but not active")
+        if task_id not in self.active_tasks:
+            logger.debug(f"Cancel received for {task_id} but not active")
             return
 
-        _, cancel_event = self.active_tasks[session_id]
-        logger.info(f"Received cancel for {session_id}")
+        _, cancel_event = self.active_tasks[task_id]
+        logger.info(f"Received cancel for {task_id}")
         if not cancel_event.is_set():
             cancel_event.set()
         else:
-            logger.debug(f"Cancel already set for {session_id}")
+            logger.debug(f"Cancel already set for {task_id}")
 
     def _handle_run_message(
         self,
@@ -353,12 +352,12 @@ class CoPilotExecutor(AppProcess):
             ack_message(reject=True, requeue=False)
             return
 
-        session_id = entry.session_id
+        task_id = entry.task_id
 
-        # Check for local duplicate - session is already running on this executor
-        if session_id in self.active_tasks:
+        # Check for local duplicate - task is already running on this executor
+        if task_id in self.active_tasks:
             logger.warning(
-                f"Session {session_id} already running locally, rejecting duplicate"
+                f"Task {task_id} already running locally, rejecting duplicate"
             )
             ack_message(reject=True, requeue=False)
             return
@@ -366,69 +365,64 @@ class CoPilotExecutor(AppProcess):
         # Try to acquire cluster-wide lock
         cluster_lock = ClusterLock(
             redis=redis.get_redis(),
-            key=f"copilot:session:{session_id}:lock",
+            key=f"copilot:task:{task_id}:lock",
             owner_id=self.executor_id,
             timeout=settings.config.cluster_lock_timeout,
         )
         current_owner = cluster_lock.try_acquire()
         if current_owner != self.executor_id:
             if current_owner is not None:
-                logger.warning(
-                    f"Session {session_id} already running on pod {current_owner}"
-                )
+                logger.warning(f"Task {task_id} already running on pod {current_owner}")
                 ack_message(reject=True, requeue=False)
             else:
                 logger.warning(
-                    f"Could not acquire lock for {session_id} - Redis unavailable"
+                    f"Could not acquire lock for {task_id} - Redis unavailable"
                 )
                 ack_message(reject=True, requeue=True)
             return
 
         # Execute the task
         try:
-            self._task_locks[session_id] = cluster_lock
+            self._task_locks[task_id] = cluster_lock
 
             logger.info(
-                f"Acquired cluster lock for {session_id}, "
-                f"executor_id={self.executor_id}"
+                f"Acquired cluster lock for {task_id}, executor_id={self.executor_id}"
             )
 
             cancel_event = threading.Event()
             future = self.executor.submit(
-                execute_copilot_turn, entry, cancel_event, cluster_lock
+                execute_copilot_task, entry, cancel_event, cluster_lock
             )
-            self.active_tasks[session_id] = (future, cancel_event)
+            self.active_tasks[task_id] = (future, cancel_event)
         except Exception as e:
-            logger.warning(f"Failed to setup execution for {session_id}: {e}")
+            logger.warning(f"Failed to setup execution for {task_id}: {e}")
             cluster_lock.release()
-            if session_id in self._task_locks:
-                del self._task_locks[session_id]
+            if task_id in self._task_locks:
+                del self._task_locks[task_id]
             ack_message(reject=True, requeue=True)
             return
 
         self._update_metrics()
 
         def on_run_done(f: Future):
-            logger.info(f"Run completed for {session_id}")
-            error_msg = None
+            logger.info(f"Run completed for {task_id}")
             try:
                 if exec_error := f.exception():
-                    error_msg = str(exec_error) or type(exec_error).__name__
-                    logger.error(f"Execution for {session_id} failed: {error_msg}")
+                    logger.error(f"Execution for {task_id} failed: {exec_error}")
+                    # Don't requeue failed tasks - they've been marked as failed
+                    # in the stream registry. Requeuing would cause infinite retries
+                    # for deterministic failures.
                     ack_message(reject=True, requeue=False)
                 else:
                     ack_message(reject=False, requeue=False)
-            except asyncio.CancelledError:
-                logger.info(f"Run completion callback cancelled for {session_id}")
             except BaseException as e:
-                error_msg = str(e) or type(e).__name__
-                logger.exception(f"Error in run completion callback: {error_msg}")
+                logger.exception(f"Error in run completion callback: {e}")
             finally:
                 # Release the cluster lock
-                if session_id in self._task_locks:
-                    logger.info(f"Releasing cluster lock for {session_id}")
-                    self._task_locks[session_id].release()
-                    del self._task_locks[session_id]
+                if task_id in self._task_locks:
+                    logger.info(f"Releasing cluster lock for {task_id}")
+                    self._task_locks[task_id].release()
+                    del self._task_locks[task_id]
                 self._cleanup_completed_tasks()
 
         future.add_done_callback(on_run_done)
@@ -439,11 +433,11 @@ class CoPilotExecutor(AppProcess):
         """Remove completed futures from active_tasks and update metrics."""
         completed_tasks = []
         with self._active_tasks_lock:
-            for session_id, (future, _) in list(self.active_tasks.items()):
+            for task_id, (future, _) in list(self.active_tasks.items()):
                 if future.done():
-                    completed_tasks.append(session_id)
-                    self.active_tasks.pop(session_id, None)
-                    logger.info(f"Cleaned up completed session {session_id}")
+                    completed_tasks.append(task_id)
+                    self.active_tasks.pop(task_id, None)
+                    logger.info(f"Cleaned up completed task {task_id}")
 
         self._update_metrics()
         return completed_tasks

autogpt_platform/backend/backend/copilot/executor/processor.py

@@ -1,6 +1,6 @@
 """CoPilot execution processor - per-worker execution logic.
 
-This module contains the processor class that handles CoPilot session execution
+This module contains the processor class that handles CoPilot task execution
 in a thread-local context, following the graph executor pattern.
 """
 
@@ -12,7 +12,7 @@ import time
 from backend.copilot import service as copilot_service
 from backend.copilot import stream_registry
 from backend.copilot.config import ChatConfig
-from backend.copilot.response_model import StreamFinish
+from backend.copilot.response_model import StreamError, StreamFinish, StreamFinishStep
 from backend.copilot.sdk import service as sdk_service
 from backend.executor.cluster_lock import ClusterLock
 from backend.util.decorator import error_logged
@@ -32,17 +32,17 @@ logger = TruncatedLogger(logging.getLogger(__name__), prefix="[CoPilotExecutor]"
 _tls = threading.local()
 
 
-def execute_copilot_turn(
+def execute_copilot_task(
     entry: CoPilotExecutionEntry,
     cancel: threading.Event,
     cluster_lock: ClusterLock,
 ):
-    """Execute a single CoPilot turn (user message → AI response).
+    """Execute a CoPilot task using the thread-local processor.
 
     This function is the entry point called by the thread pool executor.
 
     Args:
-        entry: The turn payload
+        entry: The task payload
         cancel: Threading event to signal cancellation
         cluster_lock: Distributed lock for this execution
     """
@@ -76,16 +76,16 @@ def cleanup_worker():
 
 
 class CoPilotProcessor:
-    """Per-worker execution logic for CoPilot sessions.
+    """Per-worker execution logic for CoPilot tasks.
 
     This class is instantiated once per worker thread and handles the execution
-    of CoPilot chat generation sessions. It maintains an async event loop for
+    of CoPilot chat generation tasks. It maintains an async event loop for
     running the async service code.
 
     The execution flow:
-        1. Session entry is picked from RabbitMQ queue
-        2. Manager submits to thread pool
-        3. Processor executes in its event loop
+        1. CoPilot task is picked from RabbitMQ queue
+        2. Manager submits task to thread pool
+        3. Processor executes the task in its event loop
         4. Results are published to Redis Streams
     """
 
@@ -125,10 +125,7 @@ class CoPilotProcessor:
             )
             future.result(timeout=5)
         except Exception as e:
-            error_msg = str(e) or type(e).__name__
-            logger.warning(
-                f"[CoPilotExecutor] Worker {self.tid} cleanup error: {error_msg}"
-            )
+            logger.warning(f"[CoPilotExecutor] Worker {self.tid} cleanup error: {e}")
 
         # Stop the event loop
         self.execution_loop.call_soon_threadsafe(self.execution_loop.stop)
@@ -142,17 +139,19 @@ class CoPilotProcessor:
         cancel: threading.Event,
         cluster_lock: ClusterLock,
     ):
-        """Execute a CoPilot turn.
+        """Execute a CoPilot task.
 
-        Runs the async logic in the worker's event loop and handles errors.
+        This is the main entry point for task execution. It runs the async
+        execution logic in the worker's event loop and handles errors.
 
         Args:
-            entry: The turn payload containing session and message info
+            entry: The task payload containing session and message info
             cancel: Threading event to signal cancellation
             cluster_lock: Distributed lock to prevent duplicate execution
         """
         log = CoPilotLogMetadata(
             logging.getLogger(__name__),
+            task_id=entry.task_id,
             session_id=entry.session_id,
             user_id=entry.user_id,
         )
@@ -160,30 +159,38 @@ class CoPilotProcessor:
 
         start_time = time.monotonic()
 
-        # Run the async execution in our event loop
-        future = asyncio.run_coroutine_threadsafe(
-            self._execute_async(entry, cancel, cluster_lock, log),
-            self.execution_loop,
-        )
+        try:
+            # Run the async execution in our event loop
+            future = asyncio.run_coroutine_threadsafe(
+                self._execute_async(entry, cancel, cluster_lock, log),
+                self.execution_loop,
+            )
 
-        # Wait for completion, checking cancel periodically
-        while not future.done():
-            try:
-                future.result(timeout=1.0)
-            except asyncio.TimeoutError:
-                if cancel.is_set():
-                    log.info("Cancellation requested")
-                    future.cancel()
-                    break
-                # Refresh cluster lock to maintain ownership
-                cluster_lock.refresh()
+            # Wait for completion, checking cancel periodically
+            while not future.done():
+                try:
+                    future.result(timeout=1.0)
+                except asyncio.TimeoutError:
+                    if cancel.is_set():
+                        log.info("Cancellation requested")
+                        future.cancel()
+                        break
+                    # Refresh cluster lock to maintain ownership
+                    cluster_lock.refresh()
 
-        if not future.cancelled():
-            # Get result to propagate any exceptions
-            future.result()
+            if not future.cancelled():
+                # Get result to propagate any exceptions
+                future.result()
 
-        elapsed = time.monotonic() - start_time
-        log.info(f"Execution completed in {elapsed:.2f}s")
+            elapsed = time.monotonic() - start_time
+            log.info(f"Execution completed in {elapsed:.2f}s")
+
+        except Exception as e:
+            elapsed = time.monotonic() - start_time
+            log.error(f"Execution failed after {elapsed:.2f}s: {e}")
+            # Note: _execute_async already marks the task as failed before re-raising,
+            # so we don't call _mark_task_failed here to avoid duplicate error events.
+            raise
 
     async def _execute_async(
         self,
@@ -192,20 +199,19 @@ class CoPilotProcessor:
         cluster_lock: ClusterLock,
         log: CoPilotLogMetadata,
     ):
-        """Async execution logic for a CoPilot turn.
+        """Async execution logic for CoPilot task.
 
-        Calls the stream_chat_completion service function and publishes
-        results to the stream registry.
+        This method calls the existing stream_chat_completion service function
+        and publishes results to the stream registry.
 
         Args:
-            entry: The turn payload
+            entry: The task payload
             cancel: Threading event to signal cancellation
             cluster_lock: Distributed lock for refresh
-            log: Structured logger
+            log: Structured logger for this task
         """
         last_refresh = time.monotonic()
         refresh_interval = 30.0  # Refresh lock every 30 seconds
-        error_msg = None
 
         try:
             # Choose service based on LaunchDarkly flag
@@ -222,7 +228,7 @@ class CoPilotProcessor:
             )
             log.info(f"Using {'SDK' if use_sdk else 'standard'} service")
 
-            # Stream chat completion and publish chunks to Redis.
+            # Stream chat completion and publish chunks to Redis
             async for chunk in stream_fn(
                 session_id=entry.session_id,
                 message=entry.message if entry.message else None,
@@ -230,47 +236,56 @@ class CoPilotProcessor:
                 user_id=entry.user_id,
                 context=entry.context,
             ):
+                # Check for cancellation
                 if cancel.is_set():
-                    log.info("Cancel requested, breaking stream")
-                    break
+                    log.info("Cancelled during streaming")
+                    await stream_registry.publish_chunk(
+                        entry.task_id, StreamError(errorText="Operation cancelled")
+                    )
+                    await stream_registry.publish_chunk(
+                        entry.task_id, StreamFinishStep()
+                    )
+                    await stream_registry.publish_chunk(entry.task_id, StreamFinish())
+                    await stream_registry.mark_task_completed(
+                        entry.task_id, status="failed"
+                    )
+                    return
 
+                # Refresh cluster lock periodically
                 current_time = time.monotonic()
                 if current_time - last_refresh >= refresh_interval:
                     cluster_lock.refresh()
                     last_refresh = current_time
 
-                # Skip StreamFinish — mark_session_completed publishes it.
-                if isinstance(chunk, StreamFinish):
-                    continue
+                # Publish chunk to stream registry
+                await stream_registry.publish_chunk(entry.task_id, chunk)
 
-                try:
-                    await stream_registry.publish_chunk(entry.turn_id, chunk)
-                except Exception as e:
-                    log.error(
-                        f"Error publishing chunk {type(chunk).__name__}: {e}",
-                        exc_info=True,
-                    )
+            # Mark task as completed
+            await stream_registry.mark_task_completed(entry.task_id, status="completed")
+            log.info("Task completed successfully")
 
-            # Stream loop completed
-            if cancel.is_set():
-                log.info("Stream cancelled by user")
-
-        except BaseException as e:
-            # Handle all exceptions (including CancelledError) with appropriate logging
-            if isinstance(e, asyncio.CancelledError):
-                log.info("Turn cancelled")
-                error_msg = "Operation cancelled"
-            else:
-                error_msg = str(e) or type(e).__name__
-                log.error(f"Turn failed: {error_msg}")
+        except asyncio.CancelledError:
+            log.info("Task cancelled")
+            await stream_registry.mark_task_completed(
+                entry.task_id,
+                status="failed",
+                error_message="Task was cancelled",
+            )
             raise
-        finally:
-            # If no exception but user cancelled, still mark as cancelled
-            if not error_msg and cancel.is_set():
-                error_msg = "Operation cancelled"
-            try:
-                await stream_registry.mark_session_completed(
-                    entry.session_id, error_message=error_msg
-                )
-            except Exception as mark_err:
-                log.error(f"Failed to mark session completed: {mark_err}")
+
+        except Exception as e:
+            log.error(f"Task failed: {e}")
+            await self._mark_task_failed(entry.task_id, str(e))
+            raise
+
+    async def _mark_task_failed(self, task_id: str, error_message: str):
+        """Mark a task as failed and publish error to stream registry."""
+        try:
+            await stream_registry.publish_chunk(
+                task_id, StreamError(errorText=error_message)
+            )
+            await stream_registry.publish_chunk(task_id, StreamFinishStep())
+            await stream_registry.publish_chunk(task_id, StreamFinish())
+            await stream_registry.mark_task_completed(task_id, status="failed")
+        except Exception as e:
+            logger.error(f"Failed to mark task {task_id} as failed: {e}")

autogpt_platform/backend/backend/copilot/executor/utils.py

@@ -28,7 +28,7 @@ class CoPilotLogMetadata(TruncatedLogger):
     Args:
         logger: The underlying logger instance
         max_length: Maximum log message length before truncation
-        **kwargs: Metadata key-value pairs (e.g., session_id="xyz", turn_id="abc")
+        **kwargs: Metadata key-value pairs (e.g., task_id="abc", session_id="xyz")
             These are added to json_fields in cloud mode, or to the prefix in local mode.
     """
 
@@ -135,15 +135,18 @@ class CoPilotExecutionEntry(BaseModel):
     This model represents a chat generation task to be processed by the executor.
     """
 
-    session_id: str
-    """Chat session ID (also used for dedup/locking)"""
+    task_id: str
+    """Unique identifier for this task (used for stream registry)"""
 
-    turn_id: str = ""
-    """Per-turn UUID for Redis stream isolation"""
+    session_id: str
+    """Chat session ID"""
 
     user_id: str | None
     """User ID (may be None for anonymous users)"""
 
+    operation_id: str
+    """Operation ID for webhook callbacks and completion tracking"""
+
     message: str
     """User's message to process"""
 
@@ -157,37 +160,40 @@ class CoPilotExecutionEntry(BaseModel):
 class CancelCoPilotEvent(BaseModel):
     """Event to cancel a CoPilot operation."""
 
-    session_id: str
-    """Session ID to cancel"""
+    task_id: str
+    """Task ID to cancel"""
 
 
 # ============ Queue Publishing Helpers ============ #
 
 
-async def enqueue_copilot_turn(
+async def enqueue_copilot_task(
+    task_id: str,
     session_id: str,
     user_id: str | None,
+    operation_id: str,
     message: str,
-    turn_id: str,
     is_user_message: bool = True,
     context: dict[str, str] | None = None,
 ) -> None:
     """Enqueue a CoPilot task for processing by the executor service.
 
     Args:
-        session_id: Chat session ID (also used for dedup/locking)
+        task_id: Unique identifier for this task (used for stream registry)
+        session_id: Chat session ID
         user_id: User ID (may be None for anonymous users)
+        operation_id: Operation ID for webhook callbacks and completion tracking
         message: User's message to process
-        turn_id: Per-turn UUID for Redis stream isolation
         is_user_message: Whether the message is from the user (vs system/assistant)
         context: Optional context for the message (e.g., {url: str, content: str})
     """
     from backend.util.clients import get_async_copilot_queue
 
     entry = CoPilotExecutionEntry(
+        task_id=task_id,
         session_id=session_id,
-        turn_id=turn_id,
         user_id=user_id,
+        operation_id=operation_id,
         message=message,
         is_user_message=is_user_message,
         context=context,
@@ -201,15 +207,15 @@ async def enqueue_copilot_turn(
     )
 
 
-async def enqueue_cancel_task(session_id: str) -> None:
-    """Publish a cancel request for a running CoPilot session.
+async def enqueue_cancel_task(task_id: str) -> None:
+    """Publish a cancel request for a running CoPilot task.
 
     Sends a ``CancelCoPilotEvent`` to the FANOUT exchange so all executor
     pods receive the cancellation signal.
     """
     from backend.util.clients import get_async_copilot_queue
 
-    event = CancelCoPilotEvent(session_id=session_id)
+    event = CancelCoPilotEvent(task_id=task_id)
     queue_client = await get_async_copilot_queue()
     await queue_client.publish_message(
         routing_key="",  # FANOUT ignores routing key

autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py

@@ -14,6 +14,7 @@ import pytest
 @pytest.mark.asyncio
 async def test_parallel_tool_calls_run_concurrently():
     """Multiple tool calls should complete in ~max(delays), not sum(delays)."""
+    # Import here to allow module-level mocking if needed
     from backend.copilot.response_model import (
         StreamToolInputAvailable,
         StreamToolOutputAvailable,
@@ -31,6 +32,7 @@ async def test_parallel_tool_calls_run_concurrently():
         for i in range(n_tools)
     ]
 
+    # Minimal session mock
     class FakeSession:
         session_id = "test"
         user_id = "test"
@@ -40,7 +42,7 @@ async def test_parallel_tool_calls_run_concurrently():
 
     original_yield = None
 
-    async def fake_yield(tc_list, idx, sess):
+    async def fake_yield(tc_list, idx, sess, lock=None):
         yield StreamToolInputAvailable(
             toolCallId=tc_list[idx]["id"],
             toolName=tc_list[idx]["function"]["name"],
@@ -99,7 +101,7 @@ async def test_single_tool_call_works():
         def __init__(self):
             self.messages = []
 
-    async def fake_yield(tc_list, idx, sess):
+    async def fake_yield(tc_list, idx, sess, lock=None):
         yield StreamToolInputAvailable(toolCallId="call_0", toolName="t", input={})
         yield StreamToolOutputAvailable(toolCallId="call_0", toolName="t", output="{}")
 
@@ -142,7 +144,7 @@ async def test_retryable_error_propagates():
         def __init__(self):
             self.messages = []
 
-    async def fake_yield(tc_list, idx, sess):
+    async def fake_yield(tc_list, idx, sess, lock=None):
         if idx == 1:
             raise KeyError("bad")
         from backend.copilot.response_model import StreamToolInputAvailable
@@ -173,8 +175,8 @@ async def test_retryable_error_propagates():
 
 
 @pytest.mark.asyncio
-async def test_session_shared_across_parallel_tools():
-    """All parallel tools should receive the same session instance."""
+async def test_session_lock_shared():
+    """All parallel tools should receive the same lock instance."""
     from backend.copilot.response_model import (
         StreamToolInputAvailable,
         StreamToolOutputAvailable,
@@ -197,10 +199,10 @@ async def test_session_shared_across_parallel_tools():
         def __init__(self):
             self.messages = []
 
-    observed_sessions = []
+    observed_locks = []
 
-    async def fake_yield(tc_list, idx, sess):
-        observed_sessions.append(sess)
+    async def fake_yield(tc_list, idx, sess, lock=None):
+        observed_locks.append(lock)
         yield StreamToolInputAvailable(
             toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}
         )
@@ -220,8 +222,9 @@ async def test_session_shared_across_parallel_tools():
     finally:
         svc._yield_tool_call = orig
 
-    assert len(observed_sessions) == 3
-    assert observed_sessions[0] is observed_sessions[1] is observed_sessions[2]
+    assert len(observed_locks) == 3
+    assert observed_locks[0] is observed_locks[1] is observed_locks[2]
+    assert isinstance(observed_locks[0], asyncio.Lock)
 
 
 @pytest.mark.asyncio
@@ -248,7 +251,7 @@ async def test_cancellation_cleans_up():
 
     started = asyncio.Event()
 
-    async def fake_yield(tc_list, idx, sess):
+    async def fake_yield(tc_list, idx, sess, lock=None):
         yield StreamToolInputAvailable(
             toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}
         )

autogpt_platform/backend/backend/copilot/response_model.py

@@ -5,8 +5,6 @@ This module implements the AI SDK UI Stream Protocol (v1) for streaming chat res
 See: https://ai-sdk.dev/docs/ai-sdk-ui/stream-protocol
 """
 
-import json
-import logging
 from enum import Enum
 from typing import Any
 
@@ -14,8 +12,6 @@ from pydantic import BaseModel, Field
 
 from backend.util.json import dumps as json_dumps
 
-logger = logging.getLogger(__name__)
-
 
 class ResponseType(str, Enum):
     """Types of streaming responses following AI SDK protocol."""
@@ -51,8 +47,7 @@ class StreamBaseResponse(BaseModel):
 
     def to_sse(self) -> str:
         """Convert to SSE format."""
-        json_str = self.model_dump_json(exclude_none=True)
-        return f"data: {json_str}\n\n"
+        return f"data: {self.model_dump_json()}\n\n"
 
 
 # ========== Message Lifecycle ==========
@@ -63,13 +58,15 @@ class StreamStart(StreamBaseResponse):
 
     type: ResponseType = ResponseType.START
     messageId: str = Field(..., description="Unique message ID")
-    sessionId: str | None = Field(
+    taskId: str | None = Field(
         default=None,
-        description="Session ID for SSE reconnection.",
+        description="Task ID for SSE reconnection. Clients can reconnect using GET /tasks/{taskId}/stream",
     )
 
     def to_sse(self) -> str:
-        """Convert to SSE format, excluding non-protocol fields like sessionId."""
+        """Convert to SSE format, excluding non-protocol fields like taskId."""
+        import json
+
         data: dict[str, Any] = {
             "type": self.type.value,
             "messageId": self.messageId,
@@ -166,6 +163,8 @@ class StreamToolOutputAvailable(StreamBaseResponse):
 
     def to_sse(self) -> str:
         """Convert to SSE format, excluding non-spec fields."""
+        import json
+
         data = {
             "type": self.type.value,
             "toolCallId": self.toolCallId,

autogpt_platform/backend/backend/copilot/sdk/dummy.py

@@ -1,57 +0,0 @@
-"""Dummy SDK service for testing copilot streaming.
-
-Returns mock streaming responses without calling Claude Agent SDK.
-Enable via COPILOT_TEST_MODE=true environment variable.
-
-WARNING: This is for testing only. Do not use in production.
-"""
-
-import asyncio
-import logging
-import uuid
-from collections.abc import AsyncGenerator
-
-from ..model import ChatSession
-from ..response_model import StreamBaseResponse, StreamStart, StreamTextDelta
-
-logger = logging.getLogger(__name__)
-
-
-async def stream_chat_completion_dummy(
-    session_id: str,
-    message: str | None = None,
-    tool_call_response: str | None = None,
-    is_user_message: bool = True,
-    user_id: str | None = None,
-    retry_count: int = 0,
-    session: ChatSession | None = None,
-    context: dict[str, str] | None = None,
-) -> AsyncGenerator[StreamBaseResponse, None]:
-    """Stream dummy chat completion for testing.
-
-    Returns a simple streaming response with text deltas to test:
-    - Streaming infrastructure works
-    - No timeout occurs
-    - Text arrives in chunks
-    - StreamFinish is sent by mark_session_completed
-    """
-    logger.warning(
-        f"[TEST MODE] Using dummy copilot streaming for session {session_id}"
-    )
-
-    message_id = str(uuid.uuid4())
-    text_block_id = str(uuid.uuid4())
-
-    # Start the stream
-    yield StreamStart(messageId=message_id, sessionId=session_id)
-
-    # Simulate streaming text response with delays
-    dummy_response = "I counted: 1... 2... 3. All done!"
-    words = dummy_response.split()
-
-    for i, word in enumerate(words):
-        # Add space except for last word
-        text = word if i == len(words) - 1 else f"{word} "
-        yield StreamTextDelta(id=text_block_id, delta=text)
-        # Small delay to simulate real streaming
-        await asyncio.sleep(0.1)

autogpt_platform/backend/backend/copilot/sdk/response_adapter.py

@@ -55,8 +55,13 @@ class SDKResponseAdapter:
         self.has_ended_text = False
         self.current_tool_calls: dict[str, dict[str, str]] = {}
         self.resolved_tool_calls: set[str] = set()
+        self.task_id: str | None = None
         self.step_open = False
 
+    def set_task_id(self, task_id: str) -> None:
+        """Set the task ID for reconnection support."""
+        self.task_id = task_id
+
     @property
     def has_unresolved_tool_calls(self) -> bool:
         """True when there are tool calls that haven't received output yet."""
@@ -69,7 +74,7 @@ class SDKResponseAdapter:
         if isinstance(sdk_message, SystemMessage):
             if sdk_message.subtype == "init":
                 responses.append(
-                    StreamStart(messageId=self.message_id, sessionId=self.session_id)
+                    StreamStart(messageId=self.message_id, taskId=self.task_id)
                 )
                 # Open the first step (matches non-SDK: StreamStart then StreamStartStep)
                 responses.append(StreamStartStep())

autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py

@@ -37,7 +37,9 @@ from .tool_adapter import wait_for_stash
 
 
 def _adapter() -> SDKResponseAdapter:
-    return SDKResponseAdapter(message_id="msg-1", session_id="session-1")
+    a = SDKResponseAdapter(message_id="msg-1")
+    a.set_task_id("task-1")
+    return a
 
 
 # -- SystemMessage -----------------------------------------------------------
@@ -49,7 +51,7 @@ def test_system_init_emits_start_and_step():
     assert len(results) == 2
     assert isinstance(results[0], StreamStart)
     assert results[0].messageId == "msg-1"
-    assert results[0].sessionId == "session-1"
+    assert results[0].taskId == "task-1"
     assert isinstance(results[1], StreamStartStep)
 
 

autogpt_platform/backend/backend/copilot/sdk/security_hooks.py

@@ -160,7 +160,7 @@ def create_security_hooks(
     Args:
         user_id: Current user ID for isolation validation
         sdk_cwd: SDK working directory for workspace-scoped tool validation
-        max_subtasks: Maximum concurrent Task (sub-agent) spawns allowed per session
+        max_subtasks: Maximum Task (sub-agent) spawns allowed per session
         on_stop: Callback ``(transcript_path, sdk_session_id)`` invoked when
             the SDK finishes processing — used to read the JSONL transcript
             before the CLI process exits.
@@ -172,9 +172,8 @@ def create_security_hooks(
         from claude_agent_sdk import HookMatcher
         from claude_agent_sdk.types import HookContext, HookInput, SyncHookJSONOutput
 
-        # Per-session tracking for Task sub-agent concurrency.
-        # Set of tool_use_ids that consumed a slot — len() is the active count.
-        task_tool_use_ids: set[str] = set()
+        # Per-session counter for Task sub-agent spawns
+        task_spawn_count = 0
 
         async def pre_tool_use_hook(
             input_data: HookInput,
@@ -182,6 +181,7 @@ def create_security_hooks(
             context: HookContext,
         ) -> SyncHookJSONOutput:
             """Combined pre-tool-use validation hook."""
+            nonlocal task_spawn_count
             _ = context  # unused but required by signature
             tool_name = cast(str, input_data.get("tool_name", ""))
             tool_input = cast(dict[str, Any], input_data.get("tool_input", {}))
@@ -200,18 +200,18 @@ def create_security_hooks(
                             "(remove the run_in_background parameter)."
                         ),
                     )
-                if len(task_tool_use_ids) >= max_subtasks:
+                if task_spawn_count >= max_subtasks:
                     logger.warning(
                         f"[SDK] Task limit reached ({max_subtasks}), user={user_id}"
                     )
                     return cast(
                         SyncHookJSONOutput,
                         _deny(
-                            f"Maximum {max_subtasks} concurrent sub-tasks. "
-                            "Wait for running sub-tasks to finish, "
-                            "or continue in the main conversation."
+                            f"Maximum {max_subtasks} sub-tasks per session. "
+                            "Please continue in the main conversation."
                         ),
                     )
+                task_spawn_count += 1
 
             # Strip MCP prefix for consistent validation
             is_copilot_tool = tool_name.startswith(MCP_TOOL_PREFIX)
@@ -229,24 +229,9 @@ def create_security_hooks(
             if result:
                 return cast(SyncHookJSONOutput, result)
 
-            # Reserve the Task slot only after all validations pass
-            if tool_name == "Task" and tool_use_id is not None:
-                task_tool_use_ids.add(tool_use_id)
-
             logger.debug(f"[SDK] Tool start: {tool_name}, user={user_id}")
             return cast(SyncHookJSONOutput, {})
 
-        def _release_task_slot(tool_name: str, tool_use_id: str | None) -> None:
-            """Release a Task concurrency slot if one was reserved."""
-            if tool_name == "Task" and tool_use_id in task_tool_use_ids:
-                task_tool_use_ids.discard(tool_use_id)
-                logger.info(
-                    "[SDK] Task slot released, active=%d/%d, user=%s",
-                    len(task_tool_use_ids),
-                    max_subtasks,
-                    user_id,
-                )
-
         async def post_tool_use_hook(
             input_data: HookInput,
             tool_use_id: str | None,
@@ -261,8 +246,6 @@ def create_security_hooks(
             """
             _ = context
             tool_name = cast(str, input_data.get("tool_name", ""))
-
-            _release_task_slot(tool_name, tool_use_id)
             is_builtin = not tool_name.startswith(MCP_TOOL_PREFIX)
             logger.info(
                 "[SDK] PostToolUse: %s (builtin=%s, tool_use_id=%s)",
@@ -306,9 +289,6 @@ def create_security_hooks(
                 f"[SDK] Tool failed: {tool_name}, error={error}, "
                 f"user={user_id}, tool_use_id={tool_use_id}"
             )
-
-            _release_task_slot(tool_name, tool_use_id)
-
             return cast(SyncHookJSONOutput, {})
 
         async def pre_compact_hook(

autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py

@@ -208,22 +208,19 @@ def test_bash_builtin_blocked_message_clarity():
 
 @pytest.fixture()
 def _hooks():
-    """Create security hooks and return (pre, post, post_failure) handlers."""
+    """Create security hooks and return the PreToolUse handler."""
     from .security_hooks import create_security_hooks
 
     hooks = create_security_hooks(user_id="u1", sdk_cwd=SDK_CWD, max_subtasks=2)
     pre = hooks["PreToolUse"][0].hooks[0]
-    post = hooks["PostToolUse"][0].hooks[0]
-    post_failure = hooks["PostToolUseFailure"][0].hooks[0]
-    return pre, post, post_failure
+    return pre
 
 
 @pytest.mark.skipif(not _sdk_available(), reason="claude_agent_sdk not installed")
 @pytest.mark.asyncio
 async def test_task_background_blocked(_hooks):
     """Task with run_in_background=true must be denied."""
-    pre, _, _ = _hooks
-    result = await pre(
+    result = await _hooks(
         {"tool_name": "Task", "tool_input": {"run_in_background": True, "prompt": "x"}},
         tool_use_id=None,
         context={},
@@ -236,10 +233,9 @@ async def test_task_background_blocked(_hooks):
 @pytest.mark.asyncio
 async def test_task_foreground_allowed(_hooks):
     """Task without run_in_background should be allowed."""
-    pre, _, _ = _hooks
-    result = await pre(
+    result = await _hooks(
         {"tool_name": "Task", "tool_input": {"prompt": "do stuff"}},
-        tool_use_id="tu-1",
+        tool_use_id=None,
         context={},
     )
     assert not _is_denied(result)
@@ -249,102 +245,25 @@ async def test_task_foreground_allowed(_hooks):
 @pytest.mark.asyncio
 async def test_task_limit_enforced(_hooks):
     """Task spawns beyond max_subtasks should be denied."""
-    pre, _, _ = _hooks
     # First two should pass
-    for i in range(2):
-        result = await pre(
+    for _ in range(2):
+        result = await _hooks(
             {"tool_name": "Task", "tool_input": {"prompt": "ok"}},
-            tool_use_id=f"tu-limit-{i}",
+            tool_use_id=None,
             context={},
         )
         assert not _is_denied(result)
 
     # Third should be denied (limit=2)
-    result = await pre(
+    result = await _hooks(
         {"tool_name": "Task", "tool_input": {"prompt": "over limit"}},
-        tool_use_id="tu-limit-2",
+        tool_use_id=None,
         context={},
     )
     assert _is_denied(result)
     assert "Maximum" in _reason(result)
 
 
-@pytest.mark.skipif(not _sdk_available(), reason="claude_agent_sdk not installed")
-@pytest.mark.asyncio
-async def test_task_slot_released_on_completion(_hooks):
-    """Completing a Task should free a slot so new Tasks can be spawned."""
-    pre, post, _ = _hooks
-    # Fill both slots
-    for i in range(2):
-        result = await pre(
-            {"tool_name": "Task", "tool_input": {"prompt": "ok"}},
-            tool_use_id=f"tu-comp-{i}",
-            context={},
-        )
-        assert not _is_denied(result)
-
-    # Third should be denied — at capacity
-    result = await pre(
-        {"tool_name": "Task", "tool_input": {"prompt": "over"}},
-        tool_use_id="tu-comp-2",
-        context={},
-    )
-    assert _is_denied(result)
-
-    # Complete first task — frees a slot
-    await post(
-        {"tool_name": "Task", "tool_input": {}},
-        tool_use_id="tu-comp-0",
-        context={},
-    )
-
-    # Now a new Task should be allowed
-    result = await pre(
-        {"tool_name": "Task", "tool_input": {"prompt": "after release"}},
-        tool_use_id="tu-comp-3",
-        context={},
-    )
-    assert not _is_denied(result)
-
-
-@pytest.mark.skipif(not _sdk_available(), reason="claude_agent_sdk not installed")
-@pytest.mark.asyncio
-async def test_task_slot_released_on_failure(_hooks):
-    """A failed Task should also free its concurrency slot."""
-    pre, _, post_failure = _hooks
-    # Fill both slots
-    for i in range(2):
-        result = await pre(
-            {"tool_name": "Task", "tool_input": {"prompt": "ok"}},
-            tool_use_id=f"tu-fail-{i}",
-            context={},
-        )
-        assert not _is_denied(result)
-
-    # At capacity
-    result = await pre(
-        {"tool_name": "Task", "tool_input": {"prompt": "over"}},
-        tool_use_id="tu-fail-2",
-        context={},
-    )
-    assert _is_denied(result)
-
-    # Fail first task — should free a slot
-    await post_failure(
-        {"tool_name": "Task", "tool_input": {}, "error": "something broke"},
-        tool_use_id="tu-fail-0",
-        context={},
-    )
-
-    # New Task should be allowed
-    result = await pre(
-        {"tool_name": "Task", "tool_input": {"prompt": "after failure"}},
-        tool_use_id="tu-fail-3",
-        context={},
-    )
-    assert not _is_denied(result)
-
-
 # -- _is_tool_error_or_denial ------------------------------------------------
 
 
@@ -379,9 +298,7 @@ class TestIsToolErrorOrDenial:
     def test_subtask_limit_denial(self):
         assert (
             _is_tool_error_or_denial(
-                "Maximum 2 concurrent sub-tasks. "
-                "Wait for running sub-tasks to finish, "
-                "or continue in the main conversation."
+                "Maximum 2 sub-tasks per session. Please continue in the main conversation."
             )
             is True
         )

autogpt_platform/backend/backend/copilot/sdk/service.py

@@ -13,6 +13,7 @@ from backend.data.redis_client import get_redis_async
 from backend.executor.cluster_lock import AsyncClusterLock
 from backend.util.exceptions import NotFoundError
 
+from .. import stream_registry
 from ..config import ChatConfig
 from ..model import (
     ChatMessage,
@@ -25,13 +26,19 @@ from ..response_model import (
     StreamBaseResponse,
     StreamError,
     StreamFinish,
+    StreamFinishStep,
     StreamHeartbeat,
     StreamStart,
     StreamTextDelta,
     StreamToolInputAvailable,
     StreamToolOutputAvailable,
 )
-from ..service import _build_system_prompt, _generate_session_title
+from ..service import (
+    _build_system_prompt,
+    _execute_long_running_tool_with_streaming,
+    _generate_session_title,
+)
+from ..tools.models import OperationPendingResponse, OperationStartedResponse
 from ..tools.sandbox import WORKSPACE_PREFIX, make_session_path
 from ..tracking import track_user_message
 from .response_adapter import SDKResponseAdapter
@@ -39,6 +46,7 @@ from .security_hooks import create_security_hooks
 from .tool_adapter import (
     COPILOT_TOOL_NAMES,
     SDK_DISALLOWED_TOOLS,
+    LongRunningCallback,
     create_copilot_mcp_server,
     set_execution_context,
     wait_for_stash,
@@ -75,21 +83,13 @@ class CapturedTranscript:
 
 _SDK_CWD_PREFIX = WORKSPACE_PREFIX
 
-# Special message prefixes for text-based markers (parsed by frontend)
-COPILOT_ERROR_PREFIX = "[COPILOT_ERROR]"  # Renders as ErrorCard
-COPILOT_SYSTEM_PREFIX = "[COPILOT_SYSTEM]"  # Renders as system info message
-
 # Heartbeat interval — keep SSE alive through proxies/LBs during tool execution.
-# IMPORTANT: Must be less than frontend timeout (12s in useCopilotPage.ts)
-_HEARTBEAT_INTERVAL = 10.0  # seconds
-
+_HEARTBEAT_INTERVAL = 15.0  # seconds
 
 # Appended to the system prompt to inform the agent about available tools.
 # The SDK built-in Bash is NOT available — use mcp__copilot__bash_exec instead,
 # which has kernel-level network isolation (unshare --net).
-def _build_sdk_tool_supplement(cwd: str) -> str:
-    """Build the SDK tool supplement with the actual working directory injected."""
-    return f"""
+_SDK_TOOL_SUPPLEMENT = """
 
 ## Tool notes
 
@@ -97,16 +97,9 @@ def _build_sdk_tool_supplement(cwd: str) -> str:
 - The SDK built-in Bash tool is NOT available.  Use the `bash_exec` MCP tool
   for shell commands — it runs in a network-isolated sandbox.
 
-### Working directory
-- Your working directory is: `{cwd}`
-- All SDK Read/Write/Edit/Glob/Grep tools AND `bash_exec` operate inside this
-  directory.  This is the ONLY writable path — do not attempt to read or write
-  anywhere else on the filesystem.
-- Use relative paths or absolute paths under `{cwd}` for all file operations.
-
 ### Two storage systems — CRITICAL to understand
 
-1. **Ephemeral working directory** (`{cwd}`):
+1. **Ephemeral working directory** (`/tmp/copilot-<session>/`):
    - Shared by SDK Read/Write/Edit/Glob/Grep tools AND `bash_exec`
    - Files here are **lost between turns** — do NOT rely on them persisting
    - Use for temporary work: running scripts, processing data, etc.
@@ -132,21 +125,6 @@ When you create or modify important files (code, configs, outputs), you MUST:
 2. At the start of a new turn, call `list_workspace_files` to see what files
    are available from previous turns
 
-### Sharing files with the user
-After saving a file to the persistent workspace with `write_workspace_file`,
-share it with the user by embedding the `download_url` from the response in
-your message as a Markdown link or image:
-
-- **Any file** — shows as a clickable download link:
-  `[report.csv](workspace://file_id#text/csv)`
-- **Image** — renders inline in chat:
-  `![chart](workspace://file_id#image/png)`
-- **Video** — renders inline in chat with player controls:
-  `![recording](workspace://file_id#video/mp4)`
-
-The `download_url` field in the `write_workspace_file` response is already
-in the correct format — paste it directly after the `(` in the Markdown.
-
 ### Long-running tools
 Long-running tools (create_agent, edit_agent, etc.) are handled
 asynchronously.  You will receive an immediate response; the actual result
@@ -157,10 +135,130 @@ is delivered to the user via a background stream.
   All tasks must run in the foreground.
 """
 
-
 STREAM_LOCK_PREFIX = "copilot:stream:lock:"
 
 
+def _build_long_running_callback(
+    user_id: str | None,
+) -> LongRunningCallback:
+    """Build a callback that delegates long-running tools to the non-SDK infrastructure.
+
+    Long-running tools (create_agent, edit_agent, etc.) are delegated to the
+    existing background infrastructure: stream_registry (Redis Streams),
+    database persistence, and SSE reconnection.  This means results survive
+    page refreshes / pod restarts, and the frontend shows the proper loading
+    widget with progress updates.
+
+    Args:
+        user_id: User ID for the session
+
+    The returned callback matches the ``LongRunningCallback`` signature:
+    ``(tool_name, args, session) -> MCP response dict``.
+    """
+
+    async def _callback(
+        tool_name: str, args: dict[str, Any], session: ChatSession
+    ) -> dict[str, Any]:
+        operation_id = str(uuid.uuid4())
+        task_id = str(uuid.uuid4())
+        tool_call_id = f"sdk-{uuid.uuid4().hex[:12]}"
+        session_id = session.session_id
+
+        # --- Build user-friendly messages (matches non-SDK service) ---
+        if tool_name == "create_agent":
+            desc = args.get("description", "")
+            desc_preview = (desc[:100] + "...") if len(desc) > 100 else desc
+            pending_msg = (
+                f"Creating your agent: {desc_preview}"
+                if desc_preview
+                else "Creating agent... This may take a few minutes."
+            )
+            started_msg = (
+                "Agent creation started. You can close this tab - "
+                "check your library in a few minutes."
+            )
+        elif tool_name == "edit_agent":
+            changes = args.get("changes", "")
+            changes_preview = (changes[:100] + "...") if len(changes) > 100 else changes
+            pending_msg = (
+                f"Editing agent: {changes_preview}"
+                if changes_preview
+                else "Editing agent... This may take a few minutes."
+            )
+            started_msg = (
+                "Agent edit started. You can close this tab - "
+                "check your library in a few minutes."
+            )
+        else:
+            pending_msg = f"Running {tool_name}... This may take a few minutes."
+            started_msg = (
+                f"{tool_name} started. You can close this tab - "
+                "check back in a few minutes."
+            )
+
+        # --- Register task in Redis for SSE reconnection ---
+        await stream_registry.create_task(
+            task_id=task_id,
+            session_id=session_id,
+            user_id=user_id,
+            tool_call_id=tool_call_id,
+            tool_name=tool_name,
+            operation_id=operation_id,
+        )
+
+        # --- Save OperationPendingResponse to chat history ---
+        pending_message = ChatMessage(
+            role="tool",
+            content=OperationPendingResponse(
+                message=pending_msg,
+                operation_id=operation_id,
+                tool_name=tool_name,
+            ).model_dump_json(),
+            tool_call_id=tool_call_id,
+        )
+        session.messages.append(pending_message)
+        # Collision detection happens in add_chat_messages_batch (db.py)
+        session = await upsert_chat_session(session)
+
+        # --- Spawn background task (reuses non-SDK infrastructure) ---
+        bg_task = asyncio.create_task(
+            _execute_long_running_tool_with_streaming(
+                tool_name=tool_name,
+                parameters=args,
+                tool_call_id=tool_call_id,
+                operation_id=operation_id,
+                task_id=task_id,
+                session_id=session_id,
+                user_id=user_id,
+            )
+        )
+        _background_tasks.add(bg_task)
+        bg_task.add_done_callback(_background_tasks.discard)
+        await stream_registry.set_task_asyncio_task(task_id, bg_task)
+
+        logger.info(
+            f"[SDK] Long-running tool {tool_name} delegated to background "
+            f"(operation_id={operation_id}, task_id={task_id})"
+        )
+
+        # --- Return OperationStartedResponse as MCP tool result ---
+        # This flows through SDK → response adapter → frontend, triggering
+        # the loading widget with SSE reconnection support.
+        started_json = OperationStartedResponse(
+            message=started_msg,
+            operation_id=operation_id,
+            tool_name=tool_name,
+            task_id=task_id,
+        ).model_dump_json()
+
+        return {
+            "content": [{"type": "text", "text": started_json}],
+            "isError": False,
+        }
+
+    return _callback
+
+
 def _resolve_sdk_model() -> str | None:
     """Resolve the model name for the Claude Agent SDK CLI.
 
@@ -443,20 +541,6 @@ async def stream_chat_completion_sdk(
     # Type narrowing: session is guaranteed ChatSession after the check above
     session = cast(ChatSession, session)
 
-    # Clean up stale error markers from previous turn before starting new turn
-    # If the last message contains an error marker, remove it (user is retrying)
-    if (
-        len(session.messages) > 0
-        and session.messages[-1].role == "assistant"
-        and session.messages[-1].content
-        and COPILOT_ERROR_PREFIX in session.messages[-1].content
-    ):
-        logger.info(
-            "[SDK] [%s] Removing stale error marker from previous turn",
-            session_id[:12],
-        )
-        session.messages.pop()
-
     # Append the new message to the session if it's not already there
     new_message_role = "user" if is_user_message else "assistant"
     if message and (
@@ -486,13 +570,15 @@ async def stream_chat_completion_sdk(
                 _background_tasks.add(task)
                 task.add_done_callback(_background_tasks.discard)
 
+    # Build system prompt (reuses non-SDK path with Langfuse support)
+    has_history = len(session.messages) > 1
+    system_prompt, _ = await _build_system_prompt(
+        user_id, has_conversation_history=has_history
+    )
+    system_prompt += _SDK_TOOL_SUPPLEMENT
     message_id = str(uuid.uuid4())
-    stream_id = str(uuid.uuid4())
-    stream_completed = False
-    use_resume = False
-    resume_file: str | None = None
-    captured_transcript = CapturedTranscript()
-    sdk_cwd = ""
+    task_id = str(uuid.uuid4())
+    stream_id = task_id  # Use task_id as unique stream identifier
 
     # Acquire stream lock to prevent concurrent streams to the same session
     lock = AsyncClusterLock(
@@ -513,34 +599,30 @@ async def stream_chat_completion_sdk(
             "Please wait or stop it.",
             code="stream_already_active",
         )
+        yield StreamFinish()
         return
 
-    # Make sure there is no more code between the lock acquitition and try-block.
+    yield StreamStart(messageId=message_id, taskId=task_id)
+
+    stream_completed = False
+    # Initialise variables before the try so the finally block can
+    # always attempt transcript upload regardless of errors.
+    sdk_cwd = ""
+    use_resume = False
+    resume_file: str | None = None
+    captured_transcript = CapturedTranscript()
+
     try:
-        # Build system prompt (reuses non-SDK path with Langfuse support).
-        # Pre-compute the cwd here so the exact working directory path can be
-        # injected into the supplement instead of the generic placeholder.
-        # Catch ValueError early so the failure yields a clean StreamError rather
-        # than propagating outside the stream error-handling path.
-        has_history = len(session.messages) > 1
-        try:
-            sdk_cwd = _make_sdk_cwd(session_id)
-            os.makedirs(sdk_cwd, exist_ok=True)
-        except (ValueError, OSError) as e:
-            logger.error("[SDK] [%s] Invalid SDK cwd: %s", session_id[:12], e)
-            yield StreamError(
-                errorText="Unable to initialize working directory.",
-                code="sdk_cwd_error",
-            )
-            return
-        system_prompt, _ = await _build_system_prompt(
-            user_id, has_conversation_history=has_history
+        # Use a session-specific temp dir to avoid cleanup race conditions
+        # between concurrent sessions.
+        sdk_cwd = _make_sdk_cwd(session_id)
+        os.makedirs(sdk_cwd, exist_ok=True)
+
+        set_execution_context(
+            user_id,
+            session,
+            long_running_callback=_build_long_running_callback(user_id),
         )
-        system_prompt += _build_sdk_tool_supplement(sdk_cwd)
-
-        yield StreamStart(messageId=message_id, sessionId=session_id)
-
-        set_execution_context(user_id, session)
         try:
             from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
 
@@ -632,6 +714,7 @@ async def stream_chat_completion_sdk(
             options = ClaudeAgentOptions(**sdk_options_kwargs)  # type: ignore[arg-type]
 
             adapter = SDKResponseAdapter(message_id=message_id, session_id=session_id)
+            adapter.set_task_id(task_id)
 
             async with ClaudeSDKClient(options=options) as client:
                 current_message = message or ""
@@ -645,6 +728,7 @@ async def stream_chat_completion_sdk(
                         errorText="Message cannot be empty.",
                         code="empty_prompt",
                     )
+                    yield StreamFinish()
                     return
 
                 query_message = await _build_query_message(
@@ -655,7 +739,8 @@ async def stream_chat_completion_sdk(
                     session_id,
                 )
                 logger.info(
-                    "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, query_len=%d",
+                    "[SDK] [%s] Sending query — resume=%s, "
+                    "total_msgs=%d, query_len=%d",
                     session_id[:12],
                     use_resume,
                     len(session.messages),
@@ -704,7 +789,8 @@ async def stream_chat_completion_sdk(
                             sdk_msg = done.pop().result()
                         except StopAsyncIteration:
                             logger.info(
-                                "[SDK] [%s] Stream ended normally (StopAsyncIteration)",
+                                "[SDK] [%s] Stream ended normally "
+                                "(StopAsyncIteration)",
                                 session_id[:12],
                             )
                             break
@@ -777,25 +863,6 @@ async def stream_chat_completion_sdk(
                                     - len(adapter.resolved_tool_calls),
                                 )
 
-                        # Log ResultMessage details for debugging
-                        if isinstance(sdk_msg, ResultMessage):
-                            logger.info(
-                                "[SDK] [%s] Received: ResultMessage %s "
-                                "(unresolved=%d, current=%d, resolved=%d)",
-                                session_id[:12],
-                                sdk_msg.subtype,
-                                len(adapter.current_tool_calls)
-                                - len(adapter.resolved_tool_calls),
-                                len(adapter.current_tool_calls),
-                                len(adapter.resolved_tool_calls),
-                            )
-                            if sdk_msg.subtype in ("error", "error_during_execution"):
-                                logger.error(
-                                    "[SDK] [%s] SDK execution failed with error: %s",
-                                    session_id[:12],
-                                    sdk_msg.result or "(no error message provided)",
-                                )
-
                         for response in adapter.convert_message(sdk_msg):
                             if isinstance(response, StreamStart):
                                 continue
@@ -820,15 +887,6 @@ async def stream_chat_completion_sdk(
                                     extra,
                                 )
 
-                            # Log errors being sent to frontend
-                            if isinstance(response, StreamError):
-                                logger.error(
-                                    "[SDK] [%s] Sending error to frontend: %s (code=%s)",
-                                    session_id[:12],
-                                    response.errorText,
-                                    response.code,
-                                )
-
                             yield response
 
                             if isinstance(response, StreamTextDelta):
@@ -869,6 +927,18 @@ async def stream_chat_completion_sdk(
                                 if not has_appended_assistant:
                                     session.messages.append(assistant_response)
                                     has_appended_assistant = True
+                                # Save before tool execution starts so the
+                                # pending tool call is visible on refresh /
+                                # other devices. Collision detection happens
+                                # in add_chat_messages_batch (db.py).
+                                try:
+                                    session = await upsert_chat_session(session)
+                                except Exception as save_err:
+                                    logger.warning(
+                                        "[SDK] [%s] Incremental save " "failed: %s",
+                                        session_id[:12],
+                                        save_err,
+                                    )
 
                             elif isinstance(response, StreamToolOutputAvailable):
                                 session.messages.append(
@@ -883,6 +953,17 @@ async def stream_chat_completion_sdk(
                                     )
                                 )
                                 has_tool_results = True
+                                # Save after tool completes so the result is
+                                # visible on refresh / other devices.
+                                # Collision detection happens in add_chat_messages_batch (db.py).
+                                try:
+                                    session = await upsert_chat_session(session)
+                                except Exception as save_err:
+                                    logger.warning(
+                                        "[SDK] [%s] Incremental save " "failed: %s",
+                                        session_id[:12],
+                                        save_err,
+                                    )
 
                             elif isinstance(response, StreamFinish):
                                 stream_completed = True
@@ -892,7 +973,8 @@ async def stream_chat_completion_sdk(
                     # server shutdown).  Log and let the safety-net / finally
                     # blocks handle cleanup.
                     logger.warning(
-                        "[SDK] [%s] Streaming loop cancelled (asyncio.CancelledError)",
+                        "[SDK] [%s] Streaming loop cancelled "
+                        "(asyncio.CancelledError)",
                         session_id[:12],
                     )
                     raise
@@ -934,29 +1016,25 @@ async def stream_chat_completion_sdk(
                             )
                         yield response
 
-                # If the stream ended without a ResultMessage, the SDK
-                # CLI exited unexpectedly or the user stopped execution.
-                # Close any open text/step so chunks are well-formed, and
-                # append a cancellation message so users see feedback.
-                # StreamFinish is published by mark_session_completed in the processor.
+                # If the stream ended without a ResultMessage (no
+                # StreamFinish), the SDK CLI exited unexpectedly.  Close
+                # the open step and emit StreamFinish so the frontend
+                # transitions to the "ready" state.
                 if not stream_completed:
-                    logger.info(
-                        "[SDK] [%s] Stream ended without ResultMessage (stopped by user)",
+                    logger.warning(
+                        "[SDK] [%s] Stream ended without ResultMessage "
+                        "(StopAsyncIteration) — emitting StreamFinish",
                         session_id[:12],
                     )
+                    if adapter.step_open:
+                        yield StreamFinishStep()
+                        adapter.step_open = False
                     closing_responses: list[StreamBaseResponse] = []
                     adapter._end_text_if_open(closing_responses)
                     for r in closing_responses:
                         yield r
-
-                    # Add "Stopped by user" message so it persists after refresh
-                    # Use COPILOT_SYSTEM_PREFIX so frontend renders it as system message, not assistant
-                    session.messages.append(
-                        ChatMessage(
-                            role="assistant",
-                            content=f"{COPILOT_SYSTEM_PREFIX} Execution stopped by user",
-                        )
-                    )
+                    yield StreamFinish()
+                    stream_completed = True
 
                 if (
                     assistant_response.content or assistant_response.tool_calls
@@ -976,7 +1054,7 @@ async def stream_chat_completion_sdk(
                 elif captured_transcript.path:
                     raw_transcript = read_transcript_file(captured_transcript.path)
                     logger.debug(
-                        "[SDK] Transcript source: stop hook (%s), read result: %s",
+                        "[SDK] Transcript source: stop hook (%s), " "read result: %s",
                         captured_transcript.path,
                         f"{len(raw_transcript)}B" if raw_transcript else "None",
                     )
@@ -1011,76 +1089,34 @@ async def stream_chat_completion_sdk(
                 "to use the OpenAI-compatible fallback."
             )
 
+        session = cast(ChatSession, await asyncio.shield(upsert_chat_session(session)))
         logger.info(
-            "[SDK] [%s] Stream completed successfully with %d messages",
+            "[SDK] [%s] Session saved with %d messages",
             session_id[:12],
             len(session.messages),
         )
-    except BaseException as e:
-        # Catch BaseException to handle both Exception and CancelledError
-        # (CancelledError inherits from BaseException in Python 3.8+)
-        if isinstance(e, asyncio.CancelledError):
-            logger.warning("[SDK] [%s] Session cancelled", session_id[:12])
-            error_msg = "Operation cancelled"
-        else:
-            error_msg = str(e) or type(e).__name__
-            # SDK cleanup RuntimeError is expected during cancellation, log as warning
-            if isinstance(e, RuntimeError) and "cancel scope" in str(e):
-                logger.warning(
-                    "[SDK] [%s] SDK cleanup error: %s", session_id[:12], error_msg
-                )
-            else:
-                logger.error(
-                    f"[SDK] [%s] Error: {error_msg}", session_id[:12], exc_info=True
-                )
-
-        # Append error marker to session (non-invasive text parsing approach)
-        # The finally block will persist the session with this error marker
-        if session:
-            session.messages.append(
-                ChatMessage(
-                    role="assistant", content=f"{COPILOT_ERROR_PREFIX} {error_msg}"
-                )
-            )
-            logger.debug(
-                "[SDK] [%s] Appended error marker, will be persisted in finally",
-                session_id[:12],
-            )
-
-        # Yield StreamError for immediate feedback (only for non-cancellation errors)
-        # Skip for CancelledError and RuntimeError cleanup issues (both are cancellations)
-        is_cancellation = isinstance(e, asyncio.CancelledError) or (
-            isinstance(e, RuntimeError) and "cancel scope" in str(e)
-        )
-        if not is_cancellation:
-            yield StreamError(
-                errorText=error_msg,
-                code="sdk_error",
-            )
+        if not stream_completed:
+            yield StreamFinish()
 
+    except asyncio.CancelledError:
+        # Client disconnect / server shutdown — log but re-raise so
+        # the framework can clean up.  The finally block still runs
+        # for transcript upload.
+        logger.warning("[SDK] [%s] Session cancelled (CancelledError)", session_id[:12])
         raise
-    finally:
-        # --- Persist session messages ---
-        # This MUST run in finally to persist messages even when the generator
-        # is stopped early (e.g., user clicks stop, processor breaks stream loop).
-        # Without this, messages disappear after refresh because they were never
-        # saved to the database.
-        if session is not None:
+    except Exception as e:
+        logger.error(f"[SDK] Error: {e}", exc_info=True)
+        if session:
             try:
                 await asyncio.shield(upsert_chat_session(session))
-                logger.info(
-                    "[SDK] [%s] Session persisted in finally with %d messages",
-                    session_id[:12],
-                    len(session.messages),
-                )
-            except Exception as persist_err:
-                logger.error(
-                    "[SDK] [%s] Failed to persist session in finally: %s",
-                    session_id[:12],
-                    persist_err,
-                    exc_info=True,
-                )
-
+            except Exception as save_err:
+                logger.error(f"[SDK] Failed to save session on error: {save_err}")
+        yield StreamError(
+            errorText="An error occurred. Please try again.",
+            code="sdk_error",
+        )
+        yield StreamFinish()
+    finally:
         # --- Upload transcript for next-turn --resume ---
         # This MUST run in finally so the transcript is uploaded even when
         # the streaming loop raises an exception.  The CLI uses

autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py

@@ -2,6 +2,11 @@
 
 This module provides the adapter layer that converts existing BaseTool implementations
 into in-process MCP tools that can be used with the Claude Agent SDK.
+
+Long-running tools (``is_long_running=True``) are delegated to the non-SDK
+background infrastructure (stream_registry, Redis persistence, SSE reconnection)
+via a callback provided by the service layer.  This avoids wasteful SDK polling
+and makes results survive page refreshes.
 """
 
 import asyncio
@@ -10,6 +15,7 @@ import json
 import logging
 import os
 import uuid
+from collections.abc import Awaitable, Callable
 from contextvars import ContextVar
 from typing import Any
 
@@ -37,8 +43,7 @@ _current_session: ContextVar[ChatSession | None] = ContextVar(
 # Keyed by tool_name → full output string. Consumed (popped) by the
 # response adapter when it builds StreamToolOutputAvailable.
 _pending_tool_outputs: ContextVar[dict[str, list[str]]] = ContextVar(
-    "pending_tool_outputs",
-    default=None,  # type: ignore[arg-type]
+    "pending_tool_outputs", default=None  # type: ignore[arg-type]
 )
 # Event signaled whenever stash_pending_tool_output() adds a new entry.
 # Used by the streaming loop to wait for PostToolUse hooks to complete
@@ -49,10 +54,22 @@ _stash_event: ContextVar[asyncio.Event | None] = ContextVar(
     "_stash_event", default=None
 )
 
+# Callback type for delegating long-running tools to the non-SDK infrastructure.
+# Args: (tool_name, arguments, session) → MCP-formatted response dict.
+LongRunningCallback = Callable[
+    [str, dict[str, Any], ChatSession], Awaitable[dict[str, Any]]
+]
+
+# ContextVar so the service layer can inject the callback per-request.
+_long_running_callback: ContextVar[LongRunningCallback | None] = ContextVar(
+    "long_running_callback", default=None
+)
+
 
 def set_execution_context(
     user_id: str | None,
     session: ChatSession,
+    long_running_callback: LongRunningCallback | None = None,
 ) -> None:
     """Set the execution context for tool calls.
 
@@ -62,11 +79,14 @@ def set_execution_context(
     Args:
         user_id: Current user's ID.
         session: Current chat session.
+        long_running_callback: Optional callback to delegate long-running tools
+            to the non-SDK background infrastructure (stream_registry + Redis).
     """
     _current_user_id.set(user_id)
     _current_session.set(session)
     _pending_tool_outputs.set({})
     _stash_event.set(asyncio.Event())
+    _long_running_callback.set(long_running_callback)
 
 
 def get_execution_context() -> tuple[str | None, ChatSession | None]:
@@ -256,6 +276,11 @@ def create_tool_handler(base_tool: BaseTool):
 
     This wraps the existing BaseTool._execute method to be compatible
     with the Claude Agent SDK MCP tool format.
+
+    Long-running tools (``is_long_running=True``) are delegated to the
+    non-SDK background infrastructure via a callback set in the execution
+    context.  The callback persists the operation in Redis (stream_registry)
+    so results survive page refreshes and pod restarts.
     """
 
     async def tool_handler(args: dict[str, Any]) -> dict[str, Any]:
@@ -265,6 +290,25 @@ def create_tool_handler(base_tool: BaseTool):
         if session is None:
             return _mcp_error("No session context available")
 
+        # --- Long-running: delegate to non-SDK background infrastructure ---
+        if base_tool.is_long_running:
+            callback = _long_running_callback.get(None)
+            if callback:
+                try:
+                    return await callback(base_tool.name, args, session)
+                except Exception as e:
+                    logger.error(
+                        f"Long-running callback failed for {base_tool.name}: {e}",
+                        exc_info=True,
+                    )
+                    return _mcp_error(f"Failed to start {base_tool.name}: {e}")
+            # No callback — fall through to synchronous execution
+            logger.warning(
+                f"[SDK] No long-running callback for {base_tool.name}, "
+                f"executing synchronously (may block)"
+            )
+
+        # --- Normal (fast) tool: execute synchronously ---
         try:
             return await _execute_tool_sync(base_tool, user_id, session, args)
         except Exception as e:

autogpt_platform/backend/backend/copilot/service_test.py

@@ -6,7 +6,12 @@ import pytest
 
 from . import service as chat_service
 from .model import create_chat_session, get_chat_session, upsert_chat_session
-from .response_model import StreamError, StreamTextDelta, StreamToolOutputAvailable
+from .response_model import (
+    StreamError,
+    StreamFinish,
+    StreamTextDelta,
+    StreamToolOutputAvailable,
+)
 from .sdk import service as sdk_service
 from .sdk.transcript import download_transcript
 
@@ -25,6 +30,7 @@ async def test_stream_chat_completion(setup_test_user, test_user_id):
     session = await create_chat_session(test_user_id)
 
     has_errors = False
+    has_ended = False
     assistant_message = ""
     async for chunk in chat_service.stream_chat_completion(
         session.session_id, "Hello, how are you?", user_id=session.user_id
@@ -34,9 +40,10 @@ async def test_stream_chat_completion(setup_test_user, test_user_id):
             has_errors = True
         if isinstance(chunk, StreamTextDelta):
             assistant_message += chunk.delta
+        if isinstance(chunk, StreamFinish):
+            has_ended = True
 
-    # StreamFinish is published by mark_session_completed (processor layer),
-    # not by the service. The generator completing means the stream ended.
+    assert has_ended, "Chat completion did not end"
     assert not has_errors, "Error occurred while streaming chat completion"
     assert assistant_message, "Assistant message is empty"
 
@@ -54,6 +61,7 @@ async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user
     session = await upsert_chat_session(session)
 
     has_errors = False
+    has_ended = False
     had_tool_calls = False
     async for chunk in chat_service.stream_chat_completion(
         session.session_id,
@@ -63,9 +71,13 @@ async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user
         logger.info(chunk)
         if isinstance(chunk, StreamError):
             has_errors = True
+
+        if isinstance(chunk, StreamFinish):
+            has_ended = True
         if isinstance(chunk, StreamToolOutputAvailable):
             had_tool_calls = True
 
+    assert has_ended, "Chat completion did not end"
     assert not has_errors, "Error occurred while streaming chat completion"
     assert had_tool_calls, "Tool calls did not occur"
     session = await get_chat_session(session.session_id)
@@ -102,6 +114,7 @@ async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):
     )
     turn1_text = ""
     turn1_errors: list[str] = []
+    turn1_ended = False
 
     async for chunk in sdk_service.stream_chat_completion_sdk(
         session.session_id,
@@ -112,7 +125,10 @@ async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):
             turn1_text += chunk.delta
         elif isinstance(chunk, StreamError):
             turn1_errors.append(chunk.errorText)
+        elif isinstance(chunk, StreamFinish):
+            turn1_ended = True
 
+    assert turn1_ended, "Turn 1 did not finish"
     assert not turn1_errors, f"Turn 1 errors: {turn1_errors}"
     assert turn1_text, "Turn 1 produced no text"
 
@@ -143,6 +159,7 @@ async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):
     turn2_msg = "What was the special keyword I asked you to remember?"
     turn2_text = ""
     turn2_errors: list[str] = []
+    turn2_ended = False
 
     async for chunk in sdk_service.stream_chat_completion_sdk(
         session.session_id,
@@ -154,7 +171,10 @@ async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):
             turn2_text += chunk.delta
         elif isinstance(chunk, StreamError):
             turn2_errors.append(chunk.errorText)
+        elif isinstance(chunk, StreamFinish):
+            turn2_ended = True
 
+    assert turn2_ended, "Turn 2 did not finish"
     assert not turn2_errors, f"Turn 2 errors: {turn2_errors}"
     assert turn2_text, "Turn 2 produced no text"
     assert keyword in turn2_text, (

autogpt_platform/backend/backend/copilot/test_copilot_e2e.py

@@ -1,401 +0,0 @@
-"""End-to-end tests for Copilot streaming with dummy implementations.
-
-These tests verify the complete copilot flow using dummy implementations
-for agent generator and SDK service, allowing automated testing without
-external LLM calls.
-
-Enable test mode with COPILOT_TEST_MODE=true environment variable.
-
-Note: StreamFinish is NOT emitted by the dummy service — it is published
-by mark_session_completed in the processor layer.  These tests only cover
-the service-level streaming output (StreamStart + StreamTextDelta).
-"""
-
-import asyncio
-import os
-from uuid import uuid4
-
-import pytest
-
-from backend.copilot.model import ChatMessage, ChatSession, upsert_chat_session
-from backend.copilot.response_model import (
-    StreamError,
-    StreamHeartbeat,
-    StreamStart,
-    StreamTextDelta,
-)
-from backend.copilot.sdk.dummy import stream_chat_completion_dummy
-
-
-@pytest.fixture(autouse=True)
-def enable_test_mode():
-    """Enable test mode for all tests in this module."""
-    os.environ["COPILOT_TEST_MODE"] = "true"
-    yield
-    os.environ.pop("COPILOT_TEST_MODE", None)
-
-
-@pytest.mark.asyncio
-async def test_dummy_streaming_basic_flow():
-    """Test that dummy streaming produces correct event sequence."""
-    events = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-session-basic",
-        message="Hello",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        events.append(event)
-
-    # Verify we got events
-    assert len(events) > 0, "Should receive events"
-
-    # Verify StreamStart
-    start_events = [e for e in events if isinstance(e, StreamStart)]
-    assert len(start_events) == 1
-    assert start_events[0].messageId
-    assert start_events[0].sessionId
-
-    # Verify StreamTextDelta events
-    text_events = [e for e in events if isinstance(e, StreamTextDelta)]
-    assert len(text_events) > 0
-    full_text = "".join(e.delta for e in text_events)
-    assert len(full_text) > 0
-
-    # Verify order: start before text
-    start_idx = events.index(start_events[0])
-    first_text_idx = events.index(text_events[0]) if text_events else -1
-    if first_text_idx >= 0:
-        assert start_idx < first_text_idx
-
-    print(f"✅ Basic flow: {len(events)} events, {len(text_events)} text deltas")
-
-
-@pytest.mark.asyncio
-async def test_streaming_no_timeout():
-    """Test that streaming completes within reasonable time without timeout."""
-    import time
-
-    start_time = time.monotonic()
-    event_count = 0
-
-    async for _event in stream_chat_completion_dummy(
-        session_id="test-session-timeout",
-        message="count to 10",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        event_count += 1
-
-    elapsed = time.monotonic() - start_time
-
-    # Should complete in < 5 seconds (dummy has 0.1s delays between words)
-    assert elapsed < 5.0, f"Streaming took {elapsed:.1f}s, expected < 5s"
-    assert event_count > 0, "Should receive events"
-
-    print(f"✅ No timeout: completed in {elapsed:.2f}s with {event_count} events")
-
-
-@pytest.mark.asyncio
-async def test_streaming_event_types():
-    """Test that all expected event types are present."""
-    event_types = set()
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-session-types",
-        message="test",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        event_types.add(type(event).__name__)
-
-    # Required event types (StreamFinish is published by processor, not service)
-    assert "StreamStart" in event_types, "Missing StreamStart"
-    assert "StreamTextDelta" in event_types, "Missing StreamTextDelta"
-
-    print(f"✅ Event types: {sorted(event_types)}")
-
-
-@pytest.mark.asyncio
-async def test_streaming_text_content():
-    """Test that streamed text is coherent and complete."""
-    text_events = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-session-content",
-        message="count to 3",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        if isinstance(event, StreamTextDelta):
-            text_events.append(event)
-
-    # Verify text deltas
-    assert len(text_events) > 0, "Should have text deltas"
-
-    # Reconstruct full text
-    full_text = "".join(e.delta for e in text_events)
-    assert len(full_text) > 0, "Text should not be empty"
-    assert (
-        "1" in full_text or "counted" in full_text.lower()
-    ), "Text should contain count"
-
-    # Verify all deltas have IDs
-    for text_event in text_events:
-        assert text_event.id, "Text delta must have ID"
-        assert text_event.delta, "Text delta must have content"
-
-    print(f"✅ Text content: '{full_text}' ({len(text_events)} deltas)")
-
-
-@pytest.mark.asyncio
-async def test_streaming_heartbeat_timing():
-    """Test that heartbeats are sent at correct interval during long operations."""
-    # This test would need a dummy that takes longer
-    # For now, just verify heartbeat structure if we receive one
-    heartbeats = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-session-heartbeat",
-        message="test",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        if isinstance(event, StreamHeartbeat):
-            heartbeats.append(event)
-
-    # Dummy is fast, so we might not get heartbeats
-    # But if we do, verify they're valid
-    if heartbeats:
-        print(f"✅ Heartbeat structure verified ({len(heartbeats)} received)")
-    else:
-        print("✅ No heartbeats (dummy executes quickly)")
-
-
-@pytest.mark.asyncio
-async def test_error_handling():
-    """Test that errors are properly formatted and sent."""
-    # This would require a dummy that can trigger errors
-    # For now, just verify error event structure
-
-    error = StreamError(errorText="Test error", code="test_error")
-    assert error.errorText == "Test error"
-    assert error.code == "test_error"
-    assert str(error.type.value) in ["error", "error"]
-
-    print("✅ Error structure verified")
-
-
-@pytest.mark.asyncio
-async def test_concurrent_sessions():
-    """Test that multiple sessions can stream concurrently."""
-
-    async def stream_session(session_id: str) -> int:
-        count = 0
-        async for _event in stream_chat_completion_dummy(
-            session_id=session_id,
-            message="test",
-            is_user_message=True,
-            user_id="test-user",
-        ):
-            count += 1
-        return count
-
-    # Run 3 concurrent sessions
-    results = await asyncio.gather(
-        stream_session("session-1"),
-        stream_session("session-2"),
-        stream_session("session-3"),
-    )
-
-    # All should complete successfully
-    assert all(count > 0 for count in results), "All sessions should produce events"
-    print(f"✅ Concurrent sessions: {results} events each")
-
-
-@pytest.mark.asyncio
-@pytest.mark.xfail(
-    reason="Event loop isolation issue with DB operations in tests - needs fixture refactoring"
-)
-async def test_session_state_persistence():
-    """Test that session state is maintained across multiple messages."""
-    from datetime import datetime, timezone
-
-    session_id = f"test-session-{uuid4()}"
-    user_id = "test-user"
-
-    # Create session with first message
-    session = ChatSession(
-        session_id=session_id,
-        user_id=user_id,
-        messages=[
-            ChatMessage(role="user", content="Hello"),
-            ChatMessage(role="assistant", content="Hi there!"),
-        ],
-        usage=[],
-        started_at=datetime.now(timezone.utc),
-        updated_at=datetime.now(timezone.utc),
-    )
-    await upsert_chat_session(session)
-
-    # Stream second message
-    events = []
-    async for event in stream_chat_completion_dummy(
-        session_id=session_id,
-        message="How are you?",
-        is_user_message=True,
-        user_id=user_id,
-        session=session,  # Pass existing session
-    ):
-        events.append(event)
-
-    # Verify events were produced
-    assert len(events) > 0, "Should produce events for second message"
-
-    print(f"✅ Session persistence: {len(events)} events for second message")
-
-
-@pytest.mark.asyncio
-async def test_message_deduplication():
-    """Test that duplicate messages are filtered out."""
-
-    # Simulate receiving duplicate events (e.g., from reconnection)
-    events = []
-
-    # First stream
-    async for event in stream_chat_completion_dummy(
-        session_id="test-dedup-1",
-        message="Hello",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        events.append(event)
-
-    # Count unique message IDs in StreamStart events
-    start_events = [e for e in events if isinstance(e, StreamStart)]
-    message_ids = [e.messageId for e in start_events]
-
-    # Verify all IDs are present
-    assert len(message_ids) == len(set(message_ids)), "Message IDs should be unique"
-
-    print(f"✅ Deduplication: {len(events)} events, all unique")
-
-
-@pytest.mark.asyncio
-async def test_event_ordering():
-    """Test that events arrive in correct order."""
-    events = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-ordering",
-        message="Test",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        events.append(event)
-
-    # Find event indices
-    start_idx = next(
-        (i for i, e in enumerate(events) if isinstance(e, StreamStart)), None
-    )
-    text_indices = [i for i, e in enumerate(events) if isinstance(e, StreamTextDelta)]
-
-    # Verify ordering
-    assert start_idx is not None, "Should have StreamStart"
-    assert start_idx == 0, "StreamStart should be first"
-
-    if text_indices:
-        assert all(
-            start_idx < i for i in text_indices
-        ), "Text deltas should be after start"
-
-    print(f"✅ Event ordering: start({start_idx}) < text deltas")
-
-
-@pytest.mark.asyncio
-async def test_stream_completeness():
-    """Test that stream includes all required event types."""
-    events = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-completeness",
-        message="Complete stream test",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        events.append(event)
-
-    # Check for required events (StreamFinish is published by processor)
-    has_start = any(isinstance(e, StreamStart) for e in events)
-    has_text = any(isinstance(e, StreamTextDelta) for e in events)
-
-    assert has_start, "Stream must include StreamStart"
-    assert has_text, "Stream must include text deltas"
-
-    # Verify exactly one start
-    start_count = sum(1 for e in events if isinstance(e, StreamStart))
-    assert start_count == 1, f"Should have exactly 1 StreamStart, got {start_count}"
-
-    print(
-        f"✅ Completeness: 1 start, {sum(1 for e in events if isinstance(e, StreamTextDelta))} text deltas"
-    )
-
-
-@pytest.mark.asyncio
-async def test_text_delta_consistency():
-    """Test that text deltas have consistent IDs and build coherent text."""
-    text_events = []
-
-    async for event in stream_chat_completion_dummy(
-        session_id="test-consistency",
-        message="Test consistency",
-        is_user_message=True,
-        user_id="test-user",
-    ):
-        if isinstance(event, StreamTextDelta):
-            text_events.append(event)
-
-    # Verify all text deltas have IDs
-    assert all(e.id for e in text_events), "All text deltas must have IDs"
-
-    # Verify all deltas have the same ID (same text block)
-    if text_events:
-        first_id = text_events[0].id
-        assert all(
-            e.id == first_id for e in text_events
-        ), "All text deltas should share the same block ID"
-
-    # Verify deltas build coherent text
-    full_text = "".join(e.delta for e in text_events)
-    assert len(full_text) > 0, "Deltas should build non-empty text"
-    assert (
-        full_text == full_text.strip()
-    ), "Text should not have leading/trailing whitespace artifacts"
-
-    print(
-        f"✅ Consistency: {len(text_events)} deltas with ID '{text_events[0].id if text_events else 'N/A'}', text: '{full_text}'"
-    )
-
-
-if __name__ == "__main__":
-    # Run tests directly
-
-    print("Running Copilot E2E tests with dummy implementations...")
-    print("=" * 60)
-
-    asyncio.run(test_dummy_streaming_basic_flow())
-    asyncio.run(test_streaming_no_timeout())
-    asyncio.run(test_streaming_event_types())
-    asyncio.run(test_streaming_text_content())
-    asyncio.run(test_streaming_heartbeat_timing())
-    asyncio.run(test_error_handling())
-    asyncio.run(test_concurrent_sessions())
-    asyncio.run(test_session_state_persistence())
-    asyncio.run(test_message_deduplication())
-    asyncio.run(test_event_ordering())
-    asyncio.run(test_stream_completeness())
-    asyncio.run(test_text_delta_consistency())
-
-    print("=" * 60)
-    print("✅ All E2E tests passed!")

autogpt_platform/backend/backend/copilot/tools/__init__.py

@@ -10,7 +10,7 @@ from .add_understanding import AddUnderstandingTool
 from .agent_output import AgentOutputTool
 from .base import BaseTool
 from .bash_exec import BashExecTool
-from .browse_web import BrowseWebTool
+from .check_operation_status import CheckOperationStatusTool
 from .create_agent import CreateAgentTool
 from .customize_agent import CustomizeAgentTool
 from .edit_agent import EditAgentTool
@@ -47,12 +47,11 @@ TOOL_REGISTRY: dict[str, BaseTool] = {
     "run_agent": RunAgentTool(),
     "run_block": RunBlockTool(),
     "view_agent_output": AgentOutputTool(),
+    "check_operation_status": CheckOperationStatusTool(),
     "search_docs": SearchDocsTool(),
     "get_doc_page": GetDocPageTool(),
     # Web fetch for safe URL retrieval
     "web_fetch": WebFetchTool(),
-    # Browser-based browsing for JS-rendered pages (Stagehand + Browserbase)
-    "browse_web": BrowseWebTool(),
     # Sandboxed code execution (bubblewrap)
     "bash_exec": BashExecTool(),
     # Persistent workspace tools (cloud storage, survives across sessions)

autogpt_platform/backend/backend/copilot/tools/_test_data.py

@@ -3,7 +3,6 @@ from datetime import UTC, datetime
 from os import getenv
 
 import pytest
-import pytest_asyncio
 from prisma.types import ProfileCreateInput
 from pydantic import SecretStr
 
@@ -32,16 +31,14 @@ def make_session(user_id: str):
     )
 
 
-@pytest_asyncio.fixture(scope="session", loop_scope="session")
-async def setup_test_data(server):
+@pytest.fixture(scope="session")
+async def setup_test_data():
     """
     Set up test data for run_agent tests:
     1. Create a test user
     2. Create a test graph (agent input -> agent output)
     3. Create a store listing and store listing version
     4. Approve the store listing version
-
-    Depends on ``server`` to ensure Prisma is connected.
     """
     # 1. Create a test user
     user_data = {
@@ -153,16 +150,14 @@ async def setup_test_data(server):
     }
 
 
-@pytest_asyncio.fixture(scope="session", loop_scope="session")
-async def setup_llm_test_data(server):
+@pytest.fixture(scope="session")
+async def setup_llm_test_data():
     """
     Set up test data for LLM agent tests:
     1. Create a test user
     2. Create test OpenAI credentials for the user
     3. Create a test graph with input -> LLM block -> output
     4. Create and approve a store listing
-
-    Depends on ``server`` to ensure Prisma is connected.
     """
     key = getenv("OPENAI_API_KEY")
     if not key:
@@ -320,15 +315,13 @@ async def setup_llm_test_data(server):
     }
 
 
-@pytest_asyncio.fixture(scope="session", loop_scope="session")
-async def setup_firecrawl_test_data(server):
+@pytest.fixture(scope="session")
+async def setup_firecrawl_test_data():
     """
     Set up test data for Firecrawl agent tests (missing credentials scenario):
     1. Create a test user (WITHOUT Firecrawl credentials)
     2. Create a test graph with input -> Firecrawl block -> output
     3. Create and approve a store listing
-
-    Depends on ``server`` to ensure Prisma is connected.
     """
     # 1. Create a test user
     user_data = {

autogpt_platform/backend/backend/copilot/tools/agent_generator/__init__.py

@@ -19,7 +19,6 @@ from .core import (
     get_all_relevant_agents_for_generation,
     get_library_agent_by_graph_id,
     get_library_agent_by_id,
-    get_library_agents_by_ids,
     get_library_agents_for_generation,
     graph_to_json,
     json_to_graph,
@@ -50,7 +49,6 @@ __all__ = [
     "get_all_relevant_agents_for_generation",
     "get_library_agent_by_graph_id",
     "get_library_agent_by_id",
-    "get_library_agents_by_ids",
     "get_library_agents_for_generation",
     "get_user_message_for_error",
     "graph_to_json",

autogpt_platform/backend/backend/copilot/tools/agent_generator/core.py

@@ -3,7 +3,6 @@
 import logging
 import re
 import uuid
-from collections.abc import Sequence
 from typing import Any, NotRequired, TypedDict
 
 from backend.data.db_accessors import graph_db, library_db, store_db
@@ -79,7 +78,7 @@ AgentSummary = LibraryAgentSummary | MarketplaceAgentSummary | dict[str, Any]
 
 
 def _to_dict_list(
-    agents: Sequence[AgentSummary] | Sequence[dict[str, Any]] | None,
+    agents: list[AgentSummary] | list[dict[str, Any]] | None,
 ) -> list[dict[str, Any]] | None:
     """Convert typed agent summaries to plain dicts for external service calls."""
     if agents is None:
@@ -191,36 +190,6 @@ async def get_library_agent_by_id(
 get_library_agent_by_graph_id = get_library_agent_by_id
 
 
-async def get_library_agents_by_ids(
-    user_id: str,
-    agent_ids: list[str],
-) -> list[LibraryAgentSummary]:
-    """Fetch multiple library agents by their IDs.
-
-    Args:
-        user_id: The user ID
-        agent_ids: List of agent IDs (can be graph_ids or library agent IDs)
-
-    Returns:
-        List of LibraryAgentSummary for found agents (silently skips not found)
-    """
-    agents: list[LibraryAgentSummary] = []
-    for agent_id in agent_ids:
-        try:
-            agent = await get_library_agent_by_id(user_id, agent_id)
-            if agent:
-                agents.append(agent)
-                logger.debug(f"Fetched library agent by ID: {agent['name']}")
-            else:
-                logger.warning(f"Library agent not found for ID: {agent_id}")
-        except Exception as e:
-            logger.warning(f"Failed to fetch library agent {agent_id}: {e}")
-            continue
-
-    logger.info(f"Fetched {len(agents)}/{len(agent_ids)} library agents by ID")
-    return agents
-
-
 async def get_library_agents_for_generation(
     user_id: str,
     search_query: str | None = None,
@@ -245,17 +214,10 @@ async def get_library_agents_for_generation(
     Returns:
         List of LibraryAgentSummary with schemas and recent executions for sub-agent composition
     """
-    search_term = search_query.strip() if search_query else None
-    if search_term and len(search_term) > 100:
-        raise ValueError(
-            f"Search query is too long ({len(search_term)} chars, max 100). "
-            f"Please use a shorter, more specific search term."
-        )
-
     try:
         response = await library_db().list_library_agents(
             user_id=user_id,
-            search_term=search_term,
+            search_term=search_query,
             page=1,
             page_size=max_results,
             include_executions=True,
@@ -309,16 +271,9 @@ async def search_marketplace_agents_for_generation(
     Returns:
         List of LibraryAgentSummary with full input/output schemas
     """
-    search_term = search_query.strip()
-    if len(search_term) > 100:
-        raise ValueError(
-            f"Search query is too long ({len(search_term)} chars, max 100). "
-            f"Please use a shorter, more specific search term."
-        )
-
     try:
         response = await store_db().get_store_agents(
-            search_query=search_term,
+            search_query=search_query,
             page=1,
             page_size=max_results,
         )
@@ -469,7 +424,7 @@ def extract_search_terms_from_steps(
 async def enrich_library_agents_from_steps(
     user_id: str,
     decomposition_result: DecompositionResult | dict[str, Any],
-    existing_agents: Sequence[AgentSummary] | Sequence[dict[str, Any]],
+    existing_agents: list[AgentSummary] | list[dict[str, Any]],
     exclude_graph_id: str | None = None,
     include_marketplace: bool = True,
     max_additional_results: int = 10,
@@ -493,7 +448,7 @@ async def enrich_library_agents_from_steps(
     search_terms = extract_search_terms_from_steps(decomposition_result)
 
     if not search_terms:
-        return list(existing_agents)
+        return existing_agents
 
     existing_ids: set[str] = set()
     existing_names: set[str] = set()
@@ -556,7 +511,7 @@ async def enrich_library_agents_from_steps(
 async def decompose_goal(
     description: str,
     context: str = "",
-    library_agents: Sequence[AgentSummary] | None = None,
+    library_agents: list[AgentSummary] | None = None,
 ) -> DecompositionResult | None:
     """Break down a goal into steps or return clarifying questions.
 
@@ -584,16 +539,22 @@ async def decompose_goal(
 
 async def generate_agent(
     instructions: DecompositionResult | dict[str, Any],
-    library_agents: Sequence[AgentSummary] | Sequence[dict[str, Any]] | None = None,
+    library_agents: list[AgentSummary] | list[dict[str, Any]] | None = None,
+    operation_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any] | None:
     """Generate agent JSON from instructions.
 
     Args:
         instructions: Structured instructions from decompose_goal
         library_agents: User's library agents available for sub-agent composition
+        operation_id: Operation ID for async processing (enables Redis Streams
+            completion notification)
+        task_id: Task ID for async processing (enables Redis Streams persistence
+            and SSE delivery)
 
     Returns:
-        Agent JSON dict, error dict {"type": "error", ...}, or None on error
+        Agent JSON dict, {"status": "accepted"} for async, error dict {"type": "error", ...}, or None on error
 
     Raises:
         AgentGeneratorNotConfiguredError: If the external service is not configured.
@@ -601,9 +562,13 @@ async def generate_agent(
     _check_service_configured()
     logger.info("Calling external Agent Generator service for generate_agent")
     result = await generate_agent_external(
-        dict(instructions), _to_dict_list(library_agents)
+        dict(instructions), _to_dict_list(library_agents), operation_id, task_id
     )
 
+    # Don't modify async response
+    if result and result.get("status") == "accepted":
+        return result
+
     if result:
         if isinstance(result, dict) and result.get("type") == "error":
             return result
@@ -793,7 +758,9 @@ async def get_agent_as_json(
 async def generate_agent_patch(
     update_request: str,
     current_agent: dict[str, Any],
-    library_agents: Sequence[AgentSummary] | None = None,
+    library_agents: list[AgentSummary] | None = None,
+    operation_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any] | None:
     """Update an existing agent using natural language.
 
@@ -806,10 +773,12 @@ async def generate_agent_patch(
         update_request: Natural language description of changes
         current_agent: Current agent JSON
         library_agents: User's library agents available for sub-agent composition
+        operation_id: Operation ID for async processing (enables Redis Streams callback)
+        task_id: Task ID for async processing (enables Redis Streams callback)
 
     Returns:
         Updated agent JSON, clarifying questions dict {"type": "clarifying_questions", ...},
-        error dict {"type": "error", ...}, or None on error
+        {"status": "accepted"} for async, error dict {"type": "error", ...}, or None on error
 
     Raises:
         AgentGeneratorNotConfiguredError: If the external service is not configured.
@@ -820,6 +789,8 @@ async def generate_agent_patch(
         update_request,
         current_agent,
         _to_dict_list(library_agents),
+        operation_id,
+        task_id,
     )
 
 

autogpt_platform/backend/backend/copilot/tools/agent_generator/dummy.py

@@ -102,15 +102,10 @@ async def generate_agent_dummy(
     instructions: dict[str, Any],
     library_agents: list[dict[str, Any]] | None = None,
     operation_id: str | None = None,
-    session_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any]:
-    """Return dummy agent synchronously (blocks for 30s, returns agent JSON).
-
-    Note: operation_id and session_id parameters are ignored - we always use synchronous mode.
-    """
-    logger.info(
-        "Using dummy agent generator (sync mode): returning agent JSON after 30s"
-    )
+    """Return dummy agent JSON after a simulated delay."""
+    logger.info("Using dummy agent generator for generate_agent (30s delay)")
     await asyncio.sleep(30)
     return _generate_dummy_agent_json()
 
@@ -120,16 +115,10 @@ async def generate_agent_patch_dummy(
     current_agent: dict[str, Any],
     library_agents: list[dict[str, Any]] | None = None,
     operation_id: str | None = None,
-    session_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any]:
-    """Return dummy patched agent synchronously (blocks for 30s, returns patched agent JSON).
-
-    Note: operation_id and session_id parameters are ignored - we always use synchronous mode.
-    """
-    logger.info(
-        "Using dummy agent generator patch (sync mode): returning patched agent after 30s"
-    )
-    await asyncio.sleep(30)
+    """Return dummy patched agent (returns the current agent with updated description)."""
+    logger.info("Using dummy agent generator for generate_agent_patch")
     patched = current_agent.copy()
     patched["description"] = (
         f"{current_agent.get('description', '')} (updated: {update_request})"

autogpt_platform/backend/backend/copilot/tools/agent_generator/service.py

@@ -242,18 +242,24 @@ async def decompose_goal_external(
 async def generate_agent_external(
     instructions: dict[str, Any],
     library_agents: list[dict[str, Any]] | None = None,
+    operation_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any] | None:
     """Call the external service to generate an agent from instructions.
 
     Args:
         instructions: Structured instructions from decompose_goal
         library_agents: User's library agents available for sub-agent composition
+        operation_id: Operation ID for async processing (enables Redis Streams callback)
+        task_id: Task ID for async processing (enables Redis Streams callback)
 
     Returns:
-        Agent JSON dict or error dict {"type": "error", ...} on error
+        Agent JSON dict, {"status": "accepted"} for async, or error dict {"type": "error", ...} on error
     """
     if _is_dummy_mode():
-        return await generate_agent_dummy(instructions, library_agents)
+        return await generate_agent_dummy(
+            instructions, library_agents, operation_id, task_id
+        )
 
     client = _get_client()
 
@@ -261,9 +267,25 @@ async def generate_agent_external(
     payload: dict[str, Any] = {"instructions": instructions}
     if library_agents:
         payload["library_agents"] = library_agents
+    if operation_id and task_id:
+        payload["operation_id"] = operation_id
+        payload["task_id"] = task_id
 
     try:
         response = await client.post("/api/generate-agent", json=payload)
+
+        # Handle 202 Accepted for async processing
+        if response.status_code == 202:
+            logger.info(
+                f"Agent Generator accepted async request "
+                f"(operation_id={operation_id}, task_id={task_id})"
+            )
+            return {
+                "status": "accepted",
+                "operation_id": operation_id,
+                "task_id": task_id,
+            }
+
         response.raise_for_status()
         data = response.json()
 
@@ -295,6 +317,8 @@ async def generate_agent_patch_external(
     update_request: str,
     current_agent: dict[str, Any],
     library_agents: list[dict[str, Any]] | None = None,
+    operation_id: str | None = None,
+    task_id: str | None = None,
 ) -> dict[str, Any] | None:
     """Call the external service to generate a patch for an existing agent.
 
@@ -303,14 +327,14 @@ async def generate_agent_patch_external(
         current_agent: Current agent JSON
         library_agents: User's library agents available for sub-agent composition
         operation_id: Operation ID for async processing (enables Redis Streams callback)
-        session_id: Session ID for async processing (enables Redis Streams callback)
+        task_id: Task ID for async processing (enables Redis Streams callback)
 
     Returns:
         Updated agent JSON, clarifying questions dict, {"status": "accepted"} for async, or error dict on error
     """
     if _is_dummy_mode():
         return await generate_agent_patch_dummy(
-            update_request, current_agent, library_agents
+            update_request, current_agent, library_agents, operation_id, task_id
         )
 
     client = _get_client()
@@ -322,9 +346,25 @@ async def generate_agent_patch_external(
     }
     if library_agents:
         payload["library_agents"] = library_agents
+    if operation_id and task_id:
+        payload["operation_id"] = operation_id
+        payload["task_id"] = task_id
 
     try:
         response = await client.post("/api/update-agent", json=payload)
+
+        # Handle 202 Accepted for async processing
+        if response.status_code == 202:
+            logger.info(
+                f"Agent Generator accepted async update request "
+                f"(operation_id={operation_id}, task_id={task_id})"
+            )
+            return {
+                "status": "accepted",
+                "operation_id": operation_id,
+                "task_id": task_id,
+            }
+
         response.raise_for_status()
         data = response.json()
 
@@ -379,8 +419,6 @@ async def customize_template_external(
         template_agent: The template agent JSON to customize
         modification_request: Natural language description of customizations
         context: Additional context (e.g., answers to previous questions)
-        operation_id: Operation ID for async processing (enables Redis Streams callback)
-        session_id: Session ID for async processing (enables Redis Streams callback)
 
     Returns:
         Customized agent JSON, clarifying questions dict, or error dict on error

autogpt_platform/backend/backend/copilot/tools/agent_output.py

@@ -5,7 +5,7 @@ import re
 from datetime import datetime, timedelta, timezone
 from typing import Any
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, field_validator
 
 from backend.api.features.library.model import LibraryAgent
 from backend.copilot.model import ChatSession
@@ -13,7 +13,6 @@ from backend.data.db_accessors import execution_db, library_db
 from backend.data.execution import ExecutionStatus, GraphExecution, GraphExecutionMeta
 
 from .base import BaseTool
-from .execution_utils import TERMINAL_STATUSES, wait_for_execution
 from .models import (
     AgentOutputResponse,
     ErrorResponse,
@@ -34,7 +33,6 @@ class AgentOutputInput(BaseModel):
     store_slug: str = ""
     execution_id: str = ""
     run_time: str = "latest"
-    wait_if_running: int = Field(default=0, ge=0, le=300)
 
     @field_validator(
         "agent_name",
@@ -118,11 +116,6 @@ class AgentOutputTool(BaseTool):
         Select which run to retrieve using:
         - execution_id: Specific execution ID
         - run_time: 'latest' (default), 'yesterday', 'last week', or ISO date 'YYYY-MM-DD'
-
-        Wait for completion (optional):
-        - wait_if_running: Max seconds to wait if execution is still running (0-300).
-          If the execution is running/queued, waits up to this many seconds for completion.
-          Returns current status on timeout. If already finished, returns immediately.
         """
 
     @property
@@ -152,13 +145,6 @@ class AgentOutputTool(BaseTool):
                         "Time filter: 'latest', 'yesterday', 'last week', or 'YYYY-MM-DD'"
                     ),
                 },
-                "wait_if_running": {
-                    "type": "integer",
-                    "description": (
-                        "Max seconds to wait if execution is still running (0-300). "
-                        "If running, waits for completion. Returns current state on timeout."
-                    ),
-                },
             },
             "required": [],
         }
@@ -238,14 +224,10 @@ class AgentOutputTool(BaseTool):
         execution_id: str | None,
         time_start: datetime | None,
         time_end: datetime | None,
-        include_running: bool = False,
     ) -> tuple[GraphExecution | None, list[GraphExecutionMeta], str | None]:
         """
         Fetch execution(s) based on filters.
         Returns (single_execution, available_executions_meta, error_message).
-
-        Args:
-            include_running: If True, also look for running/queued executions (for waiting)
         """
         exec_db = execution_db()
 
@@ -260,25 +242,11 @@ class AgentOutputTool(BaseTool):
                 return None, [], f"Execution '{execution_id}' not found"
             return execution, [], None
 
-        # Determine which statuses to query
-        statuses = [ExecutionStatus.COMPLETED]
-        if include_running:
-            statuses.extend(
-                [
-                    ExecutionStatus.RUNNING,
-                    ExecutionStatus.QUEUED,
-                    ExecutionStatus.INCOMPLETE,
-                    ExecutionStatus.REVIEW,
-                    ExecutionStatus.FAILED,
-                    ExecutionStatus.TERMINATED,
-                ]
-            )
-
-        # Get executions with time filters
+        # Get completed executions with time filters
         executions = await exec_db.get_graph_executions(
             graph_id=graph_id,
             user_id=user_id,
-            statuses=statuses,
+            statuses=[ExecutionStatus.COMPLETED],
             created_time_gte=time_start,
             created_time_lte=time_end,
             limit=10,
@@ -345,33 +313,10 @@ class AgentOutputTool(BaseTool):
                 for e in available_executions[:5]
             ]
 
-        # Build appropriate message based on execution status
-        if execution.status == ExecutionStatus.COMPLETED:
-            message = f"Found execution outputs for agent '{agent.name}'"
-        elif execution.status == ExecutionStatus.FAILED:
-            message = f"Execution for agent '{agent.name}' failed"
-        elif execution.status == ExecutionStatus.TERMINATED:
-            message = f"Execution for agent '{agent.name}' was terminated"
-        elif execution.status == ExecutionStatus.REVIEW:
-            message = (
-                f"Execution for agent '{agent.name}' is awaiting human review. "
-                "The user needs to approve it before it can continue."
-            )
-        elif execution.status in (
-            ExecutionStatus.RUNNING,
-            ExecutionStatus.QUEUED,
-            ExecutionStatus.INCOMPLETE,
-        ):
-            message = (
-                f"Execution for agent '{agent.name}' is still {execution.status.value}. "
-                "Results may be incomplete. Use wait_if_running to wait for completion."
-            )
-        else:
-            message = f"Found execution for agent '{agent.name}' (status: {execution.status.value})"
-
+        message = f"Found execution outputs for agent '{agent.name}'"
         if len(available_executions) > 1:
             message += (
-                f" Showing latest of {len(available_executions)} matching executions."
+                f". Showing latest of {len(available_executions)} matching executions."
             )
 
         return AgentOutputResponse(
@@ -486,17 +431,13 @@ class AgentOutputTool(BaseTool):
         # Parse time expression
         time_start, time_end = parse_time_expression(input_data.run_time)
 
-        # Check if we should wait for running executions
-        wait_timeout = input_data.wait_if_running
-
-        # Fetch execution(s) - include running if we're going to wait
+        # Fetch execution(s)
         execution, available_executions, exec_error = await self._get_execution(
             user_id=user_id,
             graph_id=agent.graph_id,
             execution_id=input_data.execution_id or None,
             time_start=time_start,
             time_end=time_end,
-            include_running=wait_timeout > 0,
         )
 
         if exec_error:
@@ -505,17 +446,4 @@ class AgentOutputTool(BaseTool):
                 session_id=session_id,
             )
 
-        # If we have an execution that's still running and we should wait
-        if execution and wait_timeout > 0 and execution.status not in TERMINAL_STATUSES:
-            logger.info(
-                f"Execution {execution.id} is {execution.status}, "
-                f"waiting up to {wait_timeout}s for completion"
-            )
-            execution = await wait_for_execution(
-                user_id=user_id,
-                graph_id=agent.graph_id,
-                execution_id=execution.id,
-                timeout_seconds=wait_timeout,
-            )
-
         return self._build_response(agent, execution, available_executions, session_id)

autogpt_platform/backend/backend/copilot/tools/agent_search.py

@@ -1,13 +1,8 @@
 """Shared agent search functionality for find_agent and find_library_agent tools."""
 
-from __future__ import annotations
-
 import logging
 import re
-from typing import TYPE_CHECKING, Literal
-
-if TYPE_CHECKING:
-    from backend.api.features.library.model import LibraryAgent
+from typing import Literal
 
 from backend.data.db_accessors import library_db, store_db
 from backend.util.exceptions import DatabaseError, NotFoundError
@@ -29,24 +24,94 @@ _UUID_PATTERN = re.compile(
     re.IGNORECASE,
 )
 
-# Keywords that should be treated as "list all" rather than a literal search
-_LIST_ALL_KEYWORDS = frozenset({"all", "*", "everything", "any", ""})
+
+def _is_uuid(text: str) -> bool:
+    """Check if text is a valid UUID v4."""
+    return bool(_UUID_PATTERN.match(text.strip()))
+
+
+async def _get_library_agent_by_id(user_id: str, agent_id: str) -> AgentInfo | None:
+    """Fetch a library agent by ID (library agent ID or graph_id).
+
+    Tries multiple lookup strategies:
+    1. First by graph_id (AgentGraph primary key)
+    2. Then by library agent ID (LibraryAgent primary key)
+
+    Args:
+        user_id: The user ID
+        agent_id: The ID to look up (can be graph_id or library agent ID)
+
+    Returns:
+        AgentInfo if found, None otherwise
+    """
+    lib_db = library_db()
+
+    try:
+        agent = await lib_db.get_library_agent_by_graph_id(user_id, agent_id)
+        if agent:
+            logger.debug(f"Found library agent by graph_id: {agent.name}")
+            return AgentInfo(
+                id=agent.id,
+                name=agent.name,
+                description=agent.description or "",
+                source="library",
+                in_library=True,
+                creator=agent.creator_name,
+                status=agent.status.value,
+                can_access_graph=agent.can_access_graph,
+                has_external_trigger=agent.has_external_trigger,
+                new_output=agent.new_output,
+                graph_id=agent.graph_id,
+            )
+    except DatabaseError:
+        raise
+    except Exception as e:
+        logger.warning(
+            f"Could not fetch library agent by graph_id {agent_id}: {e}",
+            exc_info=True,
+        )
+
+    try:
+        agent = await lib_db.get_library_agent(agent_id, user_id)
+        if agent:
+            logger.debug(f"Found library agent by library_id: {agent.name}")
+            return AgentInfo(
+                id=agent.id,
+                name=agent.name,
+                description=agent.description or "",
+                source="library",
+                in_library=True,
+                creator=agent.creator_name,
+                status=agent.status.value,
+                can_access_graph=agent.can_access_graph,
+                has_external_trigger=agent.has_external_trigger,
+                new_output=agent.new_output,
+                graph_id=agent.graph_id,
+            )
+    except NotFoundError:
+        logger.debug(f"Library agent not found by library_id: {agent_id}")
+    except DatabaseError:
+        raise
+    except Exception as e:
+        logger.warning(
+            f"Could not fetch library agent by library_id {agent_id}: {e}",
+            exc_info=True,
+        )
+
+    return None
 
 
 async def search_agents(
     query: str,
     source: SearchSource,
-    session_id: str | None = None,
+    session_id: str | None,
     user_id: str | None = None,
 ) -> ToolResponseBase:
     """
     Search for agents in marketplace or user library.
 
-    For library searches, keywords like "all", "*", "everything", or an empty
-    query will list all agents without filtering.
-
     Args:
-        query: Search query string. Special keywords list all library agents.
+        query: Search query string
         source: "marketplace" or "library"
         session_id: Chat session ID
         user_id: User ID (required for library search)
@@ -54,11 +119,7 @@ async def search_agents(
     Returns:
         AgentsFoundResponse, NoResultsResponse, or ErrorResponse
     """
-    # Normalize list-all keywords to empty string for library searches
-    if source == "library" and query.lower().strip() in _LIST_ALL_KEYWORDS:
-        query = ""
-
-    if source == "marketplace" and not query:
+    if not query:
         return ErrorResponse(
             message="Please provide a search query", session_id=session_id
         )
@@ -98,18 +159,28 @@ async def search_agents(
                     logger.info(f"Found agent by direct ID lookup: {agent.name}")
 
             if not agents:
-                search_term = query or None
-                logger.info(
-                    f"{'Listing all agents in' if not query else 'Searching'} "
-                    f"user library{'' if not query else f' for: {query}'}"
-                )
+                logger.info(f"Searching user library for: {query}")
                 results = await library_db().list_library_agents(
                     user_id=user_id,  # type: ignore[arg-type]
-                    search_term=search_term,
-                    page_size=50 if not query else 10,
+                    search_term=query,
+                    page_size=10,
                 )
                 for agent in results.agents:
-                    agents.append(_library_agent_to_info(agent))
+                    agents.append(
+                        AgentInfo(
+                            id=agent.id,
+                            name=agent.name,
+                            description=agent.description or "",
+                            source="library",
+                            in_library=True,
+                            creator=agent.creator_name,
+                            status=agent.status.value,
+                            can_access_graph=agent.can_access_graph,
+                            has_external_trigger=agent.has_external_trigger,
+                            new_output=agent.new_output,
+                            graph_id=agent.graph_id,
+                        )
+                    )
         logger.info(f"Found {len(agents)} agents in {source}")
     except NotFoundError:
         pass
@@ -122,62 +193,42 @@ async def search_agents(
         )
 
     if not agents:
-        if source == "marketplace":
-            suggestions = [
+        suggestions = (
+            [
                 "Try more general terms",
                 "Browse categories in the marketplace",
                 "Check spelling",
             ]
-            no_results_msg = (
-                f"No agents found matching '{query}'. Let the user know they can "
-                "try different keywords or browse the marketplace. Also let them "
-                "know you can create a custom agent for them based on their needs."
-            )
-        elif not query:
-            # User asked to list all but library is empty
-            suggestions = [
-                "Browse the marketplace to find and add agents",
-                "Use find_agent to search the marketplace",
-            ]
-            no_results_msg = (
-                "Your library is empty. Let the user know they can browse the "
-                "marketplace to find agents, or you can create a custom agent "
-                "for them based on their needs."
-            )
-        else:
-            suggestions = [
+            if source == "marketplace"
+            else [
                 "Try different keywords",
                 "Use find_agent to search the marketplace",
                 "Check your library at /library",
             ]
-            no_results_msg = (
-                f"No agents matching '{query}' found in your library. Let the "
-                "user know you can create a custom agent for them based on "
-                "their needs."
-            )
+        )
+        no_results_msg = (
+            f"No agents found matching '{query}'. Let the user know they can try different keywords or browse the marketplace. Also let them know you can create a custom agent for them based on their needs."
+            if source == "marketplace"
+            else f"No agents matching '{query}' found in your library. Let the user know you can create a custom agent for them based on their needs."
+        )
         return NoResultsResponse(
             message=no_results_msg, session_id=session_id, suggestions=suggestions
         )
 
-    if source == "marketplace":
-        title = (
-            f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} for '{query}'"
-        )
-    elif not query:
-        title = f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} in your library"
-    else:
-        title = f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} in your library for '{query}'"
+    title = f"Found {len(agents)} agent{'s' if len(agents) != 1 else ''} "
+    title += (
+        f"for '{query}'"
+        if source == "marketplace"
+        else f"in your library for '{query}'"
+    )
 
     message = (
         "Now you have found some options for the user to choose from. "
         "You can add a link to a recommended agent at: /marketplace/agent/agent_id "
-        "Please ask the user if they would like to use any of these agents. "
-        "Let the user know we can create a custom agent for them based on their needs."
+        "Please ask the user if they would like to use any of these agents. Let the user know we can create a custom agent for them based on their needs."
         if source == "marketplace"
-        else "Found agents in the user's library. You can provide a link to view "
-        "an agent at: /library/agents/{agent_id}. Use agent_output to get "
-        "execution results, or run_agent to execute. Let the user know we can "
-        "create a custom agent for them based on their needs."
+        else "Found agents in the user's library. You can provide a link to view an agent at: "
+        "/library/agents/{agent_id}. Use agent_output to get execution results, or run_agent to execute. Let the user know we can create a custom agent for them based on their needs."
     )
 
     return AgentsFoundResponse(
@@ -187,67 +238,3 @@ async def search_agents(
         count=len(agents),
         session_id=session_id,
     )
-
-
-def _is_uuid(text: str) -> bool:
-    """Check if text is a valid UUID v4."""
-    return bool(_UUID_PATTERN.match(text.strip()))
-
-
-def _library_agent_to_info(agent: LibraryAgent) -> AgentInfo:
-    """Convert a library agent model to an AgentInfo."""
-    return AgentInfo(
-        id=agent.id,
-        name=agent.name,
-        description=agent.description or "",
-        source="library",
-        in_library=True,
-        creator=agent.creator_name,
-        status=agent.status.value,
-        can_access_graph=agent.can_access_graph,
-        has_external_trigger=agent.has_external_trigger,
-        new_output=agent.new_output,
-        graph_id=agent.graph_id,
-    )
-
-
-async def _get_library_agent_by_id(user_id: str, agent_id: str) -> AgentInfo | None:
-    """Fetch a library agent by ID (library agent ID or graph_id).
-
-    Tries multiple lookup strategies:
-    1. First by graph_id (AgentGraph primary key)
-    2. Then by library agent ID (LibraryAgent primary key)
-    """
-    lib_db = library_db()
-
-    try:
-        agent = await lib_db.get_library_agent_by_graph_id(user_id, agent_id)
-        if agent:
-            logger.debug(f"Found library agent by graph_id: {agent.name}")
-            return _library_agent_to_info(agent)
-    except NotFoundError:
-        logger.debug(f"Library agent not found by graph_id: {agent_id}")
-    except DatabaseError:
-        raise
-    except Exception as e:
-        logger.warning(
-            f"Could not fetch library agent by graph_id {agent_id}: {e}",
-            exc_info=True,
-        )
-
-    try:
-        agent = await lib_db.get_library_agent(agent_id, user_id)
-        if agent:
-            logger.debug(f"Found library agent by library_id: {agent.name}")
-            return _library_agent_to_info(agent)
-    except NotFoundError:
-        logger.debug(f"Library agent not found by library_id: {agent_id}")
-    except DatabaseError:
-        raise
-    except Exception as e:
-        logger.warning(
-            f"Could not fetch library agent by library_id {agent_id}: {e}",
-            exc_info=True,
-        )
-
-    return None

autogpt_platform/backend/backend/copilot/tools/base.py

@@ -36,6 +36,16 @@ class BaseTool:
         """Whether this tool requires authentication."""
         return False
 
+    @property
+    def is_long_running(self) -> bool:
+        """Whether this tool is long-running and should execute in background.
+
+        Long-running tools (like agent generation) are executed via background
+        tasks to survive SSE disconnections. The result is persisted to chat
+        history and visible when the user refreshes.
+        """
+        return False
+
     def as_openai_tool(self) -> ChatCompletionToolParam:
         """Convert to OpenAI tool format."""
         return ChatCompletionToolParam(

autogpt_platform/backend/backend/copilot/tools/browse_web.py

@@ -1,227 +0,0 @@
-"""Web browsing tool — navigate real browser sessions to extract page content.
-
-Uses Stagehand + Browserbase for cloud-based browser execution. Handles
-JS-rendered pages, SPAs, and dynamic content that web_fetch cannot reach.
-
-Requires environment variables:
-    STAGEHAND_API_KEY     — Browserbase API key
-    STAGEHAND_PROJECT_ID  — Browserbase project ID
-    ANTHROPIC_API_KEY     — LLM key used by Stagehand for extraction
-"""
-
-import logging
-import os
-import threading
-from typing import Any
-
-from backend.copilot.model import ChatSession
-
-from .base import BaseTool
-from .models import BrowseWebResponse, ErrorResponse, ToolResponseBase
-
-logger = logging.getLogger(__name__)
-
-# Stagehand uses the LLM internally for natural-language extraction/actions.
-_STAGEHAND_MODEL = "anthropic/claude-sonnet-4-5-20250929"
-# Hard cap on extracted content returned to the LLM context.
-_MAX_CONTENT_CHARS = 50_000
-# Explicit timeouts for Stagehand browser operations (milliseconds).
-_GOTO_TIMEOUT_MS = 30_000  # page navigation
-_EXTRACT_TIMEOUT_MS = 60_000  # LLM extraction
-
-# ---------------------------------------------------------------------------
-# Thread-safety patch for Stagehand signal handlers (applied lazily, once).
-#
-# Stagehand calls signal.signal() during __init__, which raises ValueError
-# when called from a non-main thread (e.g. the CoPilot executor thread pool).
-# We patch _register_signal_handlers to be a no-op outside the main thread.
-# The patch is applied exactly once per process via double-checked locking.
-# ---------------------------------------------------------------------------
-_stagehand_patched = False
-_patch_lock = threading.Lock()
-
-
-def _patch_stagehand_once() -> None:
-    """Monkey-patch Stagehand signal handler registration to be thread-safe.
-
-    Must be called after ``import stagehand.main`` has succeeded.
-    Safe to call from multiple threads — applies the patch at most once.
-    """
-    global _stagehand_patched
-    if _stagehand_patched:
-        return
-    with _patch_lock:
-        if _stagehand_patched:
-            return
-        import stagehand.main  # noqa: PLC0415
-
-        _original = stagehand.main.Stagehand._register_signal_handlers
-
-        def _safe_register(self: Any) -> None:
-            if threading.current_thread() is threading.main_thread():
-                _original(self)
-
-        stagehand.main.Stagehand._register_signal_handlers = _safe_register
-        _stagehand_patched = True
-
-
-class BrowseWebTool(BaseTool):
-    """Navigate a URL with a real browser and extract its content.
-
-    Use this instead of ``web_fetch`` when the page requires JavaScript
-    to render (SPAs, dashboards, paywalled content with JS checks, etc.).
-    """
-
-    @property
-    def name(self) -> str:
-        return "browse_web"
-
-    @property
-    def description(self) -> str:
-        return (
-            "Navigate to a URL using a real browser and extract content. "
-            "Handles JavaScript-rendered pages and dynamic content that "
-            "web_fetch cannot reach. "
-            "Specify exactly what to extract via the `instruction` parameter."
-        )
-
-    @property
-    def parameters(self) -> dict[str, Any]:
-        return {
-            "type": "object",
-            "properties": {
-                "url": {
-                    "type": "string",
-                    "description": "The HTTP/HTTPS URL to navigate to.",
-                },
-                "instruction": {
-                    "type": "string",
-                    "description": (
-                        "What to extract from the page. Be specific — e.g. "
-                        "'Extract all pricing plans with features and prices', "
-                        "'Get the main article text and author', "
-                        "'List all navigation links'. "
-                        "Defaults to extracting the main page content."
-                    ),
-                    "default": "Extract the main content of this page.",
-                },
-            },
-            "required": ["url"],
-        }
-
-    @property
-    def requires_auth(self) -> bool:
-        return True
-
-    async def _execute(
-        self,
-        user_id: str | None,  # noqa: ARG002
-        session: ChatSession,
-        **kwargs: Any,
-    ) -> ToolResponseBase:
-        """Navigate to a URL with a real browser and return extracted content."""
-        url: str = (kwargs.get("url") or "").strip()
-        instruction: str = (
-            kwargs.get("instruction") or "Extract the main content of this page."
-        )
-        session_id = session.session_id if session else None
-
-        if not url:
-            return ErrorResponse(
-                message="Please provide a URL to browse.",
-                error="missing_url",
-                session_id=session_id,
-            )
-
-        if not url.startswith(("http://", "https://")):
-            return ErrorResponse(
-                message="Only HTTP/HTTPS URLs are supported.",
-                error="invalid_url",
-                session_id=session_id,
-            )
-
-        api_key = os.environ.get("STAGEHAND_API_KEY")
-        project_id = os.environ.get("STAGEHAND_PROJECT_ID")
-        model_api_key = os.environ.get("ANTHROPIC_API_KEY")
-
-        if not api_key or not project_id:
-            return ErrorResponse(
-                message=(
-                    "Web browsing is not configured on this platform. "
-                    "STAGEHAND_API_KEY and STAGEHAND_PROJECT_ID are required."
-                ),
-                error="not_configured",
-                session_id=session_id,
-            )
-
-        if not model_api_key:
-            return ErrorResponse(
-                message=(
-                    "Web browsing is not configured: ANTHROPIC_API_KEY is required "
-                    "for Stagehand's extraction model."
-                ),
-                error="not_configured",
-                session_id=session_id,
-            )
-
-        # Lazy import — Stagehand is an optional heavy dependency.
-        # Importing here scopes any ImportError to this tool only, so other
-        # tools continue to register and work normally if Stagehand is absent.
-        try:
-            from stagehand import Stagehand  # noqa: PLC0415
-        except ImportError:
-            return ErrorResponse(
-                message="Web browsing is not available: Stagehand is not installed.",
-                error="not_configured",
-                session_id=session_id,
-            )
-
-        # Apply the signal handler patch now that we know stagehand is present.
-        _patch_stagehand_once()
-
-        client: Any | None = None
-        try:
-            client = Stagehand(
-                api_key=api_key,
-                project_id=project_id,
-                model_name=_STAGEHAND_MODEL,
-                model_api_key=model_api_key,
-            )
-            await client.init()
-
-            page = client.page
-            assert page is not None, "Stagehand page is not initialized"
-            await page.goto(url, timeoutMs=_GOTO_TIMEOUT_MS)
-            result = await page.extract(instruction, timeoutMs=_EXTRACT_TIMEOUT_MS)
-
-            # Extract the text content from the Pydantic result model.
-            raw = result.model_dump().get("extraction", "")
-            content = str(raw) if raw else ""
-
-            truncated = len(content) > _MAX_CONTENT_CHARS
-            if truncated:
-                suffix = "\n\n[Content truncated]"
-                keep = max(0, _MAX_CONTENT_CHARS - len(suffix))
-                content = content[:keep] + suffix
-
-            return BrowseWebResponse(
-                message=f"Browsed {url}",
-                url=url,
-                content=content,
-                truncated=truncated,
-                session_id=session_id,
-            )
-
-        except Exception:
-            logger.exception("[browse_web] Failed for %s", url)
-            return ErrorResponse(
-                message="Failed to browse URL.",
-                error="browse_failed",
-                session_id=session_id,
-            )
-        finally:
-            if client is not None:
-                try:
-                    await client.close()
-                except Exception:
-                    pass

autogpt_platform/backend/backend/copilot/tools/browse_web_test.py

@@ -1,486 +0,0 @@
-"""Unit tests for BrowseWebTool.
-
-All tests run without a running server / database.  External dependencies
-(Stagehand, Browserbase) are mocked via sys.modules injection so the suite
-stays fast and deterministic.
-"""
-
-import sys
-import threading
-import uuid
-from datetime import UTC, datetime
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-import backend.copilot.tools.browse_web as _browse_web_mod
-from backend.copilot.model import ChatSession
-from backend.copilot.tools.browse_web import (
-    _MAX_CONTENT_CHARS,
-    BrowseWebTool,
-    _patch_stagehand_once,
-)
-from backend.copilot.tools.models import BrowseWebResponse, ErrorResponse, ResponseType
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def make_session(user_id: str = "test-user") -> ChatSession:
-    return ChatSession(
-        session_id=str(uuid.uuid4()),
-        user_id=user_id,
-        messages=[],
-        usage=[],
-        started_at=datetime.now(UTC),
-        updated_at=datetime.now(UTC),
-        successful_agent_runs={},
-        successful_agent_schedules={},
-    )
-
-
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-
-
-@pytest.fixture(autouse=True)
-def reset_stagehand_patch():
-    """Reset the process-level _stagehand_patched flag before every test."""
-    _browse_web_mod._stagehand_patched = False
-    yield
-    _browse_web_mod._stagehand_patched = False
-
-
-@pytest.fixture()
-def env_vars(monkeypatch):
-    """Inject the three env vars required by BrowseWebTool."""
-    monkeypatch.setenv("STAGEHAND_API_KEY", "test-api-key")
-    monkeypatch.setenv("STAGEHAND_PROJECT_ID", "test-project-id")
-    monkeypatch.setenv("ANTHROPIC_API_KEY", "test-anthropic-key")
-
-
-@pytest.fixture()
-def stagehand_mocks(monkeypatch):
-    """Inject mock stagehand + stagehand.main into sys.modules.
-
-    Returns a dict with the mock objects so individual tests can
-    assert on calls or inject side-effects.
-    """
-    # --- mock page ---
-    mock_result = MagicMock()
-    mock_result.model_dump.return_value = {"extraction": "Page content here"}
-
-    mock_page = AsyncMock()
-    mock_page.goto = AsyncMock(return_value=None)
-    mock_page.extract = AsyncMock(return_value=mock_result)
-
-    # --- mock client ---
-    mock_client = AsyncMock()
-    mock_client.page = mock_page
-    mock_client.init = AsyncMock(return_value=None)
-    mock_client.close = AsyncMock(return_value=None)
-
-    MockStagehand = MagicMock(return_value=mock_client)
-
-    # --- stagehand top-level module ---
-    mock_stagehand = MagicMock()
-    mock_stagehand.Stagehand = MockStagehand
-
-    # --- stagehand.main (needed by _patch_stagehand_once) ---
-    mock_main = MagicMock()
-    mock_main.Stagehand = MagicMock()
-    mock_main.Stagehand._register_signal_handlers = MagicMock()
-
-    monkeypatch.setitem(sys.modules, "stagehand", mock_stagehand)
-    monkeypatch.setitem(sys.modules, "stagehand.main", mock_main)
-
-    return {
-        "client": mock_client,
-        "page": mock_page,
-        "result": mock_result,
-        "MockStagehand": MockStagehand,
-        "mock_main": mock_main,
-    }
-
-
-# ---------------------------------------------------------------------------
-# 1. Tool metadata
-# ---------------------------------------------------------------------------
-
-
-class TestBrowseWebToolMetadata:
-    def test_name(self):
-        assert BrowseWebTool().name == "browse_web"
-
-    def test_requires_auth(self):
-        assert BrowseWebTool().requires_auth is True
-
-    def test_url_is_required_parameter(self):
-        params = BrowseWebTool().parameters
-        assert "url" in params["properties"]
-        assert "url" in params["required"]
-
-    def test_instruction_is_optional(self):
-        params = BrowseWebTool().parameters
-        assert "instruction" in params["properties"]
-        assert "instruction" not in params.get("required", [])
-
-    def test_registered_in_tool_registry(self):
-        from backend.copilot.tools import TOOL_REGISTRY
-
-        assert "browse_web" in TOOL_REGISTRY
-        assert isinstance(TOOL_REGISTRY["browse_web"], BrowseWebTool)
-
-    def test_response_type_enum_value(self):
-        assert ResponseType.BROWSE_WEB == "browse_web"
-
-
-# ---------------------------------------------------------------------------
-# 2. Input validation (no external deps)
-# ---------------------------------------------------------------------------
-
-
-class TestInputValidation:
-    async def test_missing_url_returns_error(self):
-        result = await BrowseWebTool()._execute(user_id="u1", session=make_session())
-        assert isinstance(result, ErrorResponse)
-        assert "url" in result.message.lower()
-
-    async def test_empty_url_returns_error(self):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url=""
-        )
-        assert isinstance(result, ErrorResponse)
-
-    async def test_ftp_url_rejected(self):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="ftp://example.com/file"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert "http" in result.message.lower()
-
-    async def test_file_url_rejected(self):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="file:///etc/passwd"
-        )
-        assert isinstance(result, ErrorResponse)
-
-    async def test_javascript_url_rejected(self):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="javascript:alert(1)"
-        )
-        assert isinstance(result, ErrorResponse)
-
-
-# ---------------------------------------------------------------------------
-# 3. Environment variable checks
-# ---------------------------------------------------------------------------
-
-
-class TestEnvVarChecks:
-    async def test_missing_api_key(self, monkeypatch):
-        monkeypatch.delenv("STAGEHAND_API_KEY", raising=False)
-        monkeypatch.setenv("STAGEHAND_PROJECT_ID", "proj")
-        monkeypatch.setenv("ANTHROPIC_API_KEY", "key")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "not_configured"
-
-    async def test_missing_project_id(self, monkeypatch):
-        monkeypatch.setenv("STAGEHAND_API_KEY", "key")
-        monkeypatch.delenv("STAGEHAND_PROJECT_ID", raising=False)
-        monkeypatch.setenv("ANTHROPIC_API_KEY", "key")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "not_configured"
-
-    async def test_missing_anthropic_key(self, monkeypatch):
-        monkeypatch.setenv("STAGEHAND_API_KEY", "key")
-        monkeypatch.setenv("STAGEHAND_PROJECT_ID", "proj")
-        monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "not_configured"
-
-
-# ---------------------------------------------------------------------------
-# 4. Stagehand absent (ImportError path)
-# ---------------------------------------------------------------------------
-
-
-class TestStagehandAbsent:
-    async def test_returns_not_configured_error(self, env_vars, monkeypatch):
-        """Blocking the stagehand import must return a graceful ErrorResponse."""
-        # sys.modules entry set to None → Python raises ImportError on import
-        monkeypatch.setitem(sys.modules, "stagehand", None)
-        monkeypatch.setitem(sys.modules, "stagehand.main", None)
-
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "not_configured"
-        assert "not available" in result.message or "not installed" in result.message
-
-    async def test_other_tools_unaffected_when_stagehand_absent(
-        self, env_vars, monkeypatch
-    ):
-        """Registry import must not raise even when stagehand is blocked."""
-        monkeypatch.setitem(sys.modules, "stagehand", None)
-        # This import already happened at module load; just verify the registry exists
-        from backend.copilot.tools import TOOL_REGISTRY
-
-        assert "browse_web" in TOOL_REGISTRY
-        assert "web_fetch" in TOOL_REGISTRY  # unrelated tool still present
-
-
-# ---------------------------------------------------------------------------
-# 5. Successful browse
-# ---------------------------------------------------------------------------
-
-
-class TestSuccessfulBrowse:
-    async def test_returns_browse_web_response(self, env_vars, stagehand_mocks):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.url == "https://example.com"
-        assert result.content == "Page content here"
-        assert result.truncated is False
-
-    async def test_http_url_accepted(self, env_vars, stagehand_mocks):
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="http://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-
-    async def test_session_id_propagated(self, env_vars, stagehand_mocks):
-        session = make_session()
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=session, url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.session_id == session.session_id
-
-    async def test_custom_instruction_forwarded_to_extract(
-        self, env_vars, stagehand_mocks
-    ):
-        await BrowseWebTool()._execute(
-            user_id="u1",
-            session=make_session(),
-            url="https://example.com",
-            instruction="Extract all pricing plans",
-        )
-        stagehand_mocks["page"].extract.assert_awaited_once()
-        first_arg = stagehand_mocks["page"].extract.call_args[0][0]
-        assert first_arg == "Extract all pricing plans"
-
-    async def test_default_instruction_used_when_omitted(
-        self, env_vars, stagehand_mocks
-    ):
-        await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        first_arg = stagehand_mocks["page"].extract.call_args[0][0]
-        assert "main content" in first_arg.lower()
-
-    async def test_explicit_timeouts_passed_to_stagehand(
-        self, env_vars, stagehand_mocks
-    ):
-        from backend.copilot.tools.browse_web import (
-            _EXTRACT_TIMEOUT_MS,
-            _GOTO_TIMEOUT_MS,
-        )
-
-        await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        goto_kwargs = stagehand_mocks["page"].goto.call_args[1]
-        extract_kwargs = stagehand_mocks["page"].extract.call_args[1]
-        assert goto_kwargs.get("timeoutMs") == _GOTO_TIMEOUT_MS
-        assert extract_kwargs.get("timeoutMs") == _EXTRACT_TIMEOUT_MS
-
-    async def test_client_closed_after_success(self, env_vars, stagehand_mocks):
-        await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        stagehand_mocks["client"].close.assert_awaited_once()
-
-
-# ---------------------------------------------------------------------------
-# 6. Truncation
-# ---------------------------------------------------------------------------
-
-
-class TestTruncation:
-    async def test_short_content_not_truncated(self, env_vars, stagehand_mocks):
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": "short"}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.truncated is False
-        assert result.content == "short"
-
-    async def test_oversized_content_is_truncated(self, env_vars, stagehand_mocks):
-        big = "a" * (_MAX_CONTENT_CHARS + 1000)
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": big}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.truncated is True
-        assert result.content.endswith("[Content truncated]")
-
-    async def test_truncated_content_never_exceeds_cap(self, env_vars, stagehand_mocks):
-        """The final string must be ≤ _MAX_CONTENT_CHARS regardless of input size."""
-        big = "b" * (_MAX_CONTENT_CHARS * 3)
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": big}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert len(result.content) == _MAX_CONTENT_CHARS
-
-    async def test_content_exactly_at_limit_not_truncated(
-        self, env_vars, stagehand_mocks
-    ):
-        exact = "c" * _MAX_CONTENT_CHARS
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": exact}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.truncated is False
-        assert len(result.content) == _MAX_CONTENT_CHARS
-
-    async def test_empty_extraction_returns_empty_content(
-        self, env_vars, stagehand_mocks
-    ):
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": ""}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.content == ""
-        assert result.truncated is False
-
-    async def test_none_extraction_returns_empty_content(
-        self, env_vars, stagehand_mocks
-    ):
-        stagehand_mocks["result"].model_dump.return_value = {"extraction": None}
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, BrowseWebResponse)
-        assert result.content == ""
-
-
-# ---------------------------------------------------------------------------
-# 7. Error handling
-# ---------------------------------------------------------------------------
-
-
-class TestErrorHandling:
-    async def test_stagehand_init_exception_returns_generic_error(
-        self, env_vars, stagehand_mocks
-    ):
-        stagehand_mocks["client"].init.side_effect = RuntimeError("Connection refused")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "browse_failed"
-
-    async def test_raw_exception_text_not_leaked_to_user(
-        self, env_vars, stagehand_mocks
-    ):
-        """Internal error details must not appear in the user-facing message."""
-        stagehand_mocks["client"].init.side_effect = RuntimeError("SECRET_TOKEN_abc123")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert "SECRET_TOKEN_abc123" not in result.message
-        assert result.message == "Failed to browse URL."
-
-    async def test_goto_timeout_returns_error(self, env_vars, stagehand_mocks):
-        stagehand_mocks["page"].goto.side_effect = TimeoutError("Navigation timed out")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-        assert result.error == "browse_failed"
-
-    async def test_client_closed_after_exception(self, env_vars, stagehand_mocks):
-        stagehand_mocks["page"].goto.side_effect = RuntimeError("boom")
-        await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        stagehand_mocks["client"].close.assert_awaited_once()
-
-    async def test_close_failure_does_not_propagate(self, env_vars, stagehand_mocks):
-        """If close() itself raises, the tool must still return ErrorResponse."""
-        stagehand_mocks["client"].init.side_effect = RuntimeError("init failed")
-        stagehand_mocks["client"].close.side_effect = RuntimeError("close also failed")
-        result = await BrowseWebTool()._execute(
-            user_id="u1", session=make_session(), url="https://example.com"
-        )
-        assert isinstance(result, ErrorResponse)
-
-
-# ---------------------------------------------------------------------------
-# 8. Thread-safety of _patch_stagehand_once
-# ---------------------------------------------------------------------------
-
-
-class TestPatchStagehandOnce:
-    def test_idempotent_double_call(self, stagehand_mocks):
-        """_stagehand_patched transitions False→True exactly once."""
-        assert _browse_web_mod._stagehand_patched is False
-        _patch_stagehand_once()
-        assert _browse_web_mod._stagehand_patched is True
-        _patch_stagehand_once()  # second call — still True, not re-patched
-        assert _browse_web_mod._stagehand_patched is True
-
-    def test_safe_register_is_noop_in_worker_thread(self, stagehand_mocks):
-        """The patched handler must silently do nothing when called from a worker."""
-        _patch_stagehand_once()
-        mock_main = sys.modules["stagehand.main"]
-        safe_register = mock_main.Stagehand._register_signal_handlers
-
-        errors: list[Exception] = []
-
-        def run():
-            try:
-                safe_register(MagicMock())
-            except Exception as exc:
-                errors.append(exc)
-
-        t = threading.Thread(target=run)
-        t.start()
-        t.join()
-
-        assert errors == [], f"Worker thread raised: {errors}"
-
-    def test_patched_flag_set_after_execution(self, env_vars, stagehand_mocks):
-        """After a successful browse, _stagehand_patched must be True."""
-
-        async def _run():
-            return await BrowseWebTool()._execute(
-                user_id="u1", session=make_session(), url="https://example.com"
-            )
-
-        import asyncio
-
-        asyncio.get_event_loop().run_until_complete(_run())
-        assert _browse_web_mod._stagehand_patched is True

autogpt_platform/backend/backend/copilot/tools/check_operation_status.py

@@ -0,0 +1,124 @@
+"""CheckOperationStatusTool — query the status of a long-running operation."""
+
+import logging
+from typing import Any
+
+from backend.copilot.model import ChatSession
+
+from .base import BaseTool
+from .models import ErrorResponse, ResponseType, ToolResponseBase
+
+logger = logging.getLogger(__name__)
+
+
+class OperationStatusResponse(ToolResponseBase):
+    """Response for check_operation_status tool."""
+
+    type: ResponseType = ResponseType.OPERATION_STATUS
+    task_id: str
+    operation_id: str
+    status: str  # "running", "completed", "failed"
+    tool_name: str | None = None
+    message: str = ""
+
+
+class CheckOperationStatusTool(BaseTool):
+    """Check the status of a long-running operation (create_agent, edit_agent, etc.).
+
+    The CoPilot uses this tool to report back to the user whether an
+    operation that was started earlier has completed, failed, or is still
+    running.
+    """
+
+    @property
+    def name(self) -> str:
+        return "check_operation_status"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Check the current status of a long-running operation such as "
+            "create_agent or edit_agent. Accepts either an operation_id or "
+            "task_id from a previous operation_started response. "
+            "Returns the current status: running, completed, or failed."
+        )
+
+    @property
+    def parameters(self) -> dict[str, Any]:
+        return {
+            "type": "object",
+            "properties": {
+                "operation_id": {
+                    "type": "string",
+                    "description": (
+                        "The operation_id from an operation_started response."
+                    ),
+                },
+                "task_id": {
+                    "type": "string",
+                    "description": (
+                        "The task_id from an operation_started response. "
+                        "Used as fallback if operation_id is not provided."
+                    ),
+                },
+            },
+            "required": [],
+        }
+
+    @property
+    def requires_auth(self) -> bool:
+        return False
+
+    async def _execute(
+        self,
+        user_id: str | None,
+        session: ChatSession,
+        **kwargs,
+    ) -> ToolResponseBase:
+        from backend.copilot import stream_registry
+
+        operation_id = (kwargs.get("operation_id") or "").strip()
+        task_id = (kwargs.get("task_id") or "").strip()
+
+        if not operation_id and not task_id:
+            return ErrorResponse(
+                message="Please provide an operation_id or task_id.",
+                error="missing_parameter",
+            )
+
+        task = None
+        if operation_id:
+            task = await stream_registry.find_task_by_operation_id(operation_id)
+        if task is None and task_id:
+            task = await stream_registry.get_task(task_id)
+
+        if task is None:
+            # Task not in Redis — it may have already expired (TTL).
+            # Check conversation history for the result instead.
+            return ErrorResponse(
+                message=(
+                    "Operation not found — it may have already completed and "
+                    "expired from the status tracker. Check the conversation "
+                    "history for the result."
+                ),
+                error="not_found",
+            )
+
+        status_messages = {
+            "running": (
+                f"The {task.tool_name or 'operation'} is still running. "
+                "Please wait for it to complete."
+            ),
+            "completed": (
+                f"The {task.tool_name or 'operation'} has completed successfully."
+            ),
+            "failed": f"The {task.tool_name or 'operation'} has failed.",
+        }
+
+        return OperationStatusResponse(
+            task_id=task.task_id,
+            operation_id=task.operation_id,
+            status=task.status,
+            tool_name=task.tool_name,
+            message=status_messages.get(task.status, f"Status: {task.status}"),
+        )

autogpt_platform/backend/backend/copilot/tools/create_agent.py

@@ -10,6 +10,7 @@ from .agent_generator import (
     decompose_goal,
     enrich_library_agents_from_steps,
     generate_agent,
+    get_all_relevant_agents_for_generation,
     get_user_message_for_error,
     save_agent_to_library,
 )
@@ -17,6 +18,7 @@ from .base import BaseTool
 from .models import (
     AgentPreviewResponse,
     AgentSavedResponse,
+    AsyncProcessingResponse,
     ClarificationNeededResponse,
     ClarifyingQuestion,
     ErrorResponse,
@@ -38,16 +40,17 @@ class CreateAgentTool(BaseTool):
     def description(self) -> str:
         return (
             "Create a new agent workflow from a natural language description. "
-            "First generates a preview, then saves to library if save=true. "
-            "\n\nIMPORTANT: Before calling this tool, search for relevant existing agents "
-            "using find_library_agent that could be used as building blocks. "
-            "Pass their IDs in the library_agent_ids parameter so the generator can compose them."
+            "First generates a preview, then saves to library if save=true."
         )
 
     @property
     def requires_auth(self) -> bool:
         return True
 
+    @property
+    def is_long_running(self) -> bool:
+        return True
+
     @property
     def parameters(self) -> dict[str, Any]:
         return {
@@ -67,15 +70,6 @@ class CreateAgentTool(BaseTool):
                         "Include any preferences or constraints mentioned by the user."
                     ),
                 },
-                "library_agent_ids": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": (
-                        "List of library agent IDs to use as building blocks. "
-                        "Search for relevant agents using find_library_agent first, "
-                        "then pass their IDs here so they can be composed into the new agent."
-                    ),
-                },
                 "save": {
                     "type": "boolean",
                     "description": (
@@ -103,14 +97,12 @@ class CreateAgentTool(BaseTool):
         """
         description = kwargs.get("description", "").strip()
         context = kwargs.get("context", "")
-        library_agent_ids = kwargs.get("library_agent_ids", [])
         save = kwargs.get("save", True)
         session_id = session.session_id if session else None
 
-        logger.info(
-            f"[AGENT_CREATE_DEBUG] START - description_len={len(description)}, "
-            f"library_agent_ids={library_agent_ids}, save={save}, user_id={user_id}, session_id={session_id}"
-        )
+        # Extract async processing params (passed by long-running tool handler)
+        operation_id = kwargs.get("_operation_id")
+        task_id = kwargs.get("_task_id")
 
         if not description:
             return ErrorResponse(
@@ -119,34 +111,25 @@ class CreateAgentTool(BaseTool):
                 session_id=session_id,
             )
 
-        # Fetch library agents by IDs if provided
         library_agents = None
-        if user_id and library_agent_ids:
+        if user_id:
             try:
-                from .agent_generator import get_library_agents_by_ids
-
-                library_agents = await get_library_agents_by_ids(
+                library_agents = await get_all_relevant_agents_for_generation(
                     user_id=user_id,
-                    agent_ids=library_agent_ids,
+                    search_query=description,
+                    include_marketplace=True,
                 )
                 logger.debug(
-                    f"Fetched {len(library_agents)} library agents by ID for sub-agent composition"
+                    f"Found {len(library_agents)} relevant agents for sub-agent composition"
                 )
             except Exception as e:
-                logger.warning(f"Failed to fetch library agents by IDs: {e}")
+                logger.warning(f"Failed to fetch library agents: {e}")
 
         try:
             decomposition_result = await decompose_goal(
                 description, context, library_agents
             )
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] DECOMPOSE - type={decomposition_result.get('type') if decomposition_result else None}, "
-                f"session_id={session_id}"
-            )
         except AgentGeneratorNotConfiguredError:
-            logger.error(
-                f"[AGENT_CREATE_DEBUG] ERROR - AgentGeneratorNotConfigured, session_id={session_id}"
-            )
             return ErrorResponse(
                 message=(
                     "Agent generation is not available. "
@@ -247,17 +230,10 @@ class CreateAgentTool(BaseTool):
             agent_json = await generate_agent(
                 decomposition_result,
                 library_agents,
-            )
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] GENERATE - "
-                f"success={agent_json is not None}, "
-                f"is_error={isinstance(agent_json, dict) and agent_json.get('type') == 'error'}, "
-                f"session_id={session_id}"
+                operation_id=operation_id,
+                task_id=task_id,
             )
         except AgentGeneratorNotConfiguredError:
-            logger.error(
-                f"[AGENT_CREATE_DEBUG] ERROR - AgentGeneratorNotConfigured during generation, session_id={session_id}"
-            )
             return ErrorResponse(
                 message=(
                     "Agent generation is not available. "
@@ -300,20 +276,25 @@ class CreateAgentTool(BaseTool):
                 session_id=session_id,
             )
 
+        # Check if Agent Generator accepted for async processing
+        if agent_json.get("status") == "accepted":
+            logger.info(
+                f"Agent generation delegated to async processing "
+                f"(operation_id={operation_id}, task_id={task_id})"
+            )
+            return AsyncProcessingResponse(
+                message="Agent generation started. You'll be notified when it's complete.",
+                operation_id=operation_id,
+                task_id=task_id,
+                session_id=session_id,
+            )
+
         agent_name = agent_json.get("name", "Generated Agent")
         agent_description = agent_json.get("description", "")
         node_count = len(agent_json.get("nodes", []))
         link_count = len(agent_json.get("links", []))
 
-        logger.info(
-            f"[AGENT_CREATE_DEBUG] AGENT_JSON - name={agent_name}, "
-            f"nodes={node_count}, links={link_count}, save={save}, session_id={session_id}"
-        )
-
         if not save:
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] RETURN - AgentPreviewResponse, session_id={session_id}"
-            )
             return AgentPreviewResponse(
                 message=(
                     f"I've generated an agent called '{agent_name}' with {node_count} blocks. "
@@ -339,13 +320,6 @@ class CreateAgentTool(BaseTool):
                 agent_json, user_id
             )
 
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] SAVED - graph_id={created_graph.id}, "
-                f"library_agent_id={library_agent.id}, session_id={session_id}"
-            )
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] RETURN - AgentSavedResponse, session_id={session_id}"
-            )
             return AgentSavedResponse(
                 message=f"Agent '{created_graph.name}' has been saved to your library!",
                 agent_id=created_graph.id,
@@ -356,12 +330,6 @@ class CreateAgentTool(BaseTool):
                 session_id=session_id,
             )
         except Exception as e:
-            logger.error(
-                f"[AGENT_CREATE_DEBUG] ERROR - save_failed: {str(e)}, session_id={session_id}"
-            )
-            logger.info(
-                f"[AGENT_CREATE_DEBUG] RETURN - ErrorResponse (save_failed), session_id={session_id}"
-            )
             return ErrorResponse(
                 message=f"Failed to save the agent: {str(e)}",
                 error="save_failed",

autogpt_platform/backend/backend/copilot/tools/create_agent_test.py

@@ -43,6 +43,11 @@ async def test_vague_goal_returns_suggested_goal_response(tool, session):
     }
 
     with (
+        patch(
+            "backend.copilot.tools.create_agent.get_all_relevant_agents_for_generation",
+            new_callable=AsyncMock,
+            return_value=[],
+        ),
         patch(
             "backend.copilot.tools.create_agent.decompose_goal",
             new_callable=AsyncMock,
@@ -73,6 +78,11 @@ async def test_unachievable_goal_returns_suggested_goal_response(tool, session):
     }
 
     with (
+        patch(
+            "backend.copilot.tools.create_agent.get_all_relevant_agents_for_generation",
+            new_callable=AsyncMock,
+            return_value=[],
+        ),
         patch(
             "backend.copilot.tools.create_agent.decompose_goal",
             new_callable=AsyncMock,
@@ -110,6 +120,11 @@ async def test_clarifying_questions_returns_clarification_needed_response(
     }
 
     with (
+        patch(
+            "backend.copilot.tools.create_agent.get_all_relevant_agents_for_generation",
+            new_callable=AsyncMock,
+            return_value=[],
+        ),
         patch(
             "backend.copilot.tools.create_agent.decompose_goal",
             new_callable=AsyncMock,

autogpt_platform/backend/backend/copilot/tools/customize_agent.py

@@ -46,6 +46,10 @@ class CustomizeAgentTool(BaseTool):
     def requires_auth(self) -> bool:
         return True
 
+    @property
+    def is_long_running(self) -> bool:
+        return True
+
     @property
     def parameters(self) -> dict[str, Any]:
         return {

autogpt_platform/backend/backend/copilot/tools/edit_agent.py

@@ -9,6 +9,7 @@ from .agent_generator import (
     AgentGeneratorNotConfiguredError,
     generate_agent_patch,
     get_agent_as_json,
+    get_all_relevant_agents_for_generation,
     get_user_message_for_error,
     save_agent_to_library,
 )
@@ -16,6 +17,7 @@ from .base import BaseTool
 from .models import (
     AgentPreviewResponse,
     AgentSavedResponse,
+    AsyncProcessingResponse,
     ClarificationNeededResponse,
     ClarifyingQuestion,
     ErrorResponse,
@@ -36,16 +38,17 @@ class EditAgentTool(BaseTool):
     def description(self) -> str:
         return (
             "Edit an existing agent from the user's library using natural language. "
-            "Generates updates to the agent while preserving unchanged parts. "
-            "\n\nIMPORTANT: Before calling this tool, if the changes involve adding new "
-            "functionality, search for relevant existing agents using find_library_agent "
-            "that could be used as building blocks. Pass their IDs in library_agent_ids."
+            "Generates updates to the agent while preserving unchanged parts."
         )
 
     @property
     def requires_auth(self) -> bool:
         return True
 
+    @property
+    def is_long_running(self) -> bool:
+        return True
+
     @property
     def parameters(self) -> dict[str, Any]:
         return {
@@ -71,15 +74,6 @@ class EditAgentTool(BaseTool):
                         "Additional context or answers to previous clarifying questions."
                     ),
                 },
-                "library_agent_ids": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": (
-                        "List of library agent IDs to use as building blocks for the changes. "
-                        "If adding new functionality, search for relevant agents using "
-                        "find_library_agent first, then pass their IDs here."
-                    ),
-                },
                 "save": {
                     "type": "boolean",
                     "description": (
@@ -108,10 +102,13 @@ class EditAgentTool(BaseTool):
         agent_id = kwargs.get("agent_id", "").strip()
         changes = kwargs.get("changes", "").strip()
         context = kwargs.get("context", "")
-        library_agent_ids = kwargs.get("library_agent_ids", [])
         save = kwargs.get("save", True)
         session_id = session.session_id if session else None
 
+        # Extract async processing params (passed by long-running tool handler)
+        operation_id = kwargs.get("_operation_id")
+        task_id = kwargs.get("_task_id")
+
         if not agent_id:
             return ErrorResponse(
                 message="Please provide the agent ID to edit.",
@@ -135,25 +132,21 @@ class EditAgentTool(BaseTool):
                 session_id=session_id,
             )
 
-        # Fetch library agents by IDs if provided
         library_agents = None
-        if user_id and library_agent_ids:
+        if user_id:
             try:
-                from .agent_generator import get_library_agents_by_ids
-
                 graph_id = current_agent.get("id")
-                # Filter out the current agent being edited
-                filtered_ids = [id for id in library_agent_ids if id != graph_id]
-
-                library_agents = await get_library_agents_by_ids(
+                library_agents = await get_all_relevant_agents_for_generation(
                     user_id=user_id,
-                    agent_ids=filtered_ids,
+                    search_query=changes,
+                    exclude_graph_id=graph_id,
+                    include_marketplace=True,
                 )
                 logger.debug(
-                    f"Fetched {len(library_agents)} library agents by ID for sub-agent composition"
+                    f"Found {len(library_agents)} relevant agents for sub-agent composition"
                 )
             except Exception as e:
-                logger.warning(f"Failed to fetch library agents by IDs: {e}")
+                logger.warning(f"Failed to fetch library agents: {e}")
 
         update_request = changes
         if context:
@@ -164,6 +157,8 @@ class EditAgentTool(BaseTool):
                 update_request,
                 current_agent,
                 library_agents,
+                operation_id=operation_id,
+                task_id=task_id,
             )
         except AgentGeneratorNotConfiguredError:
             return ErrorResponse(
@@ -183,6 +178,19 @@ class EditAgentTool(BaseTool):
                 session_id=session_id,
             )
 
+        # Check if Agent Generator accepted for async processing
+        if result.get("status") == "accepted":
+            logger.info(
+                f"Agent edit delegated to async processing "
+                f"(operation_id={operation_id}, task_id={task_id})"
+            )
+            return AsyncProcessingResponse(
+                message="Agent edit started. You'll be notified when it's complete.",
+                operation_id=operation_id,
+                task_id=task_id,
+                session_id=session_id,
+            )
+
         # Check if the result is an error from the external service
         if isinstance(result, dict) and result.get("type") == "error":
             error_msg = result.get("error", "Unknown error")

autogpt_platform/backend/backend/copilot/tools/execution_utils.py

@@ -1,186 +0,0 @@
-"""Shared utilities for execution waiting and status handling."""
-
-import asyncio
-import logging
-from typing import Any
-
-from backend.data.db_accessors import execution_db
-from backend.data.execution import (
-    AsyncRedisExecutionEventBus,
-    ExecutionStatus,
-    GraphExecution,
-    GraphExecutionEvent,
-)
-
-logger = logging.getLogger(__name__)
-
-# Terminal statuses that indicate execution is complete
-TERMINAL_STATUSES = frozenset(
-    {
-        ExecutionStatus.COMPLETED,
-        ExecutionStatus.FAILED,
-        ExecutionStatus.TERMINATED,
-    }
-)
-
-# Statuses where execution is paused but not finished (e.g. human-in-the-loop)
-PAUSED_STATUSES = frozenset(
-    {
-        ExecutionStatus.REVIEW,
-    }
-)
-
-# Statuses that mean "stop waiting" (terminal or paused)
-STOP_WAITING_STATUSES = TERMINAL_STATUSES | PAUSED_STATUSES
-
-_POST_SUBSCRIBE_RECHECK_DELAY = 0.1  # seconds to wait for subscription to establish
-
-
-async def wait_for_execution(
-    user_id: str,
-    graph_id: str,
-    execution_id: str,
-    timeout_seconds: int,
-) -> GraphExecution | None:
-    """
-    Wait for an execution to reach a terminal or paused status using Redis pubsub.
-
-    Handles the race condition between checking status and subscribing by
-    re-checking the DB after the subscription is established.
-
-    Args:
-        user_id: User ID
-        graph_id: Graph ID
-        execution_id: Execution ID to wait for
-        timeout_seconds: Max seconds to wait
-
-    Returns:
-        The execution with current status, or None if not found
-    """
-    exec_db = execution_db()
-
-    # Quick check — maybe it's already done
-    execution = await exec_db.get_graph_execution(
-        user_id=user_id,
-        execution_id=execution_id,
-        include_node_executions=False,
-    )
-    if not execution:
-        return None
-
-    if execution.status in STOP_WAITING_STATUSES:
-        logger.debug(
-            f"Execution {execution_id} already in stop-waiting state: "
-            f"{execution.status}"
-        )
-        return execution
-
-    logger.info(
-        f"Waiting up to {timeout_seconds}s for execution {execution_id} "
-        f"(current status: {execution.status})"
-    )
-
-    event_bus = AsyncRedisExecutionEventBus()
-    channel_key = f"{user_id}/{graph_id}/{execution_id}"
-
-    # Mutable container so _subscribe_and_wait can surface the task even if
-    # asyncio.wait_for cancels the coroutine before it returns.
-    task_holder: list[asyncio.Task] = []
-
-    try:
-        result = await asyncio.wait_for(
-            _subscribe_and_wait(
-                event_bus, channel_key, user_id, execution_id, exec_db, task_holder
-            ),
-            timeout=timeout_seconds,
-        )
-        return result
-    except asyncio.TimeoutError:
-        logger.info(f"Timeout waiting for execution {execution_id}")
-    except Exception as e:
-        logger.error(f"Error waiting for execution: {e}", exc_info=True)
-    finally:
-        for task in task_holder:
-            if not task.done():
-                task.cancel()
-                try:
-                    await task
-                except asyncio.CancelledError:
-                    pass
-        await event_bus.close()
-
-    # Return current state on timeout/error
-    return await exec_db.get_graph_execution(
-        user_id=user_id,
-        execution_id=execution_id,
-        include_node_executions=False,
-    )
-
-
-async def _subscribe_and_wait(
-    event_bus: AsyncRedisExecutionEventBus,
-    channel_key: str,
-    user_id: str,
-    execution_id: str,
-    exec_db: Any,
-    task_holder: list[asyncio.Task],
-) -> GraphExecution | None:
-    """
-    Subscribe to execution events and wait for a terminal/paused status.
-
-    Appends the consumer task to ``task_holder`` so the caller can clean it up
-    even if this coroutine is cancelled by ``asyncio.wait_for``.
-
-    To avoid the race condition where the execution completes between the
-    initial DB check and the Redis subscription, we:
-    1. Start listening (which subscribes internally)
-    2. Re-check the DB after subscription is active
-    3. If still running, wait for pubsub events
-    """
-    listen_iter = event_bus.listen_events(channel_key).__aiter__()
-
-    done = asyncio.Event()
-    result_execution: GraphExecution | None = None
-
-    async def _consume() -> None:
-        nonlocal result_execution
-        try:
-            async for event in listen_iter:
-                if isinstance(event, GraphExecutionEvent):
-                    logger.debug(f"Received execution update: {event.status}")
-                    if event.status in STOP_WAITING_STATUSES:
-                        result_execution = await exec_db.get_graph_execution(
-                            user_id=user_id,
-                            execution_id=execution_id,
-                            include_node_executions=False,
-                        )
-                        done.set()
-                        return
-        except Exception as e:
-            logger.error(f"Error in execution consumer: {e}", exc_info=True)
-            done.set()
-
-    consume_task = asyncio.create_task(_consume())
-    task_holder.append(consume_task)
-
-    # Give the subscription a moment to establish, then re-check DB
-    await asyncio.sleep(_POST_SUBSCRIBE_RECHECK_DELAY)
-
-    execution = await exec_db.get_graph_execution(
-        user_id=user_id,
-        execution_id=execution_id,
-        include_node_executions=False,
-    )
-    if execution and execution.status in STOP_WAITING_STATUSES:
-        return execution
-
-    # Wait for the pubsub consumer to find a terminal event
-    await done.wait()
-    return result_execution
-
-
-def get_execution_outputs(execution: GraphExecution | None) -> dict[str, Any] | None:
-    """Extract outputs from an execution, or return None."""
-    if execution is None:
-        return None
-    return execution.outputs

autogpt_platform/backend/backend/copilot/tools/find_block_test.py

@@ -366,15 +366,12 @@ class TestFindBlockFiltering:
             return_value=(search_results, len(search_results))
         )
 
-        with (
-            patch(
-                "backend.copilot.tools.find_block.search",
-                return_value=mock_search_db,
-            ),
-            patch(
-                "backend.copilot.tools.find_block.get_block",
-                side_effect=lambda bid: mock_blocks.get(bid),
-            ),
+        with patch(
+            "backend.copilot.tools.find_block.search",
+            return_value=mock_search_db,
+        ), patch(
+            "backend.copilot.tools.find_block.get_block",
+            side_effect=lambda bid: mock_blocks.get(bid),
         ):
             tool = FindBlockTool()
             response = await tool._execute(

autogpt_platform/backend/backend/copilot/tools/find_library_agent.py

@@ -19,10 +19,9 @@ class FindLibraryAgentTool(BaseTool):
     @property
     def description(self) -> str:
         return (
-            "Search for or list agents in the user's library. Use this to find "
-            "agents the user has already added to their library, including agents "
-            "they created or added from the marketplace. "
-            "Omit the query to list all agents."
+            "Search for agents in the user's library. Use this to find agents "
+            "the user has already added to their library, including agents they "
+            "created or added from the marketplace."
         )
 
     @property
@@ -32,13 +31,10 @@ class FindLibraryAgentTool(BaseTool):
             "properties": {
                 "query": {
                     "type": "string",
-                    "description": (
-                        "Search query to find agents by name or description. "
-                        "Omit to list all agents in the library."
-                    ),
+                    "description": "Search query to find agents by name or description.",
                 },
             },
-            "required": [],
+            "required": ["query"],
         }
 
     @property
@@ -49,7 +45,7 @@ class FindLibraryAgentTool(BaseTool):
         self, user_id: str | None, session: ChatSession, **kwargs
     ) -> ToolResponseBase:
         return await search_agents(
-            query=(kwargs.get("query") or "").strip(),
+            query=kwargs.get("query", "").strip(),
             source="library",
             session_id=session.session_id,
             user_id=user_id,

autogpt_platform/backend/backend/copilot/tools/models.py

@@ -36,15 +36,17 @@ class ResponseType(str, Enum):
     WORKSPACE_FILE_WRITTEN = "workspace_file_written"
     WORKSPACE_FILE_DELETED = "workspace_file_deleted"
     # Long-running operation types
+    OPERATION_STARTED = "operation_started"
+    OPERATION_PENDING = "operation_pending"
     OPERATION_IN_PROGRESS = "operation_in_progress"
     # Input validation
     INPUT_VALIDATION_ERROR = "input_validation_error"
     # Web fetch
     WEB_FETCH = "web_fetch"
-    # Browser-based web browsing (JS-rendered pages)
-    BROWSE_WEB = "browse_web"
     # Code execution
     BASH_EXEC = "bash_exec"
+    # Operation status check
+    OPERATION_STATUS = "operation_status"
     # Feature request types
     FEATURE_REQUEST_SEARCH = "feature_request_search"
     FEATURE_REQUEST_CREATED = "feature_request_created"
@@ -418,6 +420,34 @@ class BlockOutputResponse(ToolResponseBase):
 
 
 # Long-running operation models
+class OperationStartedResponse(ToolResponseBase):
+    """Response when a long-running operation has been started in the background.
+
+    This is returned immediately to the client while the operation continues
+    to execute. The user can close the tab and check back later.
+
+    The task_id can be used to reconnect to the SSE stream via
+    GET /chat/tasks/{task_id}/stream?last_idx=0
+    """
+
+    type: ResponseType = ResponseType.OPERATION_STARTED
+    operation_id: str
+    tool_name: str
+    task_id: str | None = None  # For SSE reconnection
+
+
+class OperationPendingResponse(ToolResponseBase):
+    """Response stored in chat history while a long-running operation is executing.
+
+    This is persisted to the database so users see a pending state when they
+    refresh before the operation completes.
+    """
+
+    type: ResponseType = ResponseType.OPERATION_PENDING
+    operation_id: str
+    tool_name: str
+
+
 class OperationInProgressResponse(ToolResponseBase):
     """Response when an operation is already in progress.
 
@@ -429,6 +459,23 @@ class OperationInProgressResponse(ToolResponseBase):
     tool_call_id: str
 
 
+class AsyncProcessingResponse(ToolResponseBase):
+    """Response when an operation has been delegated to async processing.
+
+    This is returned by tools when the external service accepts the request
+    for async processing (HTTP 202 Accepted). The Redis Streams completion
+    consumer will handle the result when the external service completes.
+
+    The status field is specifically "accepted" to allow the long-running tool
+    handler to detect this response and skip LLM continuation.
+    """
+
+    type: ResponseType = ResponseType.OPERATION_STARTED
+    status: str = "accepted"  # Must be "accepted" for detection
+    operation_id: str | None = None
+    task_id: str | None = None
+
+
 class WebFetchResponse(ToolResponseBase):
     """Response for web_fetch tool."""
 
@@ -440,15 +487,6 @@ class WebFetchResponse(ToolResponseBase):
     truncated: bool = False
 
 
-class BrowseWebResponse(ToolResponseBase):
-    """Response for browse_web tool."""
-
-    type: ResponseType = ResponseType.BROWSE_WEB
-    url: str
-    content: str
-    truncated: bool = False
-
-
 class BashExecResponse(ToolResponseBase):
     """Response for bash_exec tool."""
 

autogpt_platform/backend/backend/copilot/tools/run_agent.py

@@ -9,7 +9,6 @@ from backend.copilot.config import ChatConfig
 from backend.copilot.model import ChatSession
 from backend.copilot.tracking import track_agent_run_success, track_agent_scheduled
 from backend.data.db_accessors import graph_db, library_db, user_db
-from backend.data.execution import ExecutionStatus
 from backend.data.graph import GraphModel
 from backend.data.model import CredentialsMetaInput
 from backend.executor import utils as execution_utils
@@ -21,15 +20,12 @@ from backend.util.timezone_utils import (
 )
 
 from .base import BaseTool
-from .execution_utils import get_execution_outputs, wait_for_execution
 from .helpers import get_inputs_from_schema
 from .models import (
     AgentDetails,
     AgentDetailsResponse,
-    AgentOutputResponse,
     ErrorResponse,
     ExecutionOptions,
-    ExecutionOutputInfo,
     ExecutionStartedResponse,
     InputValidationErrorResponse,
     SetupInfo,
@@ -70,7 +66,6 @@ class RunAgentInput(BaseModel):
     schedule_name: str = ""
     cron: str = ""
     timezone: str = "UTC"
-    wait_for_result: int = Field(default=0, ge=0, le=300)
 
     @field_validator(
         "username_agent_slug",
@@ -152,14 +147,6 @@ class RunAgentTool(BaseTool):
                     "type": "string",
                     "description": "IANA timezone for schedule (default: UTC)",
                 },
-                "wait_for_result": {
-                    "type": "integer",
-                    "description": (
-                        "Max seconds to wait for execution to complete (0-300). "
-                        "If >0, blocks until the execution finishes or times out. "
-                        "Returns execution outputs when complete."
-                    ),
-                },
             },
             "required": [],
         }
@@ -354,7 +341,6 @@ class RunAgentTool(BaseTool):
                     graph=graph,
                     graph_credentials=graph_credentials,
                     inputs=params.inputs,
-                    wait_for_result=params.wait_for_result,
                 )
 
         except NotFoundError as e:
@@ -438,9 +424,8 @@ class RunAgentTool(BaseTool):
         graph: GraphModel,
         graph_credentials: dict[str, CredentialsMetaInput],
         inputs: dict[str, Any],
-        wait_for_result: int = 0,
     ) -> ToolResponseBase:
-        """Execute an agent immediately, optionally waiting for completion."""
+        """Execute an agent immediately."""
         session_id = session.session_id
 
         # Check rate limits
@@ -477,91 +462,6 @@ class RunAgentTool(BaseTool):
         )
 
         library_agent_link = f"/library/agents/{library_agent.id}"
-
-        # If wait_for_result is requested, wait for execution to complete
-        if wait_for_result > 0:
-            logger.info(
-                f"Waiting up to {wait_for_result}s for execution {execution.id}"
-            )
-            completed = await wait_for_execution(
-                user_id=user_id,
-                graph_id=library_agent.graph_id,
-                execution_id=execution.id,
-                timeout_seconds=wait_for_result,
-            )
-
-            if completed and completed.status == ExecutionStatus.COMPLETED:
-                outputs = get_execution_outputs(completed)
-                return AgentOutputResponse(
-                    message=(
-                        f"Agent '{library_agent.name}' completed successfully. "
-                        f"View at {library_agent_link}."
-                    ),
-                    session_id=session_id,
-                    agent_name=library_agent.name,
-                    agent_id=library_agent.graph_id,
-                    library_agent_id=library_agent.id,
-                    library_agent_link=library_agent_link,
-                    execution=ExecutionOutputInfo(
-                        execution_id=execution.id,
-                        status=completed.status.value,
-                        started_at=completed.started_at,
-                        ended_at=completed.ended_at,
-                        outputs=outputs or {},
-                    ),
-                )
-            elif completed and completed.status == ExecutionStatus.FAILED:
-                error_detail = completed.stats.error if completed.stats else None
-                return ErrorResponse(
-                    message=(
-                        f"Agent '{library_agent.name}' execution failed. "
-                        f"View details at {library_agent_link}."
-                    ),
-                    session_id=session_id,
-                    error=error_detail,
-                )
-            elif completed and completed.status == ExecutionStatus.TERMINATED:
-                error_detail = completed.stats.error if completed.stats else None
-                return ErrorResponse(
-                    message=(
-                        f"Agent '{library_agent.name}' execution was terminated. "
-                        f"View details at {library_agent_link}."
-                    ),
-                    session_id=session_id,
-                    error=error_detail,
-                )
-            elif completed and completed.status == ExecutionStatus.REVIEW:
-                return ExecutionStartedResponse(
-                    message=(
-                        f"Agent '{library_agent.name}' is awaiting human review. "
-                        f"Check at {library_agent_link}."
-                    ),
-                    session_id=session_id,
-                    execution_id=execution.id,
-                    graph_id=library_agent.graph_id,
-                    graph_name=library_agent.name,
-                    library_agent_id=library_agent.id,
-                    library_agent_link=library_agent_link,
-                    status=ExecutionStatus.REVIEW.value,
-                )
-            else:
-                status = completed.status.value if completed else "unknown"
-                return ExecutionStartedResponse(
-                    message=(
-                        f"Agent '{library_agent.name}' is still {status} after "
-                        f"{wait_for_result}s. Check results later at "
-                        f"{library_agent_link}. "
-                        f"Use view_agent_output with wait_if_running to check again."
-                    ),
-                    session_id=session_id,
-                    execution_id=execution.id,
-                    graph_id=library_agent.graph_id,
-                    graph_name=library_agent.name,
-                    library_agent_id=library_agent.id,
-                    library_agent_link=library_agent_link,
-                    status=status,
-                )
-
         return ExecutionStartedResponse(
             message=(
                 f"Agent '{library_agent.name}' execution started successfully. "

autogpt_platform/backend/backend/copilot/tools/run_block.py

@@ -160,10 +160,9 @@ class RunBlockTool(BaseTool):
         logger.info(f"Executing block {block.name} ({block_id}) for user {user_id}")
 
         creds_manager = IntegrationCredentialsManager()
-        (
-            matched_credentials,
-            missing_credentials,
-        ) = await self._resolve_block_credentials(user_id, block, input_data)
+        matched_credentials, missing_credentials = (
+            await self._resolve_block_credentials(user_id, block, input_data)
+        )
 
         # Get block schemas for details/validation
         try:

autogpt_platform/backend/backend/copilot/tools/workspace_files.py

@@ -214,11 +214,7 @@ class WorkspaceWriteResponse(ToolResponseBase):
     file_id: str
     name: str
     path: str
-    mime_type: str
     size_bytes: int
-    # workspace:// URL the agent can embed directly in chat to give the user a link.
-    # Format: workspace://<file_id>#<mime_type>  (frontend resolves to download URL)
-    download_url: str
     source: str | None = None  # "content", "base64", or "copied from <path>"
     content_preview: str | None = None  # First 200 chars for text files
 
@@ -684,21 +680,11 @@ class WriteWorkspaceFileTool(BaseTool):
                 except Exception:
                     pass
 
-            # Strip MIME parameters (e.g. "text/html; charset=utf-8" → "text/html")
-            # and normalise to lowercase so the fragment is URL-safe.
-            normalized_mime = (rec.mime_type or "").split(";", 1)[0].strip().lower()
-            download_url = (
-                f"workspace://{rec.id}#{normalized_mime}"
-                if normalized_mime
-                else f"workspace://{rec.id}"
-            )
             return WorkspaceWriteResponse(
                 file_id=rec.id,
                 name=rec.name,
                 path=rec.path,
-                mime_type=normalized_mime,
                 size_bytes=rec.size_bytes,
-                download_url=download_url,
                 source=source,
                 content_preview=preview,
                 message=msg,

autogpt_platform/backend/backend/data/includes.py

@@ -79,12 +79,6 @@ INTEGRATION_WEBHOOK_INCLUDE: prisma.types.IntegrationWebhookInclude = {
 }
 
 
-LIBRARY_FOLDER_INCLUDE: prisma.types.LibraryFolderInclude = {
-    "LibraryAgents": {"where": {"isDeleted": False}},
-    "Children": {"where": {"isDeleted": False}},
-}
-
-
 def library_agent_include(
     user_id: str,
     include_nodes: bool = True,
@@ -111,7 +105,6 @@ def library_agent_include(
     """
     result: prisma.types.LibraryAgentInclude = {
         "Creator": True,  # Always needed for creator info
-        "Folder": True,  # Always needed for folder info
     }
 
     # Build AgentGraph include based on requested options

autogpt_platform/backend/backend/data/integrations.py

@@ -184,7 +184,7 @@ async def find_webhook_by_credentials_and_props(
     credentials_id: str,
     webhook_type: str,
     resource: str,
-    events: Optional[list[str]],
+    events: list[str],
 ) -> Webhook | None:
     webhook = await IntegrationWebhook.prisma().find_first(
         where={
@@ -192,7 +192,7 @@ async def find_webhook_by_credentials_and_props(
             "credentialsId": credentials_id,
             "webhookType": webhook_type,
             "resource": resource,
-            **({"events": {"has_every": events}} if events else {}),
+            "events": {"has_every": events},
         },
     )
     return Webhook.from_db(webhook) if webhook else None

autogpt_platform/backend/backend/data/tally.py

@@ -1,426 +0,0 @@
-"""Tally form integration: cache submissions, match by email, extract business understanding."""
-
-import asyncio
-import json
-import logging
-from datetime import datetime, timezone
-from typing import Optional
-
-from openai import AsyncOpenAI
-
-from backend.data.redis_client import get_redis_async
-from backend.data.understanding import (
-    BusinessUnderstandingInput,
-    get_business_understanding,
-    upsert_business_understanding,
-)
-from backend.util.request import Requests
-from backend.util.settings import Settings
-
-logger = logging.getLogger(__name__)
-
-TALLY_API_BASE = "https://api.tally.so"
-_settings = Settings()
-TALLY_FORM_ID = _settings.secrets.tally_form_id
-
-# Redis key templates
-_EMAIL_INDEX_KEY = "tally:form:{form_id}:email_index"
-_QUESTIONS_KEY = "tally:form:{form_id}:questions"
-_LAST_FETCH_KEY = "tally:form:{form_id}:last_fetch"
-
-# TTLs — keep aligned so last_fetch never outlives the index
-_INDEX_TTL = 3600  # 1 hour
-_LAST_FETCH_TTL = 3600  # 1 hour (same as index)
-
-# Pagination
-_PAGE_LIMIT = 500
-_MAX_PAGES = 100
-
-# LLM extraction timeout (seconds)
-_LLM_TIMEOUT = 30
-
-
-def _mask_email(email: str) -> str:
-    """Mask an email for safe logging: 'alice@example.com' -> 'a***e@example.com'."""
-    try:
-        local, domain = email.rsplit("@", 1)
-        if len(local) <= 2:
-            masked_local = local[0] + "***"
-        else:
-            masked_local = local[0] + "***" + local[-1]
-        return f"{masked_local}@{domain}"
-    except (ValueError, IndexError):
-        return "***"
-
-
-async def _fetch_tally_page(
-    client: Requests,
-    form_id: str,
-    page: int,
-    limit: int = _PAGE_LIMIT,
-    start_date: Optional[str] = None,
-) -> dict:
-    """Fetch a single page of submissions from the Tally API."""
-    url = f"{TALLY_API_BASE}/forms/{form_id}/submissions?page={page}&limit={limit}"
-    if start_date:
-        url += f"&startDate={start_date}"
-
-    response = await client.get(url)
-    return response.json()
-
-
-def _make_tally_client(api_key: str) -> Requests:
-    """Create a Requests client configured for the Tally API."""
-    return Requests(
-        trusted_origins=[TALLY_API_BASE],
-        raise_for_status=True,
-        extra_headers={
-            "Authorization": f"Bearer {api_key}",
-            "Accept": "application/json",
-        },
-    )
-
-
-async def _fetch_all_submissions(
-    client: Requests,
-    form_id: str,
-    start_date: Optional[str] = None,
-    max_pages: int = _MAX_PAGES,
-) -> tuple[list[dict], list[dict]]:
-    """Paginate through all Tally submissions. Returns (questions, submissions)."""
-
-    questions: list[dict] = []
-    all_submissions: list[dict] = []
-    page = 1
-
-    while True:
-        data = await _fetch_tally_page(client, form_id, page, start_date=start_date)
-
-        if page == 1:
-            questions = data.get("questions", [])
-
-        submissions = data.get("submissions", [])
-        all_submissions.extend(submissions)
-
-        # Tally API uses `hasMore` for pagination
-        has_more = data.get("hasMore", False)
-        if not has_more:
-            break
-        if page >= max_pages:
-            total = data.get("totalNumberOfSubmissionsPerFilter", {}).get("all", "?")
-            logger.warning(
-                f"Tally: hit max page cap ({max_pages}) for form {form_id}, "
-                f"fetched {len(all_submissions)} of {total} total submissions"
-            )
-            break
-        page += 1
-
-    return questions, all_submissions
-
-
-def _build_email_index(
-    submissions: list[dict], questions: list[dict]
-) -> dict[str, dict]:
-    """Build an {email -> submission_data} index from submissions.
-
-    Scans question titles for email/contact fields to find the email answer.
-    """
-    # Find question IDs that are likely email fields
-    email_question_ids: list[str] = []
-    for q in questions:
-        label = (q.get("label") or q.get("title") or q.get("name") or "").lower()
-        q_type = (q.get("type") or "").lower()
-        if q_type in ("input_email", "email"):
-            email_question_ids.append(q["id"])
-        elif any(kw in label for kw in ("email", "e-mail", "contact")):
-            email_question_ids.append(q["id"])
-
-    index: dict[str, dict] = {}
-    for sub in submissions:
-        email = _extract_email_from_submission(sub, email_question_ids)
-        if email:
-            index[email.lower()] = {
-                "responses": sub.get("responses", sub.get("fields", [])),
-                "submitted_at": sub.get("submittedAt", sub.get("createdAt", "")),
-                "questions": sub.get("questions", []),
-            }
-    return index
-
-
-def _extract_email_from_submission(
-    submission: dict, email_question_ids: list[str]
-) -> Optional[str]:
-    """Extract email address from a submission by checking respondentEmail, then field responses."""
-    # Try respondent email first (Tally often includes this)
-    respondent_email = submission.get("respondentEmail")
-    if respondent_email:
-        return respondent_email
-
-    # Search through responses/fields for matching question IDs
-    responses = submission.get("responses", submission.get("fields", []))
-    if isinstance(responses, list):
-        for resp in responses:
-            q_id = resp.get("questionId") or resp.get("key") or resp.get("id")
-            if q_id in email_question_ids:
-                value = resp.get("value") or resp.get("answer")
-                if isinstance(value, str) and "@" in value:
-                    return value
-    elif isinstance(responses, dict):
-        for q_id in email_question_ids:
-            value = responses.get(q_id)
-            if isinstance(value, str) and "@" in value:
-                return value
-
-    return None
-
-
-async def _get_cached_index(
-    form_id: str,
-) -> tuple[Optional[dict], Optional[list]]:
-    """Return (email_index, questions) from Redis, or (None, None) on cache miss."""
-    redis = await get_redis_async()
-    index_key = _EMAIL_INDEX_KEY.format(form_id=form_id)
-    questions_key = _QUESTIONS_KEY.format(form_id=form_id)
-
-    raw_index = await redis.get(index_key)
-    raw_questions = await redis.get(questions_key)
-
-    if raw_index and raw_questions:
-        return json.loads(raw_index), json.loads(raw_questions)
-    return None, None
-
-
-async def _refresh_cache(form_id: str) -> tuple[dict, list]:
-    """Refresh the Tally submission cache. Uses incremental fetch when possible.
-
-    Returns (email_index, questions).
-    """
-    settings = Settings()
-    client = _make_tally_client(settings.secrets.tally_api_key)
-
-    redis = await get_redis_async()
-    last_fetch_key = _LAST_FETCH_KEY.format(form_id=form_id)
-    index_key = _EMAIL_INDEX_KEY.format(form_id=form_id)
-    questions_key = _QUESTIONS_KEY.format(form_id=form_id)
-
-    last_fetch = await redis.get(last_fetch_key)
-
-    if last_fetch:
-        # Try to load existing index for incremental merge
-        raw_existing = await redis.get(index_key)
-
-        if raw_existing is None:
-            # Index expired but last_fetch still present — fall back to full fetch
-            logger.info("Tally: last_fetch present but index missing, doing full fetch")
-            questions, submissions = await _fetch_all_submissions(client, form_id)
-            email_index = _build_email_index(submissions, questions)
-        else:
-            # Incremental fetch: only get new submissions since last fetch
-            logger.info(f"Tally incremental fetch since {last_fetch}")
-            questions, new_submissions = await _fetch_all_submissions(
-                client, form_id, start_date=last_fetch
-            )
-
-            existing_index: dict[str, dict] = json.loads(raw_existing)
-
-            if not questions:
-                raw_q = await redis.get(questions_key)
-                if raw_q:
-                    questions = json.loads(raw_q)
-
-            new_index = _build_email_index(new_submissions, questions)
-            existing_index.update(new_index)
-            email_index = existing_index
-    else:
-        # Full initial fetch
-        logger.info("Tally full initial fetch")
-        questions, submissions = await _fetch_all_submissions(client, form_id)
-        email_index = _build_email_index(submissions, questions)
-
-    # Store in Redis
-    now = datetime.now(timezone.utc).isoformat()
-    await redis.setex(index_key, _INDEX_TTL, json.dumps(email_index))
-    await redis.setex(questions_key, _INDEX_TTL, json.dumps(questions))
-    await redis.setex(last_fetch_key, _LAST_FETCH_TTL, now)
-
-    logger.info(f"Tally cache refreshed: {len(email_index)} emails indexed")
-    return email_index, questions
-
-
-async def find_submission_by_email(
-    form_id: str, email: str
-) -> Optional[tuple[dict, list]]:
-    """Look up a Tally submission by email. Uses cache when available.
-
-    Returns (submission_data, questions) or None.
-    """
-    email_lower = email.lower()
-
-    # Try cache first
-    email_index, questions = await _get_cached_index(form_id)
-    if email_index is not None and questions is not None:
-        sub = email_index.get(email_lower)
-        if sub is not None:
-            return sub, questions
-        return None
-
-    # Cache miss - refresh
-    email_index, questions = await _refresh_cache(form_id)
-    sub = email_index.get(email_lower)
-    if sub is not None:
-        return sub, questions
-    return None
-
-
-def format_submission_for_llm(submission: dict, questions: list[dict]) -> str:
-    """Format a submission as readable Q&A text for LLM consumption."""
-    # Build question ID -> title lookup
-    q_titles: dict[str, str] = {}
-    for q in questions:
-        q_id = q.get("id", "")
-        title = q.get("label") or q.get("title") or q.get("name") or f"Question {q_id}"
-        q_titles[q_id] = title
-
-    lines: list[str] = []
-    responses = submission.get("responses", [])
-
-    if isinstance(responses, list):
-        for resp in responses:
-            q_id = resp.get("questionId") or resp.get("key") or resp.get("id") or ""
-            title = q_titles.get(q_id, f"Question {q_id}")
-            value = resp.get("value") or resp.get("answer") or ""
-            lines.append(f"Q: {title}\nA: {_format_answer(value)}")
-    elif isinstance(responses, dict):
-        for q_id, value in responses.items():
-            title = q_titles.get(q_id, f"Question {q_id}")
-            lines.append(f"Q: {title}\nA: {_format_answer(value)}")
-
-    return "\n\n".join(lines)
-
-
-def _format_answer(value: object) -> str:
-    """Convert an answer value (str, list, dict, None) to a human-readable string."""
-    if value is None:
-        return "(no answer)"
-    if isinstance(value, list):
-        return ", ".join(str(v) for v in value)
-    if isinstance(value, dict):
-        parts = [f"{k}: {v}" for k, v in value.items() if v]
-        return "; ".join(parts) if parts else "(no answer)"
-    return str(value)
-
-
-_EXTRACTION_PROMPT = """\
-You are a business analyst. Given the following form submission data, extract structured business understanding information.
-
-Return a JSON object with ONLY the fields that can be confidently extracted. Use null for fields that cannot be determined.
-
-Fields:
-- user_name (string): the person's name
-- job_title (string): their job title
-- business_name (string): company/business name
-- industry (string): industry or sector
-- business_size (string): company size e.g. "1-10", "11-50", "51-200"
-- user_role (string): their role context e.g. "decision maker", "implementer"
-- key_workflows (list of strings): key business workflows
-- daily_activities (list of strings): daily activities performed
-- pain_points (list of strings): current pain points
-- bottlenecks (list of strings): process bottlenecks
-- manual_tasks (list of strings): manual/repetitive tasks
-- automation_goals (list of strings): desired automation goals
-- current_software (list of strings): software/tools currently used
-- existing_automation (list of strings): existing automations
-- additional_notes (string): any additional context
-
-Form data:
-"""
-
-_EXTRACTION_SUFFIX = "\n\nReturn ONLY valid JSON."
-
-
-async def extract_business_understanding(
-    formatted_text: str,
-) -> BusinessUnderstandingInput:
-    """Use an LLM to extract structured business understanding from form text.
-
-    Raises on timeout or unparseable response so the caller can handle it.
-    """
-    settings = Settings()
-    api_key = settings.secrets.open_router_api_key
-    client = AsyncOpenAI(api_key=api_key, base_url="https://openrouter.ai/api/v1")
-
-    try:
-        response = await asyncio.wait_for(
-            client.chat.completions.create(
-                model="openai/gpt-4o-mini",
-                messages=[
-                    {
-                        "role": "user",
-                        "content": f"{_EXTRACTION_PROMPT}{formatted_text}{_EXTRACTION_SUFFIX}",
-                    }
-                ],
-                response_format={"type": "json_object"},
-                temperature=0.0,
-            ),
-            timeout=_LLM_TIMEOUT,
-        )
-    except asyncio.TimeoutError:
-        logger.warning("Tally: LLM extraction timed out")
-        raise
-
-    raw = response.choices[0].message.content or "{}"
-    try:
-        data = json.loads(raw)
-    except json.JSONDecodeError:
-        logger.warning("Tally: LLM returned invalid JSON, skipping extraction")
-        raise
-
-    # Filter out null values before constructing
-    cleaned = {k: v for k, v in data.items() if v is not None}
-    return BusinessUnderstandingInput(**cleaned)
-
-
-async def populate_understanding_from_tally(user_id: str, email: str) -> None:
-    """Main orchestrator: check Tally for a matching submission and populate understanding.
-
-    Fire-and-forget safe — all exceptions are caught and logged.
-    """
-    try:
-        # Check if understanding already exists (idempotency)
-        existing = await get_business_understanding(user_id)
-        if existing is not None:
-            logger.debug(
-                f"Tally: user {user_id} already has business understanding, skipping"
-            )
-            return
-
-        # Check API key is configured
-        settings = Settings()
-        if not settings.secrets.tally_api_key:
-            logger.debug("Tally: no API key configured, skipping")
-            return
-
-        # Look up submission by email
-        masked = _mask_email(email)
-        result = await find_submission_by_email(TALLY_FORM_ID, email)
-        if result is None:
-            logger.debug(f"Tally: no submission found for {masked}")
-            return
-
-        submission, questions = result
-        logger.info(f"Tally: found submission for {masked}, extracting understanding")
-
-        # Format and extract
-        formatted = format_submission_for_llm(submission, questions)
-        if not formatted.strip():
-            logger.warning("Tally: formatted submission was empty, skipping")
-            return
-
-        understanding_input = await extract_business_understanding(formatted)
-
-        # Upsert into database
-        await upsert_business_understanding(user_id, understanding_input)
-        logger.info(f"Tally: successfully populated understanding for user {user_id}")
-
-    except Exception:
-        logger.exception(f"Tally: error populating understanding for user {user_id}")

autogpt_platform/backend/backend/data/tally_test.py

@@ -1,589 +0,0 @@
-"""Tests for backend.data.tally module."""
-
-import asyncio
-import json
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from backend.data.tally import (
-    _EXTRACTION_PROMPT,
-    _EXTRACTION_SUFFIX,
-    _build_email_index,
-    _format_answer,
-    _make_tally_client,
-    _mask_email,
-    _refresh_cache,
-    extract_business_understanding,
-    find_submission_by_email,
-    format_submission_for_llm,
-    populate_understanding_from_tally,
-)
-
-# ── Fixtures ──────────────────────────────────────────────────────────────────
-
-SAMPLE_QUESTIONS = [
-    {"id": "q1", "label": "What is your name?", "type": "INPUT_TEXT"},
-    {"id": "q2", "label": "Email address", "type": "INPUT_EMAIL"},
-    {"id": "q3", "label": "Company name", "type": "INPUT_TEXT"},
-    {"id": "q4", "label": "Industry", "type": "INPUT_TEXT"},
-]
-
-SAMPLE_SUBMISSIONS = [
-    {
-        "respondentEmail": None,
-        "responses": [
-            {"questionId": "q1", "value": "Alice Smith"},
-            {"questionId": "q2", "value": "alice@example.com"},
-            {"questionId": "q3", "value": "Acme Corp"},
-            {"questionId": "q4", "value": "Technology"},
-        ],
-        "submittedAt": "2025-01-15T10:00:00Z",
-    },
-    {
-        "respondentEmail": "bob@example.com",
-        "responses": [
-            {"questionId": "q1", "value": "Bob Jones"},
-            {"questionId": "q2", "value": "bob@example.com"},
-            {"questionId": "q3", "value": "Bob's Burgers"},
-            {"questionId": "q4", "value": "Food"},
-        ],
-        "submittedAt": "2025-01-16T10:00:00Z",
-    },
-]
-
-
-# ── _build_email_index ────────────────────────────────────────────────────────
-
-
-def test_build_email_index():
-    index = _build_email_index(SAMPLE_SUBMISSIONS, SAMPLE_QUESTIONS)
-    assert "alice@example.com" in index
-    assert "bob@example.com" in index
-    assert len(index) == 2
-
-
-def test_build_email_index_case_insensitive():
-    submissions = [
-        {
-            "respondentEmail": None,
-            "responses": [
-                {"questionId": "q2", "value": "Alice@Example.COM"},
-            ],
-            "submittedAt": "2025-01-15T10:00:00Z",
-        },
-    ]
-    index = _build_email_index(submissions, SAMPLE_QUESTIONS)
-    assert "alice@example.com" in index
-    assert "Alice@Example.COM" not in index
-
-
-def test_build_email_index_empty():
-    index = _build_email_index([], SAMPLE_QUESTIONS)
-    assert index == {}
-
-
-def test_build_email_index_no_email_field():
-    questions = [{"id": "q1", "label": "Name", "type": "INPUT_TEXT"}]
-    submissions = [
-        {
-            "responses": [{"questionId": "q1", "value": "Alice"}],
-            "submittedAt": "2025-01-15T10:00:00Z",
-        }
-    ]
-    index = _build_email_index(submissions, questions)
-    assert index == {}
-
-
-def test_build_email_index_respondent_email():
-    """respondentEmail takes precedence over field scanning."""
-    submissions = [
-        {
-            "respondentEmail": "direct@example.com",
-            "responses": [
-                {"questionId": "q2", "value": "field@example.com"},
-            ],
-            "submittedAt": "2025-01-15T10:00:00Z",
-        }
-    ]
-    index = _build_email_index(submissions, SAMPLE_QUESTIONS)
-    assert "direct@example.com" in index
-    assert "field@example.com" not in index
-
-
-# ── format_submission_for_llm ─────────────────────────────────────────────────
-
-
-def test_format_submission_for_llm():
-    submission = {
-        "responses": [
-            {"questionId": "q1", "value": "Alice Smith"},
-            {"questionId": "q3", "value": "Acme Corp"},
-        ],
-    }
-    result = format_submission_for_llm(submission, SAMPLE_QUESTIONS)
-    assert "Q: What is your name?" in result
-    assert "A: Alice Smith" in result
-    assert "Q: Company name" in result
-    assert "A: Acme Corp" in result
-
-
-def test_format_submission_for_llm_dict_responses():
-    submission = {
-        "responses": {
-            "q1": "Alice Smith",
-            "q3": "Acme Corp",
-        },
-    }
-    result = format_submission_for_llm(submission, SAMPLE_QUESTIONS)
-    assert "A: Alice Smith" in result
-    assert "A: Acme Corp" in result
-
-
-def test_format_answer_types():
-    assert _format_answer(None) == "(no answer)"
-    assert _format_answer("hello") == "hello"
-    assert _format_answer(["a", "b"]) == "a, b"
-    assert _format_answer({"key": "val"}) == "key: val"
-    assert _format_answer(42) == "42"
-
-
-# ── find_submission_by_email ──────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_find_submission_by_email_cache_hit():
-    cached_index = {
-        "alice@example.com": {"responses": [], "submitted_at": "2025-01-15"},
-    }
-    cached_questions = SAMPLE_QUESTIONS
-
-    with patch(
-        "backend.data.tally._get_cached_index",
-        new_callable=AsyncMock,
-        return_value=(cached_index, cached_questions),
-    ) as mock_cache:
-        result = await find_submission_by_email("form123", "alice@example.com")
-
-    mock_cache.assert_awaited_once_with("form123")
-    assert result is not None
-    sub, questions = result
-    assert sub["submitted_at"] == "2025-01-15"
-
-
-@pytest.mark.asyncio
-async def test_find_submission_by_email_cache_miss():
-    refreshed_index = {
-        "alice@example.com": {"responses": [], "submitted_at": "2025-01-15"},
-    }
-
-    with (
-        patch(
-            "backend.data.tally._get_cached_index",
-            new_callable=AsyncMock,
-            return_value=(None, None),
-        ),
-        patch(
-            "backend.data.tally._refresh_cache",
-            new_callable=AsyncMock,
-            return_value=(refreshed_index, SAMPLE_QUESTIONS),
-        ) as mock_refresh,
-    ):
-        result = await find_submission_by_email("form123", "alice@example.com")
-
-    mock_refresh.assert_awaited_once_with("form123")
-    assert result is not None
-
-
-@pytest.mark.asyncio
-async def test_find_submission_by_email_no_match():
-    cached_index = {
-        "alice@example.com": {"responses": [], "submitted_at": "2025-01-15"},
-    }
-
-    with patch(
-        "backend.data.tally._get_cached_index",
-        new_callable=AsyncMock,
-        return_value=(cached_index, SAMPLE_QUESTIONS),
-    ):
-        result = await find_submission_by_email("form123", "unknown@example.com")
-
-    assert result is None
-
-
-# ── populate_understanding_from_tally ─────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_populate_understanding_skips_existing():
-    """If user already has understanding, skip entirely."""
-    mock_understanding = MagicMock()
-
-    with (
-        patch(
-            "backend.data.tally.get_business_understanding",
-            new_callable=AsyncMock,
-            return_value=mock_understanding,
-        ) as mock_get,
-        patch(
-            "backend.data.tally.find_submission_by_email",
-            new_callable=AsyncMock,
-        ) as mock_find,
-    ):
-        await populate_understanding_from_tally("user-1", "test@example.com")
-
-    mock_get.assert_awaited_once_with("user-1")
-    mock_find.assert_not_awaited()
-
-
-@pytest.mark.asyncio
-async def test_populate_understanding_skips_no_api_key():
-    """If no Tally API key, skip gracefully."""
-    mock_settings = MagicMock()
-    mock_settings.secrets.tally_api_key = ""
-
-    with (
-        patch(
-            "backend.data.tally.get_business_understanding",
-            new_callable=AsyncMock,
-            return_value=None,
-        ),
-        patch("backend.data.tally.Settings", return_value=mock_settings),
-        patch(
-            "backend.data.tally.find_submission_by_email",
-            new_callable=AsyncMock,
-        ) as mock_find,
-    ):
-        await populate_understanding_from_tally("user-1", "test@example.com")
-
-    mock_find.assert_not_awaited()
-
-
-@pytest.mark.asyncio
-async def test_populate_understanding_handles_errors():
-    """Must never raise, even on unexpected errors."""
-    with patch(
-        "backend.data.tally.get_business_understanding",
-        new_callable=AsyncMock,
-        side_effect=RuntimeError("DB down"),
-    ):
-        # Should not raise
-        await populate_understanding_from_tally("user-1", "test@example.com")
-
-
-@pytest.mark.asyncio
-async def test_populate_understanding_full_flow():
-    """Happy path: no existing understanding, finds submission, extracts, upserts."""
-    mock_settings = MagicMock()
-    mock_settings.secrets.tally_api_key = "test-key"
-
-    submission = {
-        "responses": [
-            {"questionId": "q1", "value": "Alice"},
-            {"questionId": "q3", "value": "Acme"},
-        ],
-    }
-    mock_input = MagicMock()
-
-    with (
-        patch(
-            "backend.data.tally.get_business_understanding",
-            new_callable=AsyncMock,
-            return_value=None,
-        ),
-        patch("backend.data.tally.Settings", return_value=mock_settings),
-        patch(
-            "backend.data.tally.find_submission_by_email",
-            new_callable=AsyncMock,
-            return_value=(submission, SAMPLE_QUESTIONS),
-        ),
-        patch(
-            "backend.data.tally.extract_business_understanding",
-            new_callable=AsyncMock,
-            return_value=mock_input,
-        ) as mock_extract,
-        patch(
-            "backend.data.tally.upsert_business_understanding",
-            new_callable=AsyncMock,
-        ) as mock_upsert,
-    ):
-        await populate_understanding_from_tally("user-1", "alice@example.com")
-
-    mock_extract.assert_awaited_once()
-    mock_upsert.assert_awaited_once_with("user-1", mock_input)
-
-
-@pytest.mark.asyncio
-async def test_populate_understanding_handles_llm_timeout():
-    """LLM timeout is caught and doesn't raise."""
-    import asyncio
-
-    mock_settings = MagicMock()
-    mock_settings.secrets.tally_api_key = "test-key"
-
-    submission = {
-        "responses": [{"questionId": "q1", "value": "Alice"}],
-    }
-
-    with (
-        patch(
-            "backend.data.tally.get_business_understanding",
-            new_callable=AsyncMock,
-            return_value=None,
-        ),
-        patch("backend.data.tally.Settings", return_value=mock_settings),
-        patch(
-            "backend.data.tally.find_submission_by_email",
-            new_callable=AsyncMock,
-            return_value=(submission, SAMPLE_QUESTIONS),
-        ),
-        patch(
-            "backend.data.tally.extract_business_understanding",
-            new_callable=AsyncMock,
-            side_effect=asyncio.TimeoutError(),
-        ),
-        patch(
-            "backend.data.tally.upsert_business_understanding",
-            new_callable=AsyncMock,
-        ) as mock_upsert,
-    ):
-        await populate_understanding_from_tally("user-1", "alice@example.com")
-
-    mock_upsert.assert_not_awaited()
-
-
-# ── _mask_email ───────────────────────────────────────────────────────────────
-
-
-def test_mask_email():
-    assert _mask_email("alice@example.com") == "a***e@example.com"
-    assert _mask_email("ab@example.com") == "a***@example.com"
-    assert _mask_email("a@example.com") == "a***@example.com"
-
-
-def test_mask_email_invalid():
-    assert _mask_email("no-at-sign") == "***"
-
-
-# ── Prompt construction (curly-brace safety) ─────────────────────────────────
-
-
-def test_extraction_prompt_safe_with_curly_braces():
-    """User content with curly braces must not break prompt construction.
-
-    Previously _EXTRACTION_PROMPT.format(submission_text=...) would raise
-    KeyError/ValueError if the user text contained { or }.
-    """
-    text_with_braces = "Q: What tools do you use?\nA: We use {Slack} and {{Jira}}"
-    # This must not raise — the old .format() call would fail here
-    prompt = f"{_EXTRACTION_PROMPT}{text_with_braces}{_EXTRACTION_SUFFIX}"
-    assert text_with_braces in prompt
-    assert prompt.startswith("You are a business analyst.")
-    assert prompt.endswith("Return ONLY valid JSON.")
-
-
-def test_extraction_prompt_no_format_placeholders():
-    """_EXTRACTION_PROMPT must not contain Python format placeholders."""
-    assert "{submission_text}" not in _EXTRACTION_PROMPT
-    # Ensure no stray single-brace placeholders
-    # (double braces {{ are fine — they're literal in format strings)
-    import re
-
-    single_braces = re.findall(r"(?<!\{)\{[^{].*?\}(?!\})", _EXTRACTION_PROMPT)
-    assert single_braces == [], f"Found format placeholders: {single_braces}"
-
-
-# ── extract_business_understanding ────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_extract_business_understanding_success():
-    """Happy path: LLM returns valid JSON that maps to BusinessUnderstandingInput."""
-    mock_choice = MagicMock()
-    mock_choice.message.content = json.dumps(
-        {
-            "user_name": "Alice",
-            "business_name": "Acme Corp",
-            "industry": "Technology",
-            "pain_points": ["manual reporting"],
-        }
-    )
-    mock_response = MagicMock()
-    mock_response.choices = [mock_choice]
-
-    mock_client = AsyncMock()
-    mock_client.chat.completions.create.return_value = mock_response
-
-    with patch("backend.data.tally.AsyncOpenAI", return_value=mock_client):
-        result = await extract_business_understanding("Q: Name?\nA: Alice")
-
-    assert result.user_name == "Alice"
-    assert result.business_name == "Acme Corp"
-    assert result.industry == "Technology"
-    assert result.pain_points == ["manual reporting"]
-
-
-@pytest.mark.asyncio
-async def test_extract_business_understanding_filters_nulls():
-    """Null values from LLM should be excluded from the result."""
-    mock_choice = MagicMock()
-    mock_choice.message.content = json.dumps(
-        {"user_name": "Alice", "business_name": None, "industry": None}
-    )
-    mock_response = MagicMock()
-    mock_response.choices = [mock_choice]
-
-    mock_client = AsyncMock()
-    mock_client.chat.completions.create.return_value = mock_response
-
-    with patch("backend.data.tally.AsyncOpenAI", return_value=mock_client):
-        result = await extract_business_understanding("Q: Name?\nA: Alice")
-
-    assert result.user_name == "Alice"
-    assert result.business_name is None
-    assert result.industry is None
-
-
-@pytest.mark.asyncio
-async def test_extract_business_understanding_invalid_json():
-    """Invalid JSON from LLM should raise JSONDecodeError."""
-    mock_choice = MagicMock()
-    mock_choice.message.content = "not valid json {"
-    mock_response = MagicMock()
-    mock_response.choices = [mock_choice]
-
-    mock_client = AsyncMock()
-    mock_client.chat.completions.create.return_value = mock_response
-
-    with (
-        patch("backend.data.tally.AsyncOpenAI", return_value=mock_client),
-        pytest.raises(json.JSONDecodeError),
-    ):
-        await extract_business_understanding("Q: Name?\nA: Alice")
-
-
-@pytest.mark.asyncio
-async def test_extract_business_understanding_timeout():
-    """LLM timeout should propagate as asyncio.TimeoutError."""
-    mock_client = AsyncMock()
-    mock_client.chat.completions.create.side_effect = asyncio.TimeoutError()
-
-    with (
-        patch("backend.data.tally.AsyncOpenAI", return_value=mock_client),
-        patch("backend.data.tally._LLM_TIMEOUT", 0.001),
-        pytest.raises(asyncio.TimeoutError),
-    ):
-        await extract_business_understanding("Q: Name?\nA: Alice")
-
-
-# ── _refresh_cache ───────────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_refresh_cache_full_fetch():
-    """First fetch (no last_fetch in Redis) should do a full fetch and store in Redis."""
-    mock_settings = MagicMock()
-    mock_settings.secrets.tally_api_key = "test-key"
-
-    mock_redis = AsyncMock()
-    mock_redis.get.return_value = None  # No last_fetch, no cached index
-
-    questions = SAMPLE_QUESTIONS
-    submissions = SAMPLE_SUBMISSIONS
-
-    with (
-        patch("backend.data.tally.Settings", return_value=mock_settings),
-        patch(
-            "backend.data.tally.get_redis_async",
-            new_callable=AsyncMock,
-            return_value=mock_redis,
-        ),
-        patch(
-            "backend.data.tally._fetch_all_submissions",
-            new_callable=AsyncMock,
-            return_value=(questions, submissions),
-        ) as mock_fetch,
-    ):
-        index, returned_questions = await _refresh_cache("form123")
-
-    mock_fetch.assert_awaited_once()
-    assert "alice@example.com" in index
-    assert "bob@example.com" in index
-    assert returned_questions == questions
-    # Verify Redis setex was called for index, questions, and last_fetch
-    assert mock_redis.setex.await_count == 3
-
-
-@pytest.mark.asyncio
-async def test_refresh_cache_incremental_fetch():
-    """When last_fetch and index both exist, should do incremental fetch and merge."""
-    mock_settings = MagicMock()
-    mock_settings.secrets.tally_api_key = "test-key"
-
-    existing_index = {
-        "old@example.com": {"responses": [], "submitted_at": "2025-01-01"}
-    }
-
-    mock_redis = AsyncMock()
-
-    def mock_get(key):
-        if "last_fetch" in key:
-            return "2025-01-14T00:00:00Z"
-        if "email_index" in key:
-            return json.dumps(existing_index)
-        if "questions" in key:
-            return json.dumps(SAMPLE_QUESTIONS)
-        return None
-
-    mock_redis.get.side_effect = mock_get
-
-    new_submissions = [SAMPLE_SUBMISSIONS[0]]  # Just Alice
-
-    with (
-        patch("backend.data.tally.Settings", return_value=mock_settings),
-        patch(
-            "backend.data.tally.get_redis_async",
-            new_callable=AsyncMock,
-            return_value=mock_redis,
-        ),
-        patch(
-            "backend.data.tally._fetch_all_submissions",
-            new_callable=AsyncMock,
-            return_value=(SAMPLE_QUESTIONS, new_submissions),
-        ),
-    ):
-        index, _ = await _refresh_cache("form123")
-
-    # Should contain both old and new entries
-    assert "old@example.com" in index
-    assert "alice@example.com" in index
-
-
-# ── _make_tally_client ───────────────────────────────────────────────────────
-
-
-def test_make_tally_client_returns_configured_client():
-    """_make_tally_client should create a Requests client with auth headers."""
-    client = _make_tally_client("test-api-key")
-    assert client.extra_headers is not None
-    assert client.extra_headers.get("Authorization") == "Bearer test-api-key"
-
-
-@pytest.mark.asyncio
-async def test_fetch_tally_page_uses_provided_client():
-    """_fetch_tally_page should use the passed client, not create its own."""
-    from backend.data.tally import _fetch_tally_page
-
-    mock_response = MagicMock()
-    mock_response.json.return_value = {"submissions": [], "questions": []}
-
-    mock_client = AsyncMock()
-    mock_client.get.return_value = mock_response
-
-    result = await _fetch_tally_page(mock_client, "form123", page=1)
-
-    mock_client.get.assert_awaited_once()
-    call_url = mock_client.get.call_args[0][0]
-    assert "form123" in call_url
-    assert "page=1" in call_url
-    assert result == {"submissions": [], "questions": []}

autogpt_platform/backend/backend/integrations/providers.py

@@ -47,7 +47,6 @@ class ProviderName(str, Enum):
     SLANT3D = "slant3d"
     SMARTLEAD = "smartlead"
     SMTP = "smtp"
-    TELEGRAM = "telegram"
     TWITTER = "twitter"
     TODOIST = "todoist"
     UNREAL_SPEECH = "unreal_speech"

autogpt_platform/backend/backend/integrations/webhooks/__init__.py

@@ -15,7 +15,6 @@ def load_webhook_managers() -> dict["ProviderName", type["BaseWebhooksManager"]]
     from .compass import CompassWebhookManager
     from .github import GithubWebhooksManager
     from .slant3d import Slant3DWebhooksManager
-    from .telegram import TelegramWebhooksManager
 
     webhook_managers.update(
         {
@@ -24,7 +23,6 @@ def load_webhook_managers() -> dict["ProviderName", type["BaseWebhooksManager"]]
                 CompassWebhookManager,
                 GithubWebhooksManager,
                 Slant3DWebhooksManager,
-                TelegramWebhooksManager,
             ]
         }
     )

autogpt_platform/backend/backend/integrations/webhooks/telegram.py

@@ -1,242 +0,0 @@
-"""
-Telegram Bot API Webhooks Manager.
-
-Handles webhook registration and validation for Telegram bots.
-"""
-
-import hmac
-import logging
-
-from fastapi import HTTPException, Request
-from strenum import StrEnum
-
-from backend.data import integrations
-from backend.data.model import APIKeyCredentials, Credentials
-from backend.integrations.providers import ProviderName
-from backend.util.exceptions import MissingConfigError
-from backend.util.request import Requests
-from backend.util.settings import Config
-
-from ._base import BaseWebhooksManager
-from .utils import webhook_ingress_url
-
-logger = logging.getLogger(__name__)
-
-
-class TelegramWebhookType(StrEnum):
-    BOT = "bot"
-
-
-class TelegramWebhooksManager(BaseWebhooksManager):
-    """
-    Manages Telegram bot webhooks.
-
-    Telegram webhooks are registered via the setWebhook API method.
-    Incoming requests are validated using the secret_token header.
-    """
-
-    PROVIDER_NAME = ProviderName.TELEGRAM
-    WebhookType = TelegramWebhookType
-
-    TELEGRAM_API_BASE = "https://api.telegram.org"
-
-    async def get_suitable_auto_webhook(
-        self,
-        user_id: str,
-        credentials: Credentials,
-        webhook_type: TelegramWebhookType,
-        resource: str,
-        events: list[str],
-    ) -> integrations.Webhook:
-        """
-        Telegram only supports one webhook per bot. Instead of creating a new
-        webhook object when events change (which causes the old one to be pruned
-        and deregistered — removing the ONLY webhook for the bot), we find the
-        existing webhook and update its events in place.
-        """
-        app_config = Config()
-        if not app_config.platform_base_url:
-            raise MissingConfigError(
-                "PLATFORM_BASE_URL must be set to use Webhook functionality"
-            )
-
-        # Exact match — no re-registration needed
-        if webhook := await integrations.find_webhook_by_credentials_and_props(
-            user_id=user_id,
-            credentials_id=credentials.id,
-            webhook_type=webhook_type,
-            resource=resource,
-            events=events,
-        ):
-            return webhook
-
-        # Find any existing webhook for the same bot, regardless of events
-        if existing := await integrations.find_webhook_by_credentials_and_props(
-            user_id=user_id,
-            credentials_id=credentials.id,
-            webhook_type=webhook_type,
-            resource=resource,
-            events=None,  # Ignore events for this lookup
-        ):
-            # Re-register with Telegram using the same URL but new allowed_updates
-            ingress_url = webhook_ingress_url(self.PROVIDER_NAME, existing.id)
-            _, config = await self._register_webhook(
-                credentials,
-                webhook_type,
-                resource,
-                events,
-                ingress_url,
-                existing.secret,
-            )
-            return await integrations.update_webhook(
-                existing.id, events=events, config=config
-            )
-
-        # No existing webhook at all — create a new one
-        return await self._create_webhook(
-            user_id=user_id,
-            webhook_type=webhook_type,
-            events=events,
-            resource=resource,
-            credentials=credentials,
-        )
-
-    @classmethod
-    async def validate_payload(
-        cls,
-        webhook: integrations.Webhook,
-        request: Request,
-        credentials: Credentials | None,
-    ) -> tuple[dict, str]:
-        """
-        Validates incoming Telegram webhook request.
-
-        Telegram sends X-Telegram-Bot-Api-Secret-Token header when secret_token
-        was set in setWebhook call.
-
-        Returns:
-            tuple: (payload dict, event_type string)
-        """
-        # Verify secret token header
-        secret_header = request.headers.get("X-Telegram-Bot-Api-Secret-Token")
-        if not secret_header or not hmac.compare_digest(secret_header, webhook.secret):
-            raise HTTPException(
-                status_code=403,
-                detail="Invalid or missing X-Telegram-Bot-Api-Secret-Token",
-            )
-
-        payload = await request.json()
-
-        # Determine event type based on update content
-        if "message" in payload:
-            message = payload["message"]
-            if "text" in message:
-                event_type = "message.text"
-            elif "photo" in message:
-                event_type = "message.photo"
-            elif "voice" in message:
-                event_type = "message.voice"
-            elif "audio" in message:
-                event_type = "message.audio"
-            elif "document" in message:
-                event_type = "message.document"
-            elif "video" in message:
-                event_type = "message.video"
-            else:
-                logger.warning(
-                    "Unknown Telegram webhook payload type; "
-                    f"message.keys() = {message.keys()}"
-                )
-                event_type = "message.other"
-        elif "edited_message" in payload:
-            event_type = "message.edited_message"
-        elif "message_reaction" in payload:
-            event_type = "message_reaction"
-        else:
-            event_type = "unknown"
-
-        return payload, event_type
-
-    async def _register_webhook(
-        self,
-        credentials: Credentials,
-        webhook_type: TelegramWebhookType,
-        resource: str,
-        events: list[str],
-        ingress_url: str,
-        secret: str,
-    ) -> tuple[str, dict]:
-        """
-        Register webhook with Telegram using setWebhook API.
-
-        Args:
-            credentials: Bot token credentials
-            webhook_type: Type of webhook (always BOT for Telegram)
-            resource: Resource identifier (unused for Telegram, bots are global)
-            events: Events to subscribe to
-            ingress_url: URL to receive webhook payloads
-            secret: Secret token for request validation
-
-        Returns:
-            tuple: (provider_webhook_id, config dict)
-        """
-        if not isinstance(credentials, APIKeyCredentials):
-            raise ValueError("API key (bot token) is required for Telegram webhooks")
-
-        token = credentials.api_key.get_secret_value()
-        url = f"{self.TELEGRAM_API_BASE}/bot{token}/setWebhook"
-
-        # Map event filter to Telegram allowed_updates
-        if events:
-            telegram_updates: set[str] = set()
-            for event in events:
-                telegram_updates.add(event.split(".")[0])
-                # "message.edited_message" requires the "edited_message" update type
-                if "edited_message" in event:
-                    telegram_updates.add("edited_message")
-            sorted_updates = sorted(telegram_updates)
-        else:
-            sorted_updates = ["message", "message_reaction"]
-
-        webhook_data = {
-            "url": ingress_url,
-            "secret_token": secret,
-            "allowed_updates": sorted_updates,
-        }
-
-        response = await Requests().post(url, json=webhook_data)
-        result = response.json()
-
-        if not result.get("ok"):
-            error_desc = result.get("description", "Unknown error")
-            raise ValueError(f"Failed to set Telegram webhook: {error_desc}")
-
-        # Telegram doesn't return a webhook ID, use empty string
-        config = {
-            "url": ingress_url,
-            "allowed_updates": webhook_data["allowed_updates"],
-        }
-
-        return "", config
-
-    async def _deregister_webhook(
-        self, webhook: integrations.Webhook, credentials: Credentials
-    ) -> None:
-        """
-        Deregister webhook by calling setWebhook with empty URL.
-
-        This removes the webhook from Telegram's servers.
-        """
-        if not isinstance(credentials, APIKeyCredentials):
-            raise ValueError("API key (bot token) is required for Telegram webhooks")
-
-        token = credentials.api_key.get_secret_value()
-        url = f"{self.TELEGRAM_API_BASE}/bot{token}/setWebhook"
-
-        # Setting empty URL removes the webhook
-        response = await Requests().post(url, json={"url": ""})
-        result = response.json()
-
-        if not result.get("ok"):
-            error_desc = result.get("description", "Unknown error")
-            logger.warning(f"Failed to deregister Telegram webhook: {error_desc}")

autogpt_platform/backend/backend/util/settings.py

@@ -372,7 +372,7 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
         description="The port for the Agent Generator service",
     )
     agentgenerator_timeout: int = Field(
-        default=1800,
+        default=600,
         description="The timeout in seconds for Agent Generator service requests (includes retries for rate limits)",
     )
     agentgenerator_use_dummy: bool = Field(
@@ -691,15 +691,6 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):
 
     screenshotone_api_key: str = Field(default="", description="ScreenshotOne API Key")
 
-    tally_api_key: str = Field(
-        default="",
-        description="Tally API key for form submission lookup on signup",
-    )
-    tally_form_id: str = Field(
-        default="npGe0q",
-        description="Tally form ID for signup business understanding form",
-    )
-
     apollo_api_key: str = Field(default="", description="Apollo API Key")
     smartlead_api_key: str = Field(default="", description="SmartLead API Key")
     zerobounce_api_key: str = Field(default="", description="ZeroBounce API Key")

autogpt_platform/backend/migrations/20260123110033_add_folders_in_library/migration.sql

@@ -1,33 +0,0 @@
--- AlterTable
-ALTER TABLE "LibraryAgent" ADD COLUMN     "folderId" TEXT;
-
--- CreateTable
-CREATE TABLE "LibraryFolder" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "userId" TEXT NOT NULL,
-    "name" TEXT NOT NULL,
-    "icon" TEXT,
-    "color" TEXT,
-    "parentId" TEXT,
-    "isDeleted" BOOLEAN NOT NULL DEFAULT false,
-
-    CONSTRAINT "LibraryFolder_pkey" PRIMARY KEY ("id")
-);
-
-
--- CreateIndex
-CREATE UNIQUE INDEX "LibraryFolder_userId_parentId_name_key" ON "LibraryFolder"("userId", "parentId", "name");
-
--- CreateIndex
-CREATE INDEX "LibraryAgent_folderId_idx" ON "LibraryAgent"("folderId");
-
--- AddForeignKey
-ALTER TABLE "LibraryAgent" ADD CONSTRAINT "LibraryAgent_folderId_fkey" FOREIGN KEY ("folderId") REFERENCES "LibraryFolder"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-
--- AddForeignKey
-ALTER TABLE "LibraryFolder" ADD CONSTRAINT "LibraryFolder_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-
--- AddForeignKey
-ALTER TABLE "LibraryFolder" ADD CONSTRAINT "LibraryFolder_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES "LibraryFolder"("id") ON DELETE CASCADE ON UPDATE CASCADE;

autogpt_platform/backend/migrations/20260129090000_add_suggested_blocks_materialized_view/migration.sql

@@ -1,97 +0,0 @@
--- This migration creates a materialized view for suggested blocks based on execution counts
--- The view aggregates execution counts per block for the last 14 days
---
--- IMPORTANT: For production environments, pg_cron is REQUIRED for automatic refresh
--- Prerequisites for production:
---   1. pg_cron extension must be installed: CREATE EXTENSION pg_cron;
---   2. pg_cron must be configured in postgresql.conf:
---      shared_preload_libraries = 'pg_cron'
---      cron.database_name = 'your_database_name'
---
--- For development environments without pg_cron:
---   The migration will succeed but you must manually refresh views with:
---   SET search_path TO platform;
---   SELECT refresh_suggested_blocks_view();
-
--- Check if pg_cron extension is installed
-DO $$
-DECLARE
-    has_pg_cron BOOLEAN;
-BEGIN
-    SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron') INTO has_pg_cron;
-
-    IF NOT has_pg_cron THEN
-        RAISE WARNING 'pg_cron is not installed. Materialized view will be created but will NOT refresh automatically. For production, install pg_cron. For development, manually refresh with: SELECT refresh_suggested_blocks_view();';
-    END IF;
-END
-$$;
-
--- Create materialized view for suggested blocks based on execution counts in last 14 days
--- The 14-day threshold is hardcoded to ensure consistent behavior
-CREATE MATERIALIZED VIEW IF NOT EXISTS "mv_suggested_blocks" AS
-SELECT
-    agent_node."agentBlockId" AS block_id,
-    COUNT(execution.id) AS execution_count
-FROM "AgentNodeExecution" execution
-JOIN "AgentNode" agent_node ON execution."agentNodeId" = agent_node.id
-WHERE execution."endedTime" >= (NOW() - INTERVAL '14 days')
-GROUP BY agent_node."agentBlockId"
-ORDER BY execution_count DESC;
-
--- Create unique index for concurrent refresh support
-CREATE UNIQUE INDEX IF NOT EXISTS "idx_mv_suggested_blocks_block_id" ON "mv_suggested_blocks"("block_id");
-
--- Create refresh function
-CREATE OR REPLACE FUNCTION refresh_suggested_blocks_view()
-RETURNS void
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    target_schema text := current_schema();
-BEGIN
-    -- Use CONCURRENTLY for better performance during refresh
-    REFRESH MATERIALIZED VIEW CONCURRENTLY "mv_suggested_blocks";
-    RAISE NOTICE 'Suggested blocks materialized view refreshed in schema % at %', target_schema, NOW();
-EXCEPTION
-    WHEN OTHERS THEN
-        -- Fallback to non-concurrent refresh if concurrent fails
-        REFRESH MATERIALIZED VIEW "mv_suggested_blocks";
-        RAISE NOTICE 'Suggested blocks materialized view refreshed (non-concurrent) in schema % at %. Concurrent refresh failed due to: %', target_schema, NOW(), SQLERRM;
-END;
-$$;
-
--- Initial refresh of the materialized view
-SELECT refresh_suggested_blocks_view();
-
--- Schedule automatic refresh every hour (only if pg_cron is available)
-DO $$
-DECLARE
-    has_pg_cron BOOLEAN;
-    current_schema_name text := current_schema();
-    job_name text;
-BEGIN
-    -- Check if pg_cron extension exists
-    SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron') INTO has_pg_cron;
-
-    IF has_pg_cron THEN
-        job_name := format('refresh-suggested-blocks_%s', current_schema_name);
-
-        -- Try to unschedule existing job (ignore errors if it doesn't exist)
-        BEGIN
-            PERFORM cron.unschedule(job_name);
-        EXCEPTION WHEN OTHERS THEN
-            NULL;
-        END;
-
-        -- Schedule the new job to run every hour
-        PERFORM cron.schedule(
-            job_name,
-            '0 * * * *',  -- Every hour at minute 0
-            format('SET search_path TO %I; SELECT refresh_suggested_blocks_view();', current_schema_name)
-        );
-        RAISE NOTICE 'Scheduled job %; runs every hour for schema %', job_name, current_schema_name;
-    ELSE
-        RAISE WARNING 'Automatic refresh NOT configured - pg_cron is not available. Manually refresh with: SELECT refresh_suggested_blocks_view();';
-    END IF;
-END;
-$$;

autogpt_platform/backend/migrations/20260225163452_add_library_api_key_permission/migration.sql

@@ -1,7 +0,0 @@
--- This migration adds more than one value to an enum.
--- With PostgreSQL versions 11 and earlier, this is not possible
--- in a single migration. This can be worked around by creating
--- multiple migrations, each migration adding only one value to
--- the enum.
-ALTER TYPE "APIKeyPermission" ADD VALUE 'WRITE_GRAPH';
-ALTER TYPE "APIKeyPermission" ADD VALUE 'WRITE_LIBRARY';

autogpt_platform/backend/poetry.lock

@@ -1610,101 +1610,6 @@ mccabe = ">=0.7.0,<0.8.0"
 pycodestyle = ">=2.14.0,<2.15.0"
 pyflakes = ">=3.4.0,<3.5.0"
 
-[[package]]
-name = "fonttools"
-version = "4.61.1"
-description = "Tools to manipulate font files"
-optional = false
-python-versions = ">=3.10"
-groups = ["main"]
-files = [
-    {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c7db70d57e5e1089a274cbb2b1fd635c9a24de809a231b154965d415d6c6d24"},
-    {file = "fonttools-4.61.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5fe9fd43882620017add5eabb781ebfbc6998ee49b35bd7f8f79af1f9f99a958"},
-    {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8db08051fc9e7d8bc622f2112511b8107d8f27cd89e2f64ec45e9825e8288da"},
-    {file = "fonttools-4.61.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a76d4cb80f41ba94a6691264be76435e5f72f2cb3cab0b092a6212855f71c2f6"},
-    {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a13fc8aeb24bad755eea8f7f9d409438eb94e82cf86b08fe77a03fbc8f6a96b1"},
-    {file = "fonttools-4.61.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b846a1fcf8beadeb9ea4f44ec5bdde393e2f1569e17d700bfc49cd69bde75881"},
-    {file = "fonttools-4.61.1-cp310-cp310-win32.whl", hash = "sha256:78a7d3ab09dc47ac1a363a493e6112d8cabed7ba7caad5f54dbe2f08676d1b47"},
-    {file = "fonttools-4.61.1-cp310-cp310-win_amd64.whl", hash = "sha256:eff1ac3cc66c2ac7cda1e64b4e2f3ffef474b7335f92fc3833fc632d595fcee6"},
-    {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c6604b735bb12fef8e0efd5578c9fb5d3d8532d5001ea13a19cddf295673ee09"},
-    {file = "fonttools-4.61.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5ce02f38a754f207f2f06557523cd39a06438ba3aafc0639c477ac409fc64e37"},
-    {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77efb033d8d7ff233385f30c62c7c79271c8885d5c9657d967ede124671bbdfb"},
-    {file = "fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75c1a6dfac6abd407634420c93864a1e274ebc1c7531346d9254c0d8f6ca00f9"},
-    {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0de30bfe7745c0d1ffa2b0b7048fb7123ad0d71107e10ee090fa0b16b9452e87"},
-    {file = "fonttools-4.61.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:58b0ee0ab5b1fc9921eccfe11d1435added19d6494dde14e323f25ad2bc30c56"},
-    {file = "fonttools-4.61.1-cp311-cp311-win32.whl", hash = "sha256:f79b168428351d11e10c5aeb61a74e1851ec221081299f4cf56036a95431c43a"},
-    {file = "fonttools-4.61.1-cp311-cp311-win_amd64.whl", hash = "sha256:fe2efccb324948a11dd09d22136fe2ac8a97d6c1347cf0b58a911dcd529f66b7"},
-    {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f3cb4a569029b9f291f88aafc927dd53683757e640081ca8c412781ea144565e"},
-    {file = "fonttools-4.61.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41a7170d042e8c0024703ed13b71893519a1a6d6e18e933e3ec7507a2c26a4b2"},
-    {file = "fonttools-4.61.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10d88e55330e092940584774ee5e8a6971b01fc2f4d3466a1d6c158230880796"},
-    {file = "fonttools-4.61.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15acc09befd16a0fb8a8f62bc147e1a82817542d72184acca9ce6e0aeda9fa6d"},
-    {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e6bcdf33aec38d16508ce61fd81838f24c83c90a1d1b8c68982857038673d6b8"},
-    {file = "fonttools-4.61.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5fade934607a523614726119164ff621e8c30e8fa1ffffbbd358662056ba69f0"},
-    {file = "fonttools-4.61.1-cp312-cp312-win32.whl", hash = "sha256:75da8f28eff26defba42c52986de97b22106cb8f26515b7c22443ebc9c2d3261"},
-    {file = "fonttools-4.61.1-cp312-cp312-win_amd64.whl", hash = "sha256:497c31ce314219888c0e2fce5ad9178ca83fe5230b01a5006726cdf3ac9f24d9"},
-    {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c56c488ab471628ff3bfa80964372fc13504ece601e0d97a78ee74126b2045c"},
-    {file = "fonttools-4.61.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dc492779501fa723b04d0ab1f5be046797fee17d27700476edc7ee9ae535a61e"},
-    {file = "fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:64102ca87e84261419c3747a0d20f396eb024bdbeb04c2bfb37e2891f5fadcb5"},
-    {file = "fonttools-4.61.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c1b526c8d3f615a7b1867f38a9410849c8f4aef078535742198e942fba0e9bd"},
-    {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41ed4b5ec103bd306bb68f81dc166e77409e5209443e5773cb4ed837bcc9b0d3"},
-    {file = "fonttools-4.61.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b501c862d4901792adaec7c25b1ecc749e2662543f68bb194c42ba18d6eec98d"},
-    {file = "fonttools-4.61.1-cp313-cp313-win32.whl", hash = "sha256:4d7092bb38c53bbc78e9255a59158b150bcdc115a1e3b3ce0b5f267dc35dd63c"},
-    {file = "fonttools-4.61.1-cp313-cp313-win_amd64.whl", hash = "sha256:21e7c8d76f62ab13c9472ccf74515ca5b9a761d1bde3265152a6dc58700d895b"},
-    {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fff4f534200a04b4a36e7ae3cb74493afe807b517a09e99cb4faa89a34ed6ecd"},
-    {file = "fonttools-4.61.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d9203500f7c63545b4ce3799319fe4d9feb1a1b89b28d3cb5abd11b9dd64147e"},
-    {file = "fonttools-4.61.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa646ecec9528bef693415c79a86e733c70a4965dd938e9a226b0fc64c9d2e6c"},
-    {file = "fonttools-4.61.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f35ad7805edba3aac1a3710d104592df59f4b957e30108ae0ba6c10b11dd75"},
-    {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b931ae8f62db78861b0ff1ac017851764602288575d65b8e8ff1963fed419063"},
-    {file = "fonttools-4.61.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b148b56f5de675ee16d45e769e69f87623a4944f7443850bf9a9376e628a89d2"},
-    {file = "fonttools-4.61.1-cp314-cp314-win32.whl", hash = "sha256:9b666a475a65f4e839d3d10473fad6d47e0a9db14a2f4a224029c5bfde58ad2c"},
-    {file = "fonttools-4.61.1-cp314-cp314-win_amd64.whl", hash = "sha256:4f5686e1fe5fce75d82d93c47a438a25bf0d1319d2843a926f741140b2b16e0c"},
-    {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e76ce097e3c57c4bcb67c5aa24a0ecdbd9f74ea9219997a707a4061fbe2707aa"},
-    {file = "fonttools-4.61.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9cfef3ab326780c04d6646f68d4b4742aae222e8b8ea1d627c74e38afcbc9d91"},
-    {file = "fonttools-4.61.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a75c301f96db737e1c5ed5fd7d77d9c34466de16095a266509e13da09751bd19"},
-    {file = "fonttools-4.61.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91669ccac46bbc1d09e9273546181919064e8df73488ea087dcac3e2968df9ba"},
-    {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c33ab3ca9d3ccd581d58e989d67554e42d8d4ded94ab3ade3508455fe70e65f7"},
-    {file = "fonttools-4.61.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:664c5a68ec406f6b1547946683008576ef8b38275608e1cee6c061828171c118"},
-    {file = "fonttools-4.61.1-cp314-cp314t-win32.whl", hash = "sha256:aed04cabe26f30c1647ef0e8fbb207516fd40fe9472e9439695f5c6998e60ac5"},
-    {file = "fonttools-4.61.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2180f14c141d2f0f3da43f3a81bc8aa4684860f6b0e6f9e165a4831f24e6a23b"},
-    {file = "fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371"},
-    {file = "fonttools-4.61.1.tar.gz", hash = "sha256:6675329885c44657f826ef01d9e4fb33b9158e9d93c537d84ad8399539bc6f69"},
-]
-
-[package.extras]
-all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.45.0)", "unicodedata2 (>=17.0.0) ; python_version <= \"3.14\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"]
-graphite = ["lz4 (>=1.7.4.2)"]
-interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""]
-lxml = ["lxml (>=4.0)"]
-pathops = ["skia-pathops (>=0.5.0)"]
-plot = ["matplotlib"]
-repacker = ["uharfbuzz (>=0.45.0)"]
-symfont = ["sympy"]
-type1 = ["xattr ; sys_platform == \"darwin\""]
-unicode = ["unicodedata2 (>=17.0.0) ; python_version <= \"3.14\""]
-woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"]
-
-[[package]]
-name = "fpdf2"
-version = "2.8.6"
-description = "Simple & fast PDF generation for Python"
-optional = false
-python-versions = ">=3.10"
-groups = ["main"]
-files = [
-    {file = "fpdf2-2.8.6-py3-none-any.whl", hash = "sha256:464658b896c6b0fcbf883abb316b8f0a52d582eb959d71822ba254d6c790bfdd"},
-    {file = "fpdf2-2.8.6.tar.gz", hash = "sha256:5132f26bbeee69a7ca6a292e4da1eb3241147b5aea9348b35e780ecd02bf5fc2"},
-]
-
-[package.dependencies]
-defusedxml = "*"
-fonttools = ">=4.34.0"
-Pillow = ">=8.3.2,<9.2.dev0 || >=9.3.dev0"
-
-[package.extras]
-dev = ["bandit", "black", "mypy", "pre-commit", "pylint", "pyright", "semgrep", "zizmor"]
-docs = ["lxml", "mkdocs", "mkdocs-git-revision-date-localized-plugin", "mkdocs-include-markdown-plugin", "mkdocs-macros-plugin", "mkdocs-material", "mkdocs-minify-plugin", "mkdocs-redirects", "mkdocs-with-pdf", "mknotebooks", "pdoc3"]
-test = ["brotli", "camelot-py[base]", "endesive[full]", "pytest", "pytest-cov", "qrcode", "tabula-py", "typing-extensions (>=4.0) ; python_version < \"3.11\"", "uharfbuzz"]
-
 [[package]]
 name = "frozenlist"
 version = "1.8.0"
@@ -8625,4 +8530,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.14"
-content-hash = "3869bc3fb8ea50e7101daffce13edbe563c8af568cb751adfa31fb9bb5c8318a"
+content-hash = "3ef62836d8321b9a3b8e897dade8dc6ca9022fd9468c53f384b0871b521ab343"

autogpt_platform/backend/pyproject.toml

@@ -89,7 +89,6 @@ croniter = "^6.0.0"
 stagehand = "^0.5.1"
 gravitas-md2gdocs = "^0.1.0"
 posthog = "^7.6.0"
-fpdf2 = "^2.8.6"
 
 [tool.poetry.group.dev.dependencies]
 aiohappyeyeballs = "^2.6.1"

autogpt_platform/backend/schema.prisma

@@ -51,7 +51,6 @@ model User {
   ChatSessions         ChatSession[]
   AgentPresets         AgentPreset[]
   LibraryAgents        LibraryAgent[]
-  LibraryFolders       LibraryFolder[]
 
   Profile               Profile[]
   UserOnboarding        UserOnboarding?
@@ -396,9 +395,6 @@ model LibraryAgent {
   creatorId String?
   Creator   Profile? @relation(fields: [creatorId], references: [id])
 
-  folderId String?
-  Folder   LibraryFolder? @relation(fields: [folderId], references: [id], onDelete: Restrict)
-
   useGraphIsActiveVersion Boolean @default(false)
 
   isFavorite      Boolean @default(false)
@@ -411,30 +407,6 @@ model LibraryAgent {
   @@unique([userId, agentGraphId, agentGraphVersion])
   @@index([agentGraphId, agentGraphVersion])
   @@index([creatorId])
-  @@index([folderId])
-}
-
-model LibraryFolder {
-  id        String   @id @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @default(now()) @updatedAt
-
-  userId String
-  User   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
-
-  name  String
-  icon  String?
-  color String?
-
-  parentId String?
-  Parent   LibraryFolder?  @relation("FolderHierarchy", fields: [parentId], references: [id], onDelete: Cascade)
-  Children LibraryFolder[] @relation("FolderHierarchy")
-
-  isDeleted Boolean @default(false)
-
-  LibraryAgents LibraryAgent[]
-
-  @@unique([userId, parentId, name]) // Name unique per parent per user
 }
 
 ////////////////////////////////////////////////////////////
@@ -948,17 +920,6 @@ view mv_review_stats {
   // Refresh uses CONCURRENTLY to avoid blocking reads
 }
 
-// Note: This is actually a MATERIALIZED VIEW in the database
-// Refreshed automatically every hour via pg_cron (with fallback to manual refresh)
-view mv_suggested_blocks {
-  block_id        String @unique
-  execution_count Int
-
-  // Pre-aggregated execution counts per block for the last 14 days
-  // Used by builder suggestions for ordering blocks by popularity
-  // Refresh uses CONCURRENTLY to avoid blocking reads
-}
-
 model StoreListing {
   id        String   @id @default(uuid())
   createdAt DateTime @default(now())
@@ -1130,11 +1091,9 @@ enum APIKeyPermission {
   IDENTITY // Info about the authenticated user
   EXECUTE_GRAPH // Can execute agent graphs
   READ_GRAPH // Can get graph versions and details
-  WRITE_GRAPH // Can create and update agent graphs
   EXECUTE_BLOCK // Can execute individual blocks
   READ_BLOCK // Can get block information
   READ_STORE // Can read store agents and creators
-  WRITE_LIBRARY // Can add agents to library
   USE_TOOLS // Can use chat tools via external API
   MANAGE_INTEGRATIONS // Can initiate OAuth flows and complete them
   READ_INTEGRATIONS // Can list credentials and providers

autogpt_platform/backend/snapshots/lib_agts_search

@@ -38,8 +38,6 @@
       "can_access_graph": true,
       "is_latest_version": true,
       "is_favorite": false,
-      "folder_id": null,
-      "folder_name": null,
       "recommended_schedule_cron": null,
       "settings": {
         "human_in_the_loop_safe_mode": true,
@@ -85,8 +83,6 @@
       "can_access_graph": false,
       "is_latest_version": true,
       "is_favorite": false,
-      "folder_id": null,
-      "folder_name": null,
       "recommended_schedule_cron": null,
       "settings": {
         "human_in_the_loop_safe_mode": true,

autogpt_platform/backend/test/agent_generator/test_core_integration.py

@@ -109,7 +109,7 @@ class TestGenerateAgent:
             instructions = {"type": "instructions", "steps": ["Step 1"]}
             result = await core.generate_agent(instructions)
 
-            mock_external.assert_called_once_with(instructions, None)
+            mock_external.assert_called_once_with(instructions, None, None, None)
             assert result is not None
             assert result["name"] == "Test Agent"
             assert "id" in result
@@ -173,7 +173,9 @@ class TestGenerateAgentPatch:
             current_agent = {"nodes": [], "links": []}
             result = await core.generate_agent_patch("Add a node", current_agent)
 
-            mock_external.assert_called_once_with("Add a node", current_agent, None)
+            mock_external.assert_called_once_with(
+                "Add a node", current_agent, None, None, None
+            )
             assert result == expected_result
 
     @pytest.mark.asyncio

autogpt_platform/backend/test_requeue_integration.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""
+Integration test for the requeue fix implementation.
+Tests actual RabbitMQ behavior to verify that republishing sends messages to back of queue.
+"""
+
+import json
+import time
+from threading import Event
+from typing import List
+
+from backend.data.rabbitmq import SyncRabbitMQ
+from backend.executor.utils import create_execution_queue_config
+
+
+class QueueOrderTester:
+    """Helper class to test message ordering in RabbitMQ using a dedicated test queue."""
+
+    def __init__(self):
+        self.received_messages: List[dict] = []
+        self.stop_consuming = Event()
+        self.queue_client = SyncRabbitMQ(create_execution_queue_config())
+        self.queue_client.connect()
+
+        # Use a dedicated test queue name to avoid conflicts
+        self.test_queue_name = "test_requeue_ordering"
+        self.test_exchange = "test_exchange"
+        self.test_routing_key = "test.requeue"
+
+    def setup_queue(self):
+        """Set up a dedicated test queue for testing."""
+        channel = self.queue_client.get_channel()
+
+        # Declare test exchange
+        channel.exchange_declare(
+            exchange=self.test_exchange, exchange_type="direct", durable=True
+        )
+
+        # Declare test queue
+        channel.queue_declare(
+            queue=self.test_queue_name, durable=True, auto_delete=False
+        )
+
+        # Bind queue to exchange
+        channel.queue_bind(
+            exchange=self.test_exchange,
+            queue=self.test_queue_name,
+            routing_key=self.test_routing_key,
+        )
+
+        # Purge the queue to start fresh
+        channel.queue_purge(self.test_queue_name)
+        print(f"✅ Test queue {self.test_queue_name} setup and purged")
+
+    def create_test_message(self, message_id: str, user_id: str = "test-user") -> str:
+        """Create a test graph execution message."""
+        return json.dumps(
+            {
+                "graph_exec_id": f"exec-{message_id}",
+                "graph_id": f"graph-{message_id}",
+                "user_id": user_id,
+                "execution_context": {"timezone": "UTC"},
+                "nodes_input_masks": {},
+                "starting_nodes_input": [],
+            }
+        )
+
+    def publish_message(self, message: str):
+        """Publish a message to the test queue."""
+        channel = self.queue_client.get_channel()
+        channel.basic_publish(
+            exchange=self.test_exchange,
+            routing_key=self.test_routing_key,
+            body=message,
+        )
+
+    def consume_messages(self, max_messages: int = 10, timeout: float = 5.0):
+        """Consume messages and track their order."""
+
+        def callback(ch, method, properties, body):
+            try:
+                message_data = json.loads(body.decode())
+                self.received_messages.append(message_data)
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+
+                if len(self.received_messages) >= max_messages:
+                    self.stop_consuming.set()
+            except Exception as e:
+                print(f"Error processing message: {e}")
+                ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
+
+        # Use synchronous consumption with blocking
+        channel = self.queue_client.get_channel()
+
+        # Check if there are messages in the queue first
+        method_frame, header_frame, body = channel.basic_get(
+            queue=self.test_queue_name, auto_ack=False
+        )
+        if method_frame:
+            # There are messages, set up consumer
+            channel.basic_nack(
+                delivery_tag=method_frame.delivery_tag, requeue=True
+            )  # Put message back
+
+            # Set up consumer
+            channel.basic_consume(
+                queue=self.test_queue_name,
+                on_message_callback=callback,
+            )
+
+            # Consume with timeout
+            start_time = time.time()
+            while (
+                not self.stop_consuming.is_set()
+                and (time.time() - start_time) < timeout
+                and len(self.received_messages) < max_messages
+            ):
+                try:
+                    channel.connection.process_data_events(time_limit=0.1)
+                except Exception as e:
+                    print(f"Error during consumption: {e}")
+                    break
+
+            # Cancel the consumer
+            try:
+                channel.cancel()
+            except Exception:
+                pass
+        else:
+            # No messages in queue - this might be expected for some tests
+            pass
+
+        return self.received_messages
+
+    def cleanup(self):
+        """Clean up test resources."""
+        try:
+            channel = self.queue_client.get_channel()
+            channel.queue_delete(queue=self.test_queue_name)
+            channel.exchange_delete(exchange=self.test_exchange)
+            print(f"✅ Test queue {self.test_queue_name} cleaned up")
+        except Exception as e:
+            print(f"⚠️ Cleanup issue: {e}")
+
+
+def test_queue_ordering_behavior():
+    """
+    Integration test to verify that our republishing method sends messages to back of queue.
+    This tests the actual fix for the rate limiting queue blocking issue.
+    """
+    tester = QueueOrderTester()
+
+    try:
+        tester.setup_queue()
+
+        print("🧪 Testing actual RabbitMQ queue ordering behavior...")
+
+        # Test 1: Normal FIFO behavior
+        print("1. Testing normal FIFO queue behavior")
+
+        # Publish messages in order: A, B, C
+        msg_a = tester.create_test_message("A")
+        msg_b = tester.create_test_message("B")
+        msg_c = tester.create_test_message("C")
+
+        tester.publish_message(msg_a)
+        tester.publish_message(msg_b)
+        tester.publish_message(msg_c)
+
+        # Consume and verify FIFO order: A, B, C
+        tester.received_messages = []
+        tester.stop_consuming.clear()
+        messages = tester.consume_messages(max_messages=3)
+
+        assert len(messages) == 3, f"Expected 3 messages, got {len(messages)}"
+        assert (
+            messages[0]["graph_exec_id"] == "exec-A"
+        ), f"First message should be A, got {messages[0]['graph_exec_id']}"
+        assert (
+            messages[1]["graph_exec_id"] == "exec-B"
+        ), f"Second message should be B, got {messages[1]['graph_exec_id']}"
+        assert (
+            messages[2]["graph_exec_id"] == "exec-C"
+        ), f"Third message should be C, got {messages[2]['graph_exec_id']}"
+
+        print("✅ FIFO order confirmed: A -> B -> C")
+
+        # Test 2: Rate limiting simulation - the key test!
+        print("2. Testing rate limiting fix scenario")
+
+        # Simulate the scenario where user1 is rate limited
+        user1_msg = tester.create_test_message("RATE-LIMITED", "user1")
+        user2_msg1 = tester.create_test_message("USER2-1", "user2")
+        user2_msg2 = tester.create_test_message("USER2-2", "user2")
+
+        # Initially publish user1 message (gets consumed, then rate limited on retry)
+        tester.publish_message(user1_msg)
+
+        # Other users publish their messages
+        tester.publish_message(user2_msg1)
+        tester.publish_message(user2_msg2)
+
+        # Now simulate: user1 message gets "requeued" using our new republishing method
+        # This is what happens in manager.py when requeue_by_republishing=True
+        tester.publish_message(user1_msg)  # Goes to back via our method
+
+        # Expected order: RATE-LIMITED, USER2-1, USER2-2, RATE-LIMITED (republished to back)
+        # This shows that user2 messages get processed instead of being blocked
+        tester.received_messages = []
+        tester.stop_consuming.clear()
+        messages = tester.consume_messages(max_messages=4)
+
+        assert len(messages) == 4, f"Expected 4 messages, got {len(messages)}"
+
+        # The key verification: user2 messages are NOT blocked by user1's rate-limited message
+        user2_messages = [msg for msg in messages if msg["user_id"] == "user2"]
+        assert len(user2_messages) == 2, "Both user2 messages should be processed"
+        assert user2_messages[0]["graph_exec_id"] == "exec-USER2-1"
+        assert user2_messages[1]["graph_exec_id"] == "exec-USER2-2"
+
+        print("✅ Rate limiting fix confirmed: user2 executions NOT blocked by user1")
+
+        # Test 3: Verify our method behaves like going to back of queue
+        print("3. Testing republishing sends messages to back")
+
+        # Start with message X in queue
+        msg_x = tester.create_test_message("X")
+        tester.publish_message(msg_x)
+
+        # Add message Y
+        msg_y = tester.create_test_message("Y")
+        tester.publish_message(msg_y)
+
+        # Republish X (simulates requeue using our method)
+        tester.publish_message(msg_x)
+
+        # Expected: X, Y, X (X was republished to back)
+        tester.received_messages = []
+        tester.stop_consuming.clear()
+        messages = tester.consume_messages(max_messages=3)
+
+        assert len(messages) == 3
+        # Y should come before the republished X
+        y_index = next(
+            i for i, msg in enumerate(messages) if msg["graph_exec_id"] == "exec-Y"
+        )
+        republished_x_index = next(
+            i
+            for i, msg in enumerate(messages[1:], 1)
+            if msg["graph_exec_id"] == "exec-X"
+        )
+
+        assert (
+            y_index < republished_x_index
+        ), f"Y should come before republished X, but got order: {[m['graph_exec_id'] for m in messages]}"
+
+        print("✅ Republishing confirmed: messages go to back of queue")
+
+        print("🎉 All integration tests passed!")
+        print("🎉 Our republishing method works correctly with real RabbitMQ")
+        print("🎉 Queue blocking issue is fixed!")
+
+    finally:
+        tester.cleanup()
+
+
+def test_traditional_requeue_behavior():
+    """
+    Test that traditional requeue (basic_nack with requeue=True) sends messages to FRONT of queue.
+    This validates our hypothesis about why queue blocking occurs.
+    """
+    tester = QueueOrderTester()
+
+    try:
+        tester.setup_queue()
+        print("🧪 Testing traditional requeue behavior (basic_nack with requeue=True)")
+
+        # Step 1: Publish message A
+        msg_a = tester.create_test_message("A")
+        tester.publish_message(msg_a)
+
+        # Step 2: Publish message B
+        msg_b = tester.create_test_message("B")
+        tester.publish_message(msg_b)
+
+        # Step 3: Consume message A and requeue it using traditional method
+        channel = tester.queue_client.get_channel()
+        method_frame, header_frame, body = channel.basic_get(
+            queue=tester.test_queue_name, auto_ack=False
+        )
+
+        assert method_frame is not None, "Should have received message A"
+        consumed_msg = json.loads(body.decode())
+        assert (
+            consumed_msg["graph_exec_id"] == "exec-A"
+        ), f"Should have consumed message A, got {consumed_msg['graph_exec_id']}"
+
+        # Traditional requeue: basic_nack with requeue=True (sends to FRONT)
+        channel.basic_nack(delivery_tag=method_frame.delivery_tag, requeue=True)
+        print(f"🔄 Traditional requeue (to FRONT): {consumed_msg['graph_exec_id']}")
+
+        # Step 4: Consume all messages using basic_get for reliability
+        received_messages = []
+
+        # Get first message
+        method_frame, header_frame, body = channel.basic_get(
+            queue=tester.test_queue_name, auto_ack=True
+        )
+        if method_frame:
+            msg = json.loads(body.decode())
+            received_messages.append(msg)
+
+        # Get second message
+        method_frame, header_frame, body = channel.basic_get(
+            queue=tester.test_queue_name, auto_ack=True
+        )
+        if method_frame:
+            msg = json.loads(body.decode())
+            received_messages.append(msg)
+
+        # CRITICAL ASSERTION: Traditional requeue should put A at FRONT
+        # Expected order: A (requeued to front), B
+        assert (
+            len(received_messages) == 2
+        ), f"Expected 2 messages, got {len(received_messages)}"
+
+        first_msg = received_messages[0]["graph_exec_id"]
+        second_msg = received_messages[1]["graph_exec_id"]
+
+        # This is the critical test: requeued message A should come BEFORE B
+        assert (
+            first_msg == "exec-A"
+        ), f"Traditional requeue should put A at FRONT, but first message was: {first_msg}"
+        assert (
+            second_msg == "exec-B"
+        ), f"B should come after requeued A, but second message was: {second_msg}"
+
+        print(
+            "✅ HYPOTHESIS CONFIRMED: Traditional requeue sends messages to FRONT of queue"
+        )
+        print(f"   Order: {first_msg} (requeued to front) → {second_msg}")
+        print("   This explains why rate-limited messages block other users!")
+
+    finally:
+        tester.cleanup()
+
+
+if __name__ == "__main__":
+    test_queue_ordering_behavior()

autogpt_platform/frontend/.storybook/main.ts

@@ -6,7 +6,6 @@ const config: StorybookConfig = {
     "../src/components/tokens/**/*.stories.@(js|jsx|mjs|ts|tsx)",
     "../src/components/atoms/**/*.stories.@(js|jsx|mjs|ts|tsx)",
     "../src/components/molecules/**/*.stories.@(js|jsx|mjs|ts|tsx)",
-    "../src/components/ai-elements/**/*.stories.@(js|jsx|mjs|ts|tsx)",
   ],
   addons: [
     "@storybook/addon-a11y",

autogpt_platform/frontend/package.json

@@ -23,6 +23,8 @@
     "build-storybook": "storybook build",
     "test-storybook": "test-storybook",
     "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"pnpm run build-storybook -- --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && pnpm run test-storybook\"",
+    "react-doctor": "npx -y react-doctor@latest . --verbose",
+    "react-doctor:diff": "npx -y react-doctor@latest . --verbose --diff",
     "generate:api": "npx --yes tsx ./scripts/generate-api-queries.ts && orval --config ./orval.config.ts",
     "generate:api:force": "npx --yes tsx ./scripts/generate-api-queries.ts --force && orval --config ./orval.config.ts"
   },
@@ -32,7 +34,6 @@
   "dependencies": {
     "@ai-sdk/react": "3.0.61",
     "@faker-js/faker": "10.0.0",
-    "@ferrucc-io/emoji-picker": "0.0.48",
     "@hookform/resolvers": "5.2.2",
     "@next/third-parties": "15.4.6",
     "@phosphor-icons/react": "2.1.10",

autogpt_platform/frontend/pnpm-lock.yaml

@@ -18,9 +18,6 @@ importers:
       '@faker-js/faker':
         specifier: 10.0.0
         version: 10.0.0
-      '@ferrucc-io/emoji-picker':
-        specifier: 0.0.48
-        version: 0.0.48(@babel/core@7.28.5)(@babel/template@7.27.2)(@types/react@18.3.17)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tailwindcss@3.4.17)
       '@hookform/resolvers':
         specifier: 5.2.2
         version: 5.2.2(react-hook-form@7.66.0(react@18.3.1))
@@ -1510,14 +1507,6 @@ packages:
     resolution: {integrity: sha512-UollFEUkVXutsaP+Vndjxar40Gs5JL2HeLcl8xO1QAjJgOdhc3OmBFWyEylS+RddWaaBiAzH+5/17PLQJwDiLw==}
     engines: {node: ^20.19.0 || ^22.13.0 || ^23.5.0 || >=24.0.0, npm: '>=10'}
 
-  '@ferrucc-io/emoji-picker@0.0.48':
-    resolution: {integrity: sha512-DJ5u+6VLF9OK7x+S/luwrVb5CHC6W16jL5b8vBUYNpxKWSuFgyliDHVtw1SGe6+dr5RUbf8WQwPJdKZmU3Ittg==}
-    engines: {node: '>=18'}
-    peerDependencies:
-      react: ^18.2.0 || ^19.0.0
-      react-dom: ^18.2.0 || ^19.0.0
-      tailwindcss: '>=3.0.0'
-
   '@floating-ui/core@1.7.3':
     resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==}
 
@@ -3125,10 +3114,6 @@ packages:
   '@shikijs/vscode-textmate@10.0.2':
     resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==}
 
-  '@sindresorhus/is@4.6.0':
-    resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
-    engines: {node: '>=10'}
-
   '@standard-schema/spec@1.0.0':
     resolution: {integrity: sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==}
 
@@ -3391,19 +3376,10 @@ packages:
       react: '>=16.8'
       react-dom: '>=16.8'
 
-  '@tanstack/react-virtual@3.13.18':
-    resolution: {integrity: sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A==}
-    peerDependencies:
-      react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
-      react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
-
   '@tanstack/table-core@8.21.3':
     resolution: {integrity: sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg==}
     engines: {node: '>=12'}
 
-  '@tanstack/virtual-core@3.13.18':
-    resolution: {integrity: sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==}
-
   '@testing-library/dom@10.4.1':
     resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==}
     engines: {node: '>=18'}
@@ -4397,10 +4373,6 @@ packages:
     resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
     engines: {node: '>=10'}
 
-  char-regex@1.0.2:
-    resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==}
-    engines: {node: '>=10'}
-
   character-entities-html4@2.1.0:
     resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==}
 
@@ -5018,9 +4990,6 @@ packages:
   emoji-regex@9.2.2:
     resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
 
-  emojilib@2.4.0:
-    resolution: {integrity: sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw==}
-
   emojis-list@3.0.0:
     resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==}
     engines: {node: '>= 4'}
@@ -6001,24 +5970,6 @@ packages:
     resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==}
     hasBin: true
 
-  jotai@2.17.1:
-    resolution: {integrity: sha512-TFNZZDa/0ewCLQyRC/Sq9crtixNj/Xdf/wmj9631xxMuKToVJZDbqcHIYN0OboH+7kh6P6tpIK7uKWClj86PKw==}
-    engines: {node: '>=12.20.0'}
-    peerDependencies:
-      '@babel/core': '>=7.0.0'
-      '@babel/template': '>=7.0.0'
-      '@types/react': '>=17.0.0'
-      react: '>=17.0.0'
-    peerDependenciesMeta:
-      '@babel/core':
-        optional: true
-      '@babel/template':
-        optional: true
-      '@types/react':
-        optional: true
-      react:
-        optional: true
-
   js-tokens@4.0.0:
     resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==}
 
@@ -6637,10 +6588,6 @@ packages:
   node-abort-controller@3.1.1:
     resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
 
-  node-emoji@2.2.0:
-    resolution: {integrity: sha512-Z3lTE9pLaJF47NyMhd4ww1yFTAP8YhYI8SleJiHzM46Fgpm5cnNzSl9XfzFNqbaz+VlJrIj3fXQ4DeN1Rjm6cw==}
-    engines: {node: '>=18'}
-
   node-fetch-h2@2.3.0:
     resolution: {integrity: sha512-ofRW94Ab0T4AOh5Fk8t0h8OBWrmjb0SSB20xh1H8YnPV9EJ+f5AMoYSUQ2zgJ4Iq2HAK0I2l5/Nequ8YzFS3Hg==}
     engines: {node: 4.x || >=6.0.0}
@@ -7739,10 +7686,6 @@ packages:
     resolution: {integrity: sha512-LH7FpTAkeD+y5xQC4fzS+tFtaNlvt3Ib1zKzvhjv/Y+cioV4zIuw4IZr2yhRLu67CWL7FR9/6KXKnjRoZTvGGQ==}
     engines: {node: '>=12'}
 
-  skin-tone@2.0.0:
-    resolution: {integrity: sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==}
-    engines: {node: '>=8'}
-
   slash@3.0.0:
     resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==}
     engines: {node: '>=8'}
@@ -8220,13 +8163,6 @@ packages:
     resolution: {integrity: sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==}
     engines: {node: '>=4'}
 
-  unicode-emoji-json@0.8.0:
-    resolution: {integrity: sha512-3wDXXvp6YGoKGhS2O2H7+V+bYduOBydN1lnI0uVfr1cIdY02uFFiEH1i3kE5CCE4l6UqbLKVmEFW9USxTAMD1g==}
-
-  unicode-emoji-modifier-base@1.0.0:
-    resolution: {integrity: sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==}
-    engines: {node: '>=4'}
-
   unicode-match-property-ecmascript@2.0.0:
     resolution: {integrity: sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==}
     engines: {node: '>=4'}
@@ -9836,22 +9772,6 @@ snapshots:
 
   '@faker-js/faker@10.0.0': {}
 
-  '@ferrucc-io/emoji-picker@0.0.48(@babel/core@7.28.5)(@babel/template@7.27.2)(@types/react@18.3.17)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tailwindcss@3.4.17)':
-    dependencies:
-      '@tanstack/react-virtual': 3.13.18(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
-      clsx: 2.1.1
-      jotai: 2.17.1(@babel/core@7.28.5)(@babel/template@7.27.2)(@types/react@18.3.17)(react@18.3.1)
-      node-emoji: 2.2.0
-      react: 18.3.1
-      react-dom: 18.3.1(react@18.3.1)
-      tailwind-merge: 2.6.0
-      tailwindcss: 3.4.17
-      unicode-emoji-json: 0.8.0
-    transitivePeerDependencies:
-      - '@babel/core'
-      - '@babel/template'
-      - '@types/react'
-
   '@floating-ui/core@1.7.3':
     dependencies:
       '@floating-ui/utils': 0.2.10
@@ -11613,8 +11533,6 @@ snapshots:
 
   '@shikijs/vscode-textmate@10.0.2': {}
 
-  '@sindresorhus/is@4.6.0': {}
-
   '@standard-schema/spec@1.0.0': {}
 
   '@standard-schema/spec@1.1.0': {}
@@ -12083,16 +12001,8 @@ snapshots:
       react: 18.3.1
       react-dom: 18.3.1(react@18.3.1)
 
-  '@tanstack/react-virtual@3.13.18(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
-    dependencies:
-      '@tanstack/virtual-core': 3.13.18
-      react: 18.3.1
-      react-dom: 18.3.1(react@18.3.1)
-
   '@tanstack/table-core@8.21.3': {}
 
-  '@tanstack/virtual-core@3.13.18': {}
-
   '@testing-library/dom@10.4.1':
     dependencies:
       '@babel/code-frame': 7.27.1
@@ -13184,8 +13094,6 @@ snapshots:
       ansi-styles: 4.3.0
       supports-color: 7.2.0
 
-  char-regex@1.0.2: {}
-
   character-entities-html4@2.1.0: {}
 
   character-entities-legacy@3.0.0: {}
@@ -13829,8 +13737,6 @@ snapshots:
 
   emoji-regex@9.2.2: {}
 
-  emojilib@2.4.0: {}
-
   emojis-list@3.0.0: {}
 
   endent@2.1.0:
@@ -15112,13 +15018,6 @@ snapshots:
 
   jiti@2.6.1: {}
 
-  jotai@2.17.1(@babel/core@7.28.5)(@babel/template@7.27.2)(@types/react@18.3.17)(react@18.3.1):
-    optionalDependencies:
-      '@babel/core': 7.28.5
-      '@babel/template': 7.27.2
-      '@types/react': 18.3.17
-      react: 18.3.1
-
   js-tokens@4.0.0: {}
 
   js-yaml@4.1.0:
@@ -15987,13 +15886,6 @@ snapshots:
 
   node-abort-controller@3.1.1: {}
 
-  node-emoji@2.2.0:
-    dependencies:
-      '@sindresorhus/is': 4.6.0
-      char-regex: 1.0.2
-      emojilib: 2.4.0
-      skin-tone: 2.0.0
-
   node-fetch-h2@2.3.0:
     dependencies:
       http2-client: 1.3.5
@@ -17294,10 +17186,6 @@ snapshots:
     dependencies:
       jsep: 1.4.0
 
-  skin-tone@2.0.0:
-    dependencies:
-      unicode-emoji-modifier-base: 1.0.0
-
   slash@3.0.0: {}
 
   sonner@2.0.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1):
@@ -17813,10 +17701,6 @@ snapshots:
 
   unicode-canonical-property-names-ecmascript@2.0.1: {}
 
-  unicode-emoji-json@0.8.0: {}
-
-  unicode-emoji-modifier-base@1.0.0: {}
-
   unicode-match-property-ecmascript@2.0.0:
     dependencies:
       unicode-canonical-property-names-ecmascript: 2.0.1

autogpt_platform/frontend/src/app/(platform)/auth/authorize/page.tsx

@@ -19,8 +19,6 @@ const SCOPE_DESCRIPTIONS: { [key in APIKeyPermission]: string } = {
   IDENTITY: "View your user ID, e-mail, and timezone",
   EXECUTE_GRAPH: "Run your agents",
   READ_GRAPH: "View your agents and their configurations",
-  WRITE_GRAPH: "Create agent graphs",
-  WRITE_LIBRARY: "Add agents to your library",
   EXECUTE_BLOCK: "Execute individual blocks",
   READ_BLOCK: "View available blocks",
   READ_STORE: "Access the Marketplace",

autogpt_platform/frontend/src/app/(platform)/build/BuilderContent.tsx

@@ -0,0 +1,13 @@
+"use client";
+import { ReactFlowProvider } from "@xyflow/react";
+import { Flow } from "./components/FlowEditor/Flow/Flow";
+
+export function BuilderContent() {
+  return (
+    <div className="relative h-full w-full">
+      <ReactFlowProvider>
+        <Flow />
+      </ReactFlowProvider>
+    </div>
+  );
+}

autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/edges/CustomEdge.tsx

@@ -63,19 +63,8 @@ const CustomEdge = ({
 
   return (
     <>
-      <path
-        d={edgePath}
-        fill="none"
-        stroke="black"
-        strokeOpacity={0}
-        strokeWidth={20}
-        className="react-flow__edge-interaction cursor-pointer"
-        onMouseEnter={() => setIsHovered(true)}
-        onMouseLeave={() => setIsHovered(false)}
-      />
       <BaseEdge
         path={edgePath}
-        interactionWidth={0}
         markerEnd={markerEnd}
         className={cn(
           isStatic && "!stroke-[1.5px] [stroke-dasharray:6]",

autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/CustomNode/components/NodeHeader.tsx

@@ -7,7 +7,7 @@ import {
   TooltipTrigger,
 } from "@/components/atoms/Tooltip/BaseTooltip";
 import { beautifyString, cn } from "@/lib/utils";
-import { useState } from "react";
+import { useCallback, useState } from "react";
 import { CustomNodeData } from "../CustomNode";
 import { NodeBadges } from "./NodeBadges";
 import { NodeContextMenu } from "./NodeContextMenu";
@@ -25,6 +25,9 @@ export const NodeHeader = ({ data, nodeId }: Props) => {
 
   const [isEditingTitle, setIsEditingTitle] = useState(false);
   const [editedTitle, setEditedTitle] = useState(title);
+  const titleInputRef = useCallback((node: HTMLInputElement | null) => {
+    node?.focus();
+  }, []);
 
   const handleTitleEdit = () => {
     updateNodeData(nodeId, {
@@ -52,10 +55,10 @@ export const NodeHeader = ({ data, nodeId }: Props) => {
           >
             {isEditingTitle ? (
               <input
+                ref={titleInputRef}
                 id="node-title-input"
                 value={editedTitle}
                 onChange={(e) => setEditedTitle(e.target.value)}
-                autoFocus
                 className={cn(
                   "m-0 h-fit w-full border-none bg-transparent p-0 focus:outline-none focus:ring-0",
                   "font-sans text-[1rem] font-semibold leading-[1.5rem] text-zinc-800",

autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/CustomNode/components/NodeOutput/NodeOutput.tsx

@@ -46,7 +46,7 @@ export const NodeDataRenderer = ({ nodeId }: { nodeId: string }) => {
               <div className="space-y-2">
                 <Text variant="small-medium">Input</Text>
 
-                <ContentRenderer value={latestInputData} shortContent={true} />
+                <ContentRenderer value={latestInputData} shortContent={false} />
 
                 <div className="mt-1 flex justify-end gap-1">
                   <NodeDataViewer
@@ -98,7 +98,7 @@ export const NodeDataRenderer = ({ nodeId }: { nodeId: string }) => {
                           Data:
                         </Text>
                         <div className="relative space-y-2">
-                          {value.slice(0, 3).map((item, index) => (
+                          {value.map((item, index) => (
                             <div key={index}>
                               <ContentRenderer
                                 value={item}

autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/CustomNode/components/NodeOutput/components/ContentRenderer.tsx

@@ -37,15 +37,15 @@ export const ContentRenderer: React.FC<{
     !shortContent
   ) {
     return (
-      <div className="overflow-hidden [&>*]:rounded-xlarge [&>*]:!text-xs [&_pre]:whitespace-pre-wrap [&_pre]:break-words">
+      <div className="[&>*]:rounded-xlarge [&>*]:!text-xs">
         {renderer?.render(value, metadata)}
       </div>
     );
   }
 
   return (
-    <div className="overflow-hidden [&>*]:rounded-xlarge [&>*]:!text-xs">
-      <TextRenderer value={value} truncateLengthLimit={200} />
+    <div className="[&>*]:rounded-xlarge [&>*]:!text-xs">
+      <TextRenderer value={value} truncateLengthLimit={100} />
     </div>
   );
 };

autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/CustomNode/components/NodeOutput/components/NodeDataViewer/NodeDataViewer.tsx

@@ -1,3 +1,4 @@
+import { ScrollArea } from "@/components/__legacy__/ui/scroll-area";
 import { Button } from "@/components/atoms/Button/Button";
 import { Text } from "@/components/atoms/Text/Text";
 import {
@@ -163,119 +164,129 @@ export const NodeDataViewer: FC<NodeDataViewerProps> = ({
           </div>
         </div>
 
-        <div className="flex-1">
-          <div className="my-4">
-            {shouldGroupExecutions ? (
-              <div className="space-y-4">
-                {groupedExecutions.map((execution) => (
-                  <div
-                    key={execution.execId}
-                    className="rounded-3xl border border-slate-200 bg-white p-4 shadow-sm"
-                  >
-                    <div className="flex items-center gap-2">
-                      <Text variant="body" className="text-slate-600">
-                        Execution ID:
-                      </Text>
-                      <Text
-                        variant="body-medium"
-                        className="rounded-full border border-gray-300 bg-gray-50 px-2 py-1 font-mono text-xs"
-                      >
-                        {execution.execId}
-                      </Text>
-                    </div>
-                    <div className="mt-2 space-y-4">
-                      {execution.outputItems.length > 0 ? (
-                        execution.outputItems.map((item, index) => (
-                          <div key={item.key} className="group">
-                            <OutputItem
-                              value={item.value}
-                              metadata={item.metadata}
-                              renderer={item.renderer}
-                            />
-                            <div className="mt-2 flex gap-3">
-                              <Button
-                                variant="secondary"
-                                className="min-w-0 p-1"
-                                size="icon"
-                                onClick={() =>
-                                  handleCopyGroupedItem(
-                                    execution.execId,
-                                    index,
-                                    item,
-                                  )
-                                }
-                                aria-label="Copy item"
-                              >
-                                {copiedKey ===
-                                `${execution.execId}-${index}` ? (
-                                  <CheckIcon className="size-4 text-green-600" />
-                                ) : (
-                                  <CopyIcon className="size-4 text-black" />
-                                )}
-                              </Button>
-                              <Button
-                                variant="secondary"
-                                size="icon"
-                                className="min-w-0 p-1"
-                                onClick={() => handleDownloadGroupedItem(item)}
-                                aria-label="Download item"
-                              >
-                                <DownloadIcon className="size-4 text-black" />
-                              </Button>
+        <div className="flex-1 overflow-hidden">
+          <ScrollArea className="h-full">
+            <div className="my-4">
+              {shouldGroupExecutions ? (
+                <div className="space-y-4">
+                  {groupedExecutions.map((execution) => (
+                    <div
+                      key={execution.execId}
+                      className="rounded-3xl border border-slate-200 bg-white p-4 shadow-sm"
+                    >
+                      <div className="flex items-center gap-2">
+                        <Text variant="body" className="text-slate-600">
+                          Execution ID:
+                        </Text>
+                        <Text
+                          variant="body-medium"
+                          className="rounded-full border border-gray-300 bg-gray-50 px-2 py-1 font-mono text-xs"
+                        >
+                          {execution.execId}
+                        </Text>
+                      </div>
+                      <div className="mt-2 space-y-4">
+                        {execution.outputItems.length > 0 ? (
+                          execution.outputItems.map((item, index) => (
+                            <div
+                              key={item.key}
+                              className="group flex items-start gap-4"
+                            >
+                              <div className="w-full flex-1">
+                                <OutputItem
+                                  value={item.value}
+                                  metadata={item.metadata}
+                                  renderer={item.renderer}
+                                />
+                              </div>
+
+                              <div className="flex w-fit gap-3">
+                                <Button
+                                  variant="secondary"
+                                  className="min-w-0 p-1"
+                                  size="icon"
+                                  onClick={() =>
+                                    handleCopyGroupedItem(
+                                      execution.execId,
+                                      index,
+                                      item,
+                                    )
+                                  }
+                                  aria-label="Copy item"
+                                >
+                                  {copiedKey ===
+                                  `${execution.execId}-${index}` ? (
+                                    <CheckIcon className="size-4 text-green-600" />
+                                  ) : (
+                                    <CopyIcon className="size-4 text-black" />
+                                  )}
+                                </Button>
+                                <Button
+                                  variant="secondary"
+                                  size="icon"
+                                  className="min-w-0 p-1"
+                                  onClick={() =>
+                                    handleDownloadGroupedItem(item)
+                                  }
+                                  aria-label="Download item"
+                                >
+                                  <DownloadIcon className="size-4 text-black" />
+                                </Button>
+                              </div>
                             </div>
-                          </div>
-                        ))
-                      ) : (
-                        <div className="py-4 text-center text-gray-500">
-                          No data available
-                        </div>
-                      )}
-                    </div>
-                  </div>
-                ))}
-              </div>
-            ) : dataArray.length > 0 ? (
-              <div className="space-y-4">
-                {outputItems.map((item, index) => (
-                  <div key={item.key} className="group">
-                    <OutputItem
-                      value={item.value}
-                      metadata={item.metadata}
-                      renderer={item.renderer}
-                    />
-                    <div className="mt-2 flex gap-3">
-                      <Button
-                        variant="secondary"
-                        className="min-w-0 p-1"
-                        size="icon"
-                        onClick={() => handleCopyItem(index)}
-                        aria-label="Copy item"
-                      >
-                        {copiedIndex === index ? (
-                          <CheckIcon className="size-4 text-green-600" />
+                          ))
                         ) : (
-                          <CopyIcon className="size-4 text-black" />
+                          <div className="py-4 text-center text-gray-500">
+                            No data available
+                          </div>
                         )}
-                      </Button>
-                      <Button
-                        variant="secondary"
-                        size="icon"
-                        className="min-w-0 p-1"
-                        onClick={() => handleDownloadItem(index)}
-                        aria-label="Download item"
-                      >
-                        <DownloadIcon className="size-4 text-black" />
-                      </Button>
+                      </div>
                     </div>
-                  </div>
-                ))}
-              </div>
-            ) : (
-              <div className="py-8 text-center text-gray-500">
-                No data available
-              </div>
-            )}
-          </div>
+                  ))}
+                </div>
+              ) : dataArray.length > 0 ? (
+                <div className="space-y-4">
+                  {outputItems.map((item, index) => (
+                    <div key={item.key} className="group relative">
+                      <OutputItem
+                        value={item.value}
+                        metadata={item.metadata}
+                        renderer={item.renderer}
+                      />
+                      <div className="absolute right-3 top-3 flex gap-3">
+                        <Button
+                          variant="secondary"
+                          className="min-w-0 p-1"
+                          size="icon"
+                          onClick={() => handleCopyItem(index)}
+                          aria-label="Copy item"
+                        >
+                          {copiedIndex === index ? (
+                            <CheckIcon className="size-4 text-green-600" />
+                          ) : (
+                            <CopyIcon className="size-4 text-black" />
+                          )}
+                        </Button>
+                        <Button
+                          variant="secondary"
+                          size="icon"
+                          className="min-w-0 p-1"
+                          onClick={() => handleDownloadItem(index)}
+                          aria-label="Download item"
+                        >
+                          <DownloadIcon className="size-4 text-black" />
+                        </Button>
+                      </div>
+                    </div>
+                  ))}
+                </div>
+              ) : (
+                <div className="py-8 text-center text-gray-500">
+                  No data available
+                </div>
+              )}
+            </div>
+          </ScrollArea>
         </div>
 
         <div className="flex justify-end pt-4">

autogpt_platform/frontend/src/app/(platform)/build/components/MCPToolDialog.tsx

@@ -300,7 +300,6 @@ export function MCPToolDialog({
                 value={serverUrl}
                 onChange={(e) => setServerUrl(e.target.value)}
                 onKeyDown={(e) => e.key === "Enter" && handleDiscoverTools()}
-                autoFocus
               />
             </div>
 
@@ -327,7 +326,6 @@ export function MCPToolDialog({
                   value={manualToken}
                   onChange={(e) => setManualToken(e.target.value)}
                   onKeyDown={(e) => e.key === "Enter" && handleDiscoverTools()}
-                  autoFocus
                 />
               </div>
             )}