Merge branch 'hackathon-copilot-search' of github.com:Significant-Gravitas/AutoGPT into feat/backfill_block_and_docs

- Changed from patching prisma.get_client() to patching query_raw_with_schema
- Follows the pattern used in hybrid_search_test.py
- Tests now properly exercise the schema-prefixing wrapper logic
- Fixes issue where SET search_path call was unmocked
- Removed unused mocker parameters
- All 18 tests passing

.branchlet.json

@@ -0,0 +1,37 @@
+{
+  "worktreeCopyPatterns": [
+    ".env*",
+    ".vscode/**",
+    ".auth/**",
+    ".claude/**",
+    "autogpt_platform/.env*",
+    "autogpt_platform/backend/.env*",
+    "autogpt_platform/frontend/.env*",
+    "autogpt_platform/frontend/.auth/**",
+    "autogpt_platform/db/docker/.env*"
+  ],
+  "worktreeCopyIgnores": [
+    "**/node_modules/**",
+    "**/dist/**",
+    "**/.git/**",
+    "**/Thumbs.db",
+    "**/.DS_Store",
+    "**/.next/**",
+    "**/__pycache__/**",
+    "**/.ruff_cache/**",
+    "**/.pytest_cache/**",
+    "**/*.pyc",
+    "**/playwright-report/**",
+    "**/logs/**",
+    "**/site/**"
+  ],
+  "worktreePathTemplate": "$BASE_PATH.worktree",
+  "postCreateCmd": [
+    "cd autogpt_platform/autogpt_libs && poetry install",
+    "cd autogpt_platform/backend && poetry install && poetry run prisma generate",
+    "cd autogpt_platform/frontend && pnpm install",
+    "cd docs && pip install -r requirements.txt"
+  ],
+  "terminalCommand": "code .",
+  "deleteBranchWithWorktree": false
+}

.dockerignore

@@ -16,6 +16,7 @@
 !autogpt_platform/backend/poetry.lock
 !autogpt_platform/backend/README.md
 !autogpt_platform/backend/.env
+!autogpt_platform/backend/gen_prisma_types_stub.py
 
 # Platform - Market
 !autogpt_platform/market/market/

.github/workflows/claude-dependabot.yml

@@ -74,7 +74,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate
+        run: poetry run prisma generate && poetry run gen-prisma-stub
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js

.github/workflows/claude.yml

@@ -90,7 +90,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate
+        run: poetry run prisma generate && poetry run gen-prisma-stub
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js

.github/workflows/copilot-setup-steps.yml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Generate Prisma Client
         working-directory: autogpt_platform/backend
-        run: poetry run prisma generate
+        run: poetry run prisma generate && poetry run gen-prisma-stub
 
       # Frontend Node.js/pnpm setup (mirrors platform-frontend-ci.yml)
       - name: Set up Node.js
@@ -108,6 +108,16 @@ jobs:
       #   run: pnpm playwright install --with-deps chromium
 
       # Docker setup for development environment
+      - name: Free up disk space
+        run: |
+          # Remove large unused tools to free disk space for Docker builds
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker system prune -af
+          df -h
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 

.github/workflows/platform-backend-ci.yml

@@ -134,7 +134,7 @@ jobs:
         run: poetry install
 
       - name: Generate Prisma Client
-        run: poetry run prisma generate
+        run: poetry run prisma generate && poetry run gen-prisma-stub
 
       - id: supabase
         name: Start Supabase
@@ -176,7 +176,7 @@ jobs:
           }
 
       - name: Run Database Migrations
-        run: poetry run prisma migrate dev --name updates
+        run: poetry run prisma migrate deploy
         env:
           DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
           DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}

autogpt_platform/Makefile

@@ -12,6 +12,7 @@ reset-db:
 	rm -rf db/docker/volumes/db/data
 	cd backend && poetry run prisma migrate deploy
 	cd backend && poetry run prisma generate
+	cd backend && poetry run gen-prisma-stub
 	
 # View logs for core services
 logs-core:
@@ -33,6 +34,7 @@ init-env:
 migrate:
 	cd backend && poetry run prisma migrate deploy
 	cd backend && poetry run prisma generate
+	cd backend && poetry run gen-prisma-stub
 
 run-backend:
 	cd backend && poetry run app

autogpt_platform/backend/.gitignore

@@ -18,3 +18,4 @@ load-tests/results/
 load-tests/*.json
 load-tests/*.log
 load-tests/node_modules/*
+migrations/*/rollback*.sql

autogpt_platform/backend/Dockerfile

@@ -48,7 +48,8 @@ RUN poetry install --no-ansi --no-root
 # Generate Prisma client
 COPY autogpt_platform/backend/schema.prisma ./
 COPY autogpt_platform/backend/backend/data/partial_types.py ./backend/data/partial_types.py
-RUN poetry run prisma generate
+COPY autogpt_platform/backend/gen_prisma_types_stub.py ./
+RUN poetry run prisma generate && poetry run gen-prisma-stub
 
 FROM debian:13-slim AS server_dependencies
 

autogpt_platform/backend/backend/api/conn_manager.py

@@ -122,24 +122,6 @@ class ConnectionManager:
 
         return len(connections)
 
-    async def broadcast_to_all(self, *, method: WSMethod, data: dict) -> int:
-        """Broadcast a message to all active websocket connections."""
-        message = WSMessage(
-            method=method,
-            data=data,
-        ).model_dump_json()
-
-        connections = tuple(self.active_connections)
-        if not connections:
-            return 0
-
-        await asyncio.gather(
-            *(connection.send_text(message) for connection in connections),
-            return_exceptions=True,
-        )
-
-        return len(connections)
-
     async def _subscribe(self, channel_key: str, websocket: WebSocket) -> str:
         if channel_key not in self.subscriptions:
             self.subscriptions[channel_key] = set()

autogpt_platform/backend/backend/api/features/admin/execution_analytics_routes.py

@@ -173,64 +173,30 @@ async def get_execution_analytics_config(
         # Return with provider prefix for clarity
         return f"{provider_name}: {model_name}"
 
-    # Get all models from the registry (dynamic, not hardcoded enum)
-    from backend.data import llm_registry
-    from backend.server.v2.llm import db as llm_db
-
-    # Get the recommended model from the database (configurable via admin UI)
-    recommended_model_slug = await llm_db.get_recommended_model_slug()
-
-    # Build the available models list
-    first_enabled_slug = None
-    for registry_model in llm_registry.iter_dynamic_models():
-        # Only include enabled models in the list
-        if not registry_model.is_enabled:
-            continue
-
-        # Track first enabled model as fallback
-        if first_enabled_slug is None:
-            first_enabled_slug = registry_model.slug
-
-        model_enum = LlmModel(registry_model.slug)  # Create enum instance from slug
-        label = generate_model_label(model_enum)
+    # Include all LlmModel values (no more filtering by hardcoded list)
+    recommended_model = LlmModel.GPT4O_MINI.value
+    for model in LlmModel:
+        label = generate_model_label(model)
         # Add "(Recommended)" suffix to the recommended model
-        if registry_model.slug == recommended_model_slug:
+        if model.value == recommended_model:
             label += " (Recommended)"
 
         available_models.append(
             ModelInfo(
-                value=registry_model.slug,
+                value=model.value,
                 label=label,
-                provider=registry_model.metadata.provider,
+                provider=model.provider,
             )
         )
 
     # Sort models by provider and name for better UX
     available_models.sort(key=lambda x: (x.provider, x.label))
 
-    # Handle case where no models are available
-    if not available_models:
-        logger.warning(
-            "No enabled LLM models found in registry. "
-            "Ensure models are configured and enabled in the LLM Registry."
-        )
-        # Provide a placeholder entry so admins see meaningful feedback
-        available_models.append(
-            ModelInfo(
-                value="",
-                label="No models available - configure in LLM Registry",
-                provider="none",
-            )
-        )
-
-    # Use the DB recommended model, or fallback to first enabled model
-    final_recommended = recommended_model_slug or first_enabled_slug or ""
-
     return ExecutionAnalyticsConfig(
         available_models=available_models,
         default_system_prompt=DEFAULT_SYSTEM_PROMPT,
         default_user_prompt=DEFAULT_USER_PROMPT,
-        recommended_model=final_recommended,
+        recommended_model=recommended_model,
     )
 
 

autogpt_platform/backend/backend/api/features/admin/llm_routes.py

@@ -1,554 +0,0 @@
-import logging
-
-import autogpt_libs.auth
-import fastapi
-
-from backend.data import llm_registry
-from backend.data.block_cost_config import refresh_llm_costs
-from backend.server.v2.llm import db as llm_db
-from backend.server.v2.llm import model as llm_model
-
-logger = logging.getLogger(__name__)
-
-router = fastapi.APIRouter(
-    prefix="/admin/llm",
-    tags=["llm", "admin"],
-    dependencies=[fastapi.Security(autogpt_libs.auth.requires_admin_user)],
-)
-
-
-async def _refresh_runtime_state() -> None:
-    """Refresh the LLM registry and clear all related caches to ensure real-time updates."""
-    logger.info("Refreshing LLM registry runtime state...")
-
-    # Refresh registry from database
-    await llm_registry.refresh_llm_registry()
-    refresh_llm_costs()
-
-    # Clear block schema caches so they're regenerated with updated model options
-    from backend.data.block import BlockSchema
-
-    BlockSchema.clear_all_schema_caches()
-    logger.info("Cleared all block schema caches")
-
-    # Clear the /blocks endpoint cache so frontend gets updated schemas
-    try:
-        from backend.api.features.v1 import _get_cached_blocks
-
-        _get_cached_blocks.cache_clear()
-        logger.info("Cleared /blocks endpoint cache")
-    except Exception as e:
-        logger.warning("Failed to clear /blocks cache: %s", e)
-
-    # Clear the v2 builder providers cache (if it exists)
-    try:
-        from backend.api.features.builder import db as builder_db
-
-        if hasattr(builder_db, "_get_all_providers"):
-            builder_db._get_all_providers.cache_clear()
-            logger.info("Cleared v2 builder providers cache")
-    except Exception as e:
-        logger.debug("Could not clear v2 builder cache: %s", e)
-
-    # Notify all executor services to refresh their registry cache
-    from backend.data.llm_registry import publish_registry_refresh_notification
-
-    await publish_registry_refresh_notification()
-    logger.info("Published registry refresh notification")
-
-
-@router.get(
-    "/providers",
-    summary="List LLM providers",
-    response_model=llm_model.LlmProvidersResponse,
-)
-async def list_llm_providers(include_models: bool = True):
-    providers = await llm_db.list_providers(include_models=include_models)
-    return llm_model.LlmProvidersResponse(providers=providers)
-
-
-@router.post(
-    "/providers",
-    summary="Create LLM provider",
-    response_model=llm_model.LlmProvider,
-)
-async def create_llm_provider(request: llm_model.UpsertLlmProviderRequest):
-    provider = await llm_db.upsert_provider(request=request)
-    await _refresh_runtime_state()
-    return provider
-
-
-@router.patch(
-    "/providers/{provider_id}",
-    summary="Update LLM provider",
-    response_model=llm_model.LlmProvider,
-)
-async def update_llm_provider(
-    provider_id: str,
-    request: llm_model.UpsertLlmProviderRequest,
-):
-    provider = await llm_db.upsert_provider(request=request, provider_id=provider_id)
-    await _refresh_runtime_state()
-    return provider
-
-
-@router.get(
-    "/models",
-    summary="List LLM models",
-    response_model=llm_model.LlmModelsResponse,
-)
-async def list_llm_models(provider_id: str | None = fastapi.Query(default=None)):
-    models = await llm_db.list_models(provider_id=provider_id)
-    return llm_model.LlmModelsResponse(models=models)
-
-
-@router.post(
-    "/models",
-    summary="Create LLM model",
-    response_model=llm_model.LlmModel,
-)
-async def create_llm_model(request: llm_model.CreateLlmModelRequest):
-    model = await llm_db.create_model(request=request)
-    await _refresh_runtime_state()
-    return model
-
-
-@router.patch(
-    "/models/{model_id}",
-    summary="Update LLM model",
-    response_model=llm_model.LlmModel,
-)
-async def update_llm_model(
-    model_id: str,
-    request: llm_model.UpdateLlmModelRequest,
-):
-    model = await llm_db.update_model(model_id=model_id, request=request)
-    await _refresh_runtime_state()
-    return model
-
-
-@router.patch(
-    "/models/{model_id}/toggle",
-    summary="Toggle LLM model availability",
-    response_model=llm_model.ToggleLlmModelResponse,
-)
-async def toggle_llm_model(
-    model_id: str,
-    request: llm_model.ToggleLlmModelRequest,
-):
-    """
-    Toggle a model's enabled status, optionally migrating workflows when disabling.
-
-    If disabling a model and `migrate_to_slug` is provided, all workflows using
-    this model will be migrated to the specified replacement model before disabling.
-    A migration record is created which can be reverted later using the revert endpoint.
-
-    Optional fields:
-    - `migration_reason`: Reason for the migration (e.g., "Provider outage")
-    - `custom_credit_cost`: Custom pricing override for billing during migration
-    """
-    try:
-        result = await llm_db.toggle_model(
-            model_id=model_id,
-            is_enabled=request.is_enabled,
-            migrate_to_slug=request.migrate_to_slug,
-            migration_reason=request.migration_reason,
-            custom_credit_cost=request.custom_credit_cost,
-        )
-        await _refresh_runtime_state()
-        if result.nodes_migrated > 0:
-            logger.info(
-                "Toggled model '%s' to %s and migrated %d nodes to '%s' (migration_id=%s)",
-                result.model.slug,
-                "enabled" if request.is_enabled else "disabled",
-                result.nodes_migrated,
-                result.migrated_to_slug,
-                result.migration_id,
-            )
-        return result
-    except ValueError as exc:
-        logger.warning("Model toggle validation failed: %s", exc)
-        raise fastapi.HTTPException(status_code=400, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to toggle LLM model %s: %s", model_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to toggle model availability",
-        ) from exc
-
-
-@router.get(
-    "/models/{model_id}/usage",
-    summary="Get model usage count",
-    response_model=llm_model.LlmModelUsageResponse,
-)
-async def get_llm_model_usage(model_id: str):
-    """Get the number of workflow nodes using this model."""
-    try:
-        return await llm_db.get_model_usage(model_id=model_id)
-    except ValueError as exc:
-        raise fastapi.HTTPException(status_code=404, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to get model usage %s: %s", model_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to get model usage",
-        ) from exc
-
-
-@router.delete(
-    "/models/{model_id}",
-    summary="Delete LLM model and migrate workflows",
-    response_model=llm_model.DeleteLlmModelResponse,
-)
-async def delete_llm_model(
-    model_id: str,
-    replacement_model_slug: str = fastapi.Query(
-        ..., description="Slug of the model to migrate existing workflows to"
-    ),
-):
-    """
-    Delete a model and automatically migrate all workflows using it to a replacement model.
-
-    This endpoint:
-    1. Validates the replacement model exists and is enabled
-    2. Counts how many workflow nodes use the model being deleted
-    3. Updates all AgentNode.constantInput->model fields to the replacement
-    4. Deletes the model record
-    5. Refreshes all caches and notifies executors
-
-    Example: DELETE /admin/llm/models/{id}?replacement_model_slug=gpt-4o
-    """
-    try:
-        result = await llm_db.delete_model(
-            model_id=model_id, replacement_model_slug=replacement_model_slug
-        )
-        await _refresh_runtime_state()
-        logger.info(
-            "Deleted model '%s' and migrated %d nodes to '%s'",
-            result.deleted_model_slug,
-            result.nodes_migrated,
-            result.replacement_model_slug,
-        )
-        return result
-    except ValueError as exc:
-        # Validation errors (model not found, replacement invalid, etc.)
-        logger.warning("Model deletion validation failed: %s", exc)
-        raise fastapi.HTTPException(status_code=400, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to delete LLM model %s: %s", model_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to delete model and migrate workflows",
-        ) from exc
-
-
-# ============================================================================
-# Migration Management Endpoints
-# ============================================================================
-
-
-@router.get(
-    "/migrations",
-    summary="List model migrations",
-    response_model=llm_model.LlmMigrationsResponse,
-)
-async def list_llm_migrations(
-    include_reverted: bool = fastapi.Query(
-        default=False, description="Include reverted migrations in the list"
-    ),
-):
-    """
-    List all model migrations.
-
-    Migrations are created when disabling a model with the migrate_to_slug option.
-    They can be reverted to restore the original model configuration.
-    """
-    try:
-        migrations = await llm_db.list_migrations(include_reverted=include_reverted)
-        return llm_model.LlmMigrationsResponse(migrations=migrations)
-    except Exception as exc:
-        logger.exception("Failed to list migrations: %s", exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to list migrations",
-        ) from exc
-
-
-@router.get(
-    "/migrations/{migration_id}",
-    summary="Get migration details",
-    response_model=llm_model.LlmModelMigration,
-)
-async def get_llm_migration(migration_id: str):
-    """Get details of a specific migration."""
-    try:
-        migration = await llm_db.get_migration(migration_id)
-        if not migration:
-            raise fastapi.HTTPException(
-                status_code=404, detail=f"Migration '{migration_id}' not found"
-            )
-        return migration
-    except fastapi.HTTPException:
-        raise
-    except Exception as exc:
-        logger.exception("Failed to get migration %s: %s", migration_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to get migration",
-        ) from exc
-
-
-@router.post(
-    "/migrations/{migration_id}/revert",
-    summary="Revert a model migration",
-    response_model=llm_model.RevertMigrationResponse,
-)
-async def revert_llm_migration(
-    migration_id: str,
-    request: llm_model.RevertMigrationRequest | None = None,
-):
-    """
-    Revert a model migration, restoring affected workflows to their original model.
-
-    This only reverts the specific nodes that were part of the migration.
-    The source model must exist for the revert to succeed.
-
-    Options:
-    - `re_enable_source_model`: Whether to re-enable the source model if disabled (default: True)
-
-    Response includes:
-    - `nodes_reverted`: Number of nodes successfully reverted
-    - `nodes_already_changed`: Number of nodes that were modified since migration (not reverted)
-    - `source_model_re_enabled`: Whether the source model was re-enabled
-
-    Requirements:
-    - Migration must not already be reverted
-    - Source model must exist
-    """
-    try:
-        re_enable = request.re_enable_source_model if request else True
-        result = await llm_db.revert_migration(
-            migration_id,
-            re_enable_source_model=re_enable,
-        )
-        await _refresh_runtime_state()
-        logger.info(
-            "Reverted migration '%s': %d nodes restored from '%s' to '%s' "
-            "(%d already changed, source re-enabled=%s)",
-            migration_id,
-            result.nodes_reverted,
-            result.target_model_slug,
-            result.source_model_slug,
-            result.nodes_already_changed,
-            result.source_model_re_enabled,
-        )
-        return result
-    except ValueError as exc:
-        logger.warning("Migration revert validation failed: %s", exc)
-        raise fastapi.HTTPException(status_code=400, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to revert migration %s: %s", migration_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to revert migration",
-        ) from exc
-
-
-# ============================================================================
-# Creator Management Endpoints
-# ============================================================================
-
-
-@router.get(
-    "/creators",
-    summary="List model creators",
-    response_model=llm_model.LlmCreatorsResponse,
-)
-async def list_llm_creators():
-    """
-    List all model creators.
-
-    Creators are organizations that create/train models (e.g., OpenAI, Meta, Anthropic).
-    This is distinct from providers who host/serve the models (e.g., OpenRouter).
-    """
-    try:
-        creators = await llm_db.list_creators()
-        return llm_model.LlmCreatorsResponse(creators=creators)
-    except Exception as exc:
-        logger.exception("Failed to list creators: %s", exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to list creators",
-        ) from exc
-
-
-@router.get(
-    "/creators/{creator_id}",
-    summary="Get creator details",
-    operation_id="getV2GetLlmCreatorDetails",
-    response_model=llm_model.LlmModelCreator,
-)
-async def get_llm_creator(creator_id: str):
-    """Get details of a specific model creator."""
-    try:
-        creator = await llm_db.get_creator(creator_id)
-        if not creator:
-            raise fastapi.HTTPException(
-                status_code=404, detail=f"Creator '{creator_id}' not found"
-            )
-        return creator
-    except fastapi.HTTPException:
-        raise
-    except Exception as exc:
-        logger.exception("Failed to get creator %s: %s", creator_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to get creator",
-        ) from exc
-
-
-@router.post(
-    "/creators",
-    summary="Create model creator",
-    response_model=llm_model.LlmModelCreator,
-)
-async def create_llm_creator(request: llm_model.UpsertLlmCreatorRequest):
-    """
-    Create a new model creator.
-
-    A creator represents an organization that creates/trains AI models,
-    such as OpenAI, Anthropic, Meta, or Google.
-    """
-    try:
-        creator = await llm_db.upsert_creator(request=request)
-        await _refresh_runtime_state()
-        logger.info("Created model creator '%s' (%s)", creator.display_name, creator.id)
-        return creator
-    except Exception as exc:
-        logger.exception("Failed to create creator: %s", exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to create creator",
-        ) from exc
-
-
-@router.patch(
-    "/creators/{creator_id}",
-    summary="Update model creator",
-    response_model=llm_model.LlmModelCreator,
-)
-async def update_llm_creator(
-    creator_id: str,
-    request: llm_model.UpsertLlmCreatorRequest,
-):
-    """Update an existing model creator."""
-    try:
-        creator = await llm_db.upsert_creator(request=request, creator_id=creator_id)
-        await _refresh_runtime_state()
-        logger.info("Updated model creator '%s' (%s)", creator.display_name, creator_id)
-        return creator
-    except Exception as exc:
-        logger.exception("Failed to update creator %s: %s", creator_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to update creator",
-        ) from exc
-
-
-@router.delete(
-    "/creators/{creator_id}",
-    summary="Delete model creator",
-    response_model=dict,
-)
-async def delete_llm_creator(creator_id: str):
-    """
-    Delete a model creator.
-
-    This will remove the creator association from all models that reference it
-    (sets creatorId to NULL), but will not delete the models themselves.
-    """
-    try:
-        await llm_db.delete_creator(creator_id)
-        await _refresh_runtime_state()
-        logger.info("Deleted model creator '%s'", creator_id)
-        return {"success": True, "message": f"Creator '{creator_id}' deleted"}
-    except ValueError as exc:
-        logger.warning("Creator deletion validation failed: %s", exc)
-        raise fastapi.HTTPException(status_code=404, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to delete creator %s: %s", creator_id, exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to delete creator",
-        ) from exc
-
-
-# ============================================================================
-# Recommended Model Endpoints
-# ============================================================================
-
-
-@router.get(
-    "/recommended-model",
-    summary="Get recommended model",
-    response_model=llm_model.RecommendedModelResponse,
-)
-async def get_recommended_model():
-    """
-    Get the currently recommended LLM model.
-
-    The recommended model is shown to users as the default/suggested option
-    in model selection dropdowns.
-    """
-    try:
-        model = await llm_db.get_recommended_model()
-        return llm_model.RecommendedModelResponse(
-            model=model,
-            slug=model.slug if model else None,
-        )
-    except Exception as exc:
-        logger.exception("Failed to get recommended model: %s", exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to get recommended model",
-        ) from exc
-
-
-@router.post(
-    "/recommended-model",
-    summary="Set recommended model",
-    response_model=llm_model.SetRecommendedModelResponse,
-)
-async def set_recommended_model(request: llm_model.SetRecommendedModelRequest):
-    """
-    Set a model as the recommended model.
-
-    This clears the recommended flag from any other model and sets it on
-    the specified model. The model must be enabled to be set as recommended.
-
-    The recommended model is displayed to users as the default/suggested
-    option in model selection dropdowns throughout the platform.
-    """
-    try:
-        model, previous_slug = await llm_db.set_recommended_model(request.model_id)
-        await _refresh_runtime_state()
-        logger.info(
-            "Set recommended model to '%s' (previous: %s)",
-            model.slug,
-            previous_slug or "none",
-        )
-        return llm_model.SetRecommendedModelResponse(
-            model=model,
-            previous_recommended_slug=previous_slug,
-            message=f"Model '{model.display_name}' is now the recommended model",
-        )
-    except ValueError as exc:
-        logger.warning("Set recommended model validation failed: %s", exc)
-        raise fastapi.HTTPException(status_code=400, detail=str(exc)) from exc
-    except Exception as exc:
-        logger.exception("Failed to set recommended model: %s", exc)
-        raise fastapi.HTTPException(
-            status_code=500,
-            detail="Failed to set recommended model",
-        ) from exc

autogpt_platform/backend/backend/api/features/admin/llm_routes_test.py

@@ -1,405 +0,0 @@
-from unittest.mock import AsyncMock
-
-import fastapi
-import fastapi.testclient
-import pytest
-import pytest_mock
-from autogpt_libs.auth.jwt_utils import get_jwt_payload
-from pytest_snapshot.plugin import Snapshot
-
-import backend.api.features.admin.llm_routes as llm_routes
-
-app = fastapi.FastAPI()
-app.include_router(llm_routes.router)
-
-client = fastapi.testclient.TestClient(app)
-
-
-@pytest.fixture(autouse=True)
-def setup_app_admin_auth(mock_jwt_admin):
-    """Setup admin auth overrides for all tests in this module"""
-    app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]
-    yield
-    app.dependency_overrides.clear()
-
-
-def test_list_llm_providers_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful listing of LLM providers"""
-    # Mock the database function
-    mock_providers = [
-        {
-            "id": "provider-1",
-            "name": "openai",
-            "display_name": "OpenAI",
-            "description": "OpenAI LLM provider",
-            "supports_tools": True,
-            "supports_json_output": True,
-            "supports_reasoning": False,
-            "supports_parallel_tool": True,
-            "metadata": {},
-            "models": [],
-        },
-        {
-            "id": "provider-2",
-            "name": "anthropic",
-            "display_name": "Anthropic",
-            "description": "Anthropic LLM provider",
-            "supports_tools": True,
-            "supports_json_output": True,
-            "supports_reasoning": False,
-            "supports_parallel_tool": True,
-            "metadata": {},
-            "models": [],
-        },
-    ]
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.list_providers",
-        new=AsyncMock(return_value=mock_providers),
-    )
-
-    response = client.get("/admin/llm/providers")
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert len(response_data["providers"]) == 2
-    assert response_data["providers"][0]["name"] == "openai"
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "list_llm_providers_success.json")
-
-
-def test_list_llm_models_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful listing of LLM models"""
-    # Mock the database function
-    mock_models = [
-        {
-            "id": "model-1",
-            "slug": "gpt-4o",
-            "display_name": "GPT-4o",
-            "description": "GPT-4 Optimized",
-            "provider_id": "provider-1",
-            "context_window": 128000,
-            "max_output_tokens": 16384,
-            "is_enabled": True,
-            "capabilities": {},
-            "metadata": {},
-            "costs": [
-                {
-                    "id": "cost-1",
-                    "credit_cost": 10,
-                    "credential_provider": "openai",
-                    "metadata": {},
-                }
-            ],
-        }
-    ]
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.list_models",
-        new=AsyncMock(return_value=mock_models),
-    )
-
-    response = client.get("/admin/llm/models")
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert len(response_data["models"]) == 1
-    assert response_data["models"][0]["slug"] == "gpt-4o"
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "list_llm_models_success.json")
-
-
-def test_create_llm_provider_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful creation of LLM provider"""
-    mock_provider = {
-        "id": "new-provider-id",
-        "name": "groq",
-        "display_name": "Groq",
-        "description": "Groq LLM provider",
-        "supports_tools": True,
-        "supports_json_output": True,
-        "supports_reasoning": False,
-        "supports_parallel_tool": False,
-        "metadata": {},
-    }
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.upsert_provider",
-        new=AsyncMock(return_value=mock_provider),
-    )
-
-    mock_refresh = mocker.patch(
-        "backend.api.features.admin.llm_routes._refresh_runtime_state",
-        new=AsyncMock(),
-    )
-
-    request_data = {
-        "name": "groq",
-        "display_name": "Groq",
-        "description": "Groq LLM provider",
-        "supports_tools": True,
-        "supports_json_output": True,
-        "supports_reasoning": False,
-        "supports_parallel_tool": False,
-        "metadata": {},
-    }
-
-    response = client.post("/admin/llm/providers", json=request_data)
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert response_data["name"] == "groq"
-    assert response_data["display_name"] == "Groq"
-
-    # Verify refresh was called
-    mock_refresh.assert_called_once()
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "create_llm_provider_success.json")
-
-
-def test_create_llm_model_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful creation of LLM model"""
-    mock_model = {
-        "id": "new-model-id",
-        "slug": "gpt-4.1-mini",
-        "display_name": "GPT-4.1 Mini",
-        "description": "Latest GPT-4.1 Mini model",
-        "provider_id": "provider-1",
-        "context_window": 128000,
-        "max_output_tokens": 16384,
-        "is_enabled": True,
-        "capabilities": {},
-        "metadata": {},
-        "costs": [
-            {
-                "id": "cost-id",
-                "credit_cost": 5,
-                "credential_provider": "openai",
-                "metadata": {},
-            }
-        ],
-    }
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.create_model",
-        new=AsyncMock(return_value=mock_model),
-    )
-
-    mock_refresh = mocker.patch(
-        "backend.api.features.admin.llm_routes._refresh_runtime_state",
-        new=AsyncMock(),
-    )
-
-    request_data = {
-        "slug": "gpt-4.1-mini",
-        "display_name": "GPT-4.1 Mini",
-        "description": "Latest GPT-4.1 Mini model",
-        "provider_id": "provider-1",
-        "context_window": 128000,
-        "max_output_tokens": 16384,
-        "is_enabled": True,
-        "capabilities": {},
-        "metadata": {},
-        "costs": [
-            {
-                "credit_cost": 5,
-                "credential_provider": "openai",
-                "metadata": {},
-            }
-        ],
-    }
-
-    response = client.post("/admin/llm/models", json=request_data)
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert response_data["slug"] == "gpt-4.1-mini"
-    assert response_data["is_enabled"] is True
-
-    # Verify refresh was called
-    mock_refresh.assert_called_once()
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "create_llm_model_success.json")
-
-
-def test_update_llm_model_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful update of LLM model"""
-    mock_model = {
-        "id": "model-1",
-        "slug": "gpt-4o",
-        "display_name": "GPT-4o Updated",
-        "description": "Updated description",
-        "provider_id": "provider-1",
-        "context_window": 256000,
-        "max_output_tokens": 32768,
-        "is_enabled": True,
-        "capabilities": {},
-        "metadata": {},
-        "costs": [
-            {
-                "id": "cost-1",
-                "credit_cost": 15,
-                "credential_provider": "openai",
-                "metadata": {},
-            }
-        ],
-    }
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.update_model",
-        new=AsyncMock(return_value=mock_model),
-    )
-
-    mock_refresh = mocker.patch(
-        "backend.api.features.admin.llm_routes._refresh_runtime_state",
-        new=AsyncMock(),
-    )
-
-    request_data = {
-        "display_name": "GPT-4o Updated",
-        "description": "Updated description",
-        "context_window": 256000,
-        "max_output_tokens": 32768,
-    }
-
-    response = client.patch("/admin/llm/models/model-1", json=request_data)
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert response_data["display_name"] == "GPT-4o Updated"
-    assert response_data["context_window"] == 256000
-
-    # Verify refresh was called
-    mock_refresh.assert_called_once()
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "update_llm_model_success.json")
-
-
-def test_toggle_llm_model_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful toggling of LLM model enabled status"""
-    mock_model = {
-        "id": "model-1",
-        "slug": "gpt-4o",
-        "display_name": "GPT-4o",
-        "description": "GPT-4 Optimized",
-        "provider_id": "provider-1",
-        "context_window": 128000,
-        "max_output_tokens": 16384,
-        "is_enabled": False,
-        "capabilities": {},
-        "metadata": {},
-        "costs": [],
-    }
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.toggle_model",
-        new=AsyncMock(return_value=mock_model),
-    )
-
-    mock_refresh = mocker.patch(
-        "backend.api.features.admin.llm_routes._refresh_runtime_state",
-        new=AsyncMock(),
-    )
-
-    request_data = {"is_enabled": False}
-
-    response = client.patch("/admin/llm/models/model-1/toggle", json=request_data)
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert response_data["is_enabled"] is False
-
-    # Verify refresh was called
-    mock_refresh.assert_called_once()
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "toggle_llm_model_success.json")
-
-
-def test_delete_llm_model_success(
-    mocker: pytest_mock.MockFixture,
-    configured_snapshot: Snapshot,
-) -> None:
-    """Test successful deletion of LLM model with migration"""
-    mock_response = {
-        "deleted_model_slug": "gpt-3.5-turbo",
-        "deleted_model_display_name": "GPT-3.5 Turbo",
-        "replacement_model_slug": "gpt-4o-mini",
-        "nodes_migrated": 42,
-        "message": "Successfully deleted model 'GPT-3.5 Turbo' (gpt-3.5-turbo) "
-        "and migrated 42 workflow node(s) to 'gpt-4o-mini'.",
-    }
-
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.delete_model",
-        new=AsyncMock(return_value=type("obj", (object,), mock_response)()),
-    )
-
-    mock_refresh = mocker.patch(
-        "backend.api.features.admin.llm_routes._refresh_runtime_state",
-        new=AsyncMock(),
-    )
-
-    response = client.delete(
-        "/admin/llm/models/model-1?replacement_model_slug=gpt-4o-mini"
-    )
-
-    assert response.status_code == 200
-    response_data = response.json()
-    assert response_data["deleted_model_slug"] == "gpt-3.5-turbo"
-    assert response_data["nodes_migrated"] == 42
-    assert response_data["replacement_model_slug"] == "gpt-4o-mini"
-
-    # Verify refresh was called
-    mock_refresh.assert_called_once()
-
-    # Snapshot test the response
-    configured_snapshot.assert_match(response_data, "delete_llm_model_success.json")
-
-
-def test_delete_llm_model_validation_error(
-    mocker: pytest_mock.MockFixture,
-) -> None:
-    """Test deletion fails with proper error when validation fails"""
-    mocker.patch(
-        "backend.api.features.admin.llm_routes.llm_db.delete_model",
-        new=AsyncMock(side_effect=ValueError("Replacement model 'invalid' not found")),
-    )
-
-    response = client.delete("/admin/llm/models/model-1?replacement_model_slug=invalid")
-
-    assert response.status_code == 400
-    assert "Replacement model 'invalid' not found" in response.json()["detail"]
-
-
-def test_delete_llm_model_missing_replacement(
-    mocker: pytest_mock.MockFixture,
-) -> None:
-    """Test deletion fails when replacement_model_slug is not provided"""
-    response = client.delete("/admin/llm/models/model-1")
-
-    # FastAPI will return 422 for missing required query params
-    assert response.status_code == 422

autogpt_platform/backend/backend/api/features/builder/db.py

@@ -15,7 +15,6 @@ from backend.blocks import load_all_blocks
 from backend.blocks.llm import LlmModel
 from backend.data.block import AnyBlockSchema, BlockCategory, BlockInfo, BlockSchema
 from backend.data.db import query_raw_with_schema
-from backend.data.llm_registry import get_all_model_slugs_for_validation
 from backend.integrations.providers import ProviderName
 from backend.util.cache import cached
 from backend.util.models import Pagination
@@ -32,14 +31,7 @@ from .model import (
 )
 
 logger = logging.getLogger(__name__)
-
-
-def _get_llm_models() -> list[str]:
-    """Get LLM model names for search matching from the registry."""
-    return [
-        slug.lower().replace("-", " ") for slug in get_all_model_slugs_for_validation()
-    ]
-
+llm_models = [name.name.lower().replace("_", " ") for name in LlmModel]
 
 MAX_LIBRARY_AGENT_RESULTS = 100
 MAX_MARKETPLACE_AGENT_RESULTS = 100
@@ -504,8 +496,8 @@ async def _get_static_counts():
 def _matches_llm_model(schema_cls: type[BlockSchema], query: str) -> bool:
     for field in schema_cls.model_fields.values():
         if field.annotation == LlmModel:
-            # Check if query matches any value in llm_models from registry
-            if any(query in name for name in _get_llm_models()):
+            # Check if query matches any value in llm_models
+            if any(query in name for name in llm_models):
                 return True
     return False
 

autogpt_platform/backend/backend/api/features/chat/tools/run_agent_test.py

@@ -1,4 +1,5 @@
 import uuid
+from unittest.mock import AsyncMock, patch
 
 import orjson
 import pytest
@@ -17,6 +18,17 @@ setup_test_data = setup_test_data
 setup_firecrawl_test_data = setup_firecrawl_test_data
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 @pytest.mark.asyncio(scope="session")
 async def test_run_agent(setup_test_data):
     """Test that the run_agent tool successfully executes an approved agent"""

autogpt_platform/backend/backend/api/features/library/db.py

@@ -489,7 +489,7 @@ async def update_agent_version_in_library(
     agent_graph_version: int,
 ) -> library_model.LibraryAgent:
     """
-    Updates the agent version in the library if useGraphIsActiveVersion is True.
+    Updates the agent version in the library for any agent owned by the user.
 
     Args:
         user_id: Owner of the LibraryAgent.
@@ -498,20 +498,31 @@ async def update_agent_version_in_library(
 
     Raises:
         DatabaseError: If there's an error with the update.
+        NotFoundError: If no library agent is found for this user and agent.
     """
     logger.debug(
         f"Updating agent version in library for user #{user_id}, "
         f"agent #{agent_graph_id} v{agent_graph_version}"
     )
-    try:
-        library_agent = await prisma.models.LibraryAgent.prisma().find_first_or_raise(
+    async with transaction() as tx:
+        library_agent = await prisma.models.LibraryAgent.prisma(tx).find_first_or_raise(
             where={
                 "userId": user_id,
                 "agentGraphId": agent_graph_id,
-                "useGraphIsActiveVersion": True,
             },
         )
-        lib = await prisma.models.LibraryAgent.prisma().update(
+
+        # Delete any conflicting LibraryAgent for the target version
+        await prisma.models.LibraryAgent.prisma(tx).delete_many(
+            where={
+                "userId": user_id,
+                "agentGraphId": agent_graph_id,
+                "agentGraphVersion": agent_graph_version,
+                "id": {"not": library_agent.id},
+            }
+        )
+
+        lib = await prisma.models.LibraryAgent.prisma(tx).update(
             where={"id": library_agent.id},
             data={
                 "AgentGraph": {
@@ -525,13 +536,13 @@ async def update_agent_version_in_library(
             },
             include={"AgentGraph": True},
         )
-        if lib is None:
-            raise NotFoundError(f"Library agent {library_agent.id} not found")
 
-        return library_model.LibraryAgent.from_db(lib)
-    except prisma.errors.PrismaError as e:
-        logger.error(f"Database error updating agent version in library: {e}")
-        raise DatabaseError("Failed to update agent version in library") from e
+    if lib is None:
+        raise NotFoundError(
+            f"Failed to update library agent for {agent_graph_id} v{agent_graph_version}"
+        )
+
+    return library_model.LibraryAgent.from_db(lib)
 
 
 async def update_library_agent(
@@ -825,6 +836,7 @@ async def add_store_agent_to_library(
                     }
                 },
                 "isCreatedByUser": False,
+                "useGraphIsActiveVersion": False,
                 "settings": SafeJson(
                     _initialize_graph_settings(graph_model).model_dump()
                 ),

autogpt_platform/backend/backend/api/features/library/model.py

@@ -48,6 +48,7 @@ class LibraryAgent(pydantic.BaseModel):
     id: str
     graph_id: str
     graph_version: int
+    owner_user_id: str  # ID of user who owns/created this agent graph
 
     image_url: str | None
 
@@ -163,6 +164,7 @@ class LibraryAgent(pydantic.BaseModel):
             id=agent.id,
             graph_id=agent.agentGraphId,
             graph_version=agent.agentGraphVersion,
+            owner_user_id=agent.userId,
             image_url=agent.imageUrl,
             creator_name=creator_name,
             creator_image_url=creator_image_url,

autogpt_platform/backend/backend/api/features/library/routes_test.py

@@ -42,6 +42,7 @@ async def test_get_library_agents_success(
                 id="test-agent-1",
                 graph_id="test-agent-1",
                 graph_version=1,
+                owner_user_id=test_user_id,
                 name="Test Agent 1",
                 description="Test Description 1",
                 image_url=None,
@@ -64,6 +65,7 @@ async def test_get_library_agents_success(
                 id="test-agent-2",
                 graph_id="test-agent-2",
                 graph_version=1,
+                owner_user_id=test_user_id,
                 name="Test Agent 2",
                 description="Test Description 2",
                 image_url=None,
@@ -138,6 +140,7 @@ async def test_get_favorite_library_agents_success(
                 id="test-agent-1",
                 graph_id="test-agent-1",
                 graph_version=1,
+                owner_user_id=test_user_id,
                 name="Favorite Agent 1",
                 description="Test Favorite Description 1",
                 image_url=None,
@@ -205,6 +208,7 @@ def test_add_agent_to_library_success(
         id="test-library-agent-id",
         graph_id="test-agent-1",
         graph_version=1,
+        owner_user_id=test_user_id,
         name="Test Agent 1",
         description="Test Description 1",
         image_url=None,

autogpt_platform/backend/backend/api/features/store/content_handlers.py

@@ -0,0 +1,417 @@
+"""
+Content Type Handlers for Unified Embeddings
+
+Pluggable system for different content sources (store agents, blocks, docs).
+Each handler knows how to fetch and process its content type for embedding.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from prisma.enums import ContentType
+
+from backend.data.db import query_raw_with_schema
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ContentItem:
+    """Represents a piece of content to be embedded."""
+
+    content_id: str  # Unique identifier (DB ID or file path)
+    content_type: ContentType
+    searchable_text: str  # Combined text for embedding
+    metadata: dict[str, Any]  # Content-specific metadata
+    user_id: str | None = None  # For user-scoped content
+
+
+class ContentHandler(ABC):
+    """Base handler for fetching and processing content for embeddings."""
+
+    @property
+    @abstractmethod
+    def content_type(self) -> ContentType:
+        """The ContentType this handler manages."""
+        pass
+
+    @abstractmethod
+    async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+        """
+        Fetch items that don't have embeddings yet.
+
+        Args:
+            batch_size: Maximum number of items to return
+
+        Returns:
+            List of ContentItem objects ready for embedding
+        """
+        pass
+
+    @abstractmethod
+    async def get_stats(self) -> dict[str, int]:
+        """
+        Get statistics about embedding coverage.
+
+        Returns:
+            Dict with keys: total, with_embeddings, without_embeddings
+        """
+        pass
+
+
+class StoreAgentHandler(ContentHandler):
+    """Handler for marketplace store agent listings."""
+
+    @property
+    def content_type(self) -> ContentType:
+        return ContentType.STORE_AGENT
+
+    async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+        """Fetch approved store listings without embeddings."""
+        from backend.api.features.store.embeddings import build_searchable_text
+
+        missing = await query_raw_with_schema(
+            """
+            SELECT
+                slv.id,
+                slv.name,
+                slv.description,
+                slv."subHeading",
+                slv.categories
+            FROM {schema_prefix}"StoreListingVersion" slv
+            LEFT JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+                ON slv.id = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{schema_prefix}"ContentType"
+            WHERE slv."submissionStatus" = 'APPROVED'
+            AND slv."isDeleted" = false
+            AND uce."contentId" IS NULL
+            LIMIT $1
+            """,
+            batch_size,
+        )
+
+        return [
+            ContentItem(
+                content_id=row["id"],
+                content_type=ContentType.STORE_AGENT,
+                searchable_text=build_searchable_text(
+                    name=row["name"],
+                    description=row["description"],
+                    sub_heading=row["subHeading"],
+                    categories=row["categories"] or [],
+                ),
+                metadata={
+                    "name": row["name"],
+                    "categories": row["categories"] or [],
+                },
+                user_id=None,  # Store agents are public
+            )
+            for row in missing
+        ]
+
+    async def get_stats(self) -> dict[str, int]:
+        """Get statistics about store agent embedding coverage."""
+        # Count approved versions
+        approved_result = await query_raw_with_schema(
+            """
+            SELECT COUNT(*) as count
+            FROM {schema_prefix}"StoreListingVersion"
+            WHERE "submissionStatus" = 'APPROVED'
+            AND "isDeleted" = false
+            """
+        )
+        total_approved = approved_result[0]["count"] if approved_result else 0
+
+        # Count versions with embeddings
+        embedded_result = await query_raw_with_schema(
+            """
+            SELECT COUNT(*) as count
+            FROM {schema_prefix}"StoreListingVersion" slv
+            JOIN {schema_prefix}"UnifiedContentEmbedding" uce ON slv.id = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{schema_prefix}"ContentType"
+            WHERE slv."submissionStatus" = 'APPROVED'
+            AND slv."isDeleted" = false
+            """
+        )
+        with_embeddings = embedded_result[0]["count"] if embedded_result else 0
+
+        return {
+            "total": total_approved,
+            "with_embeddings": with_embeddings,
+            "without_embeddings": total_approved - with_embeddings,
+        }
+
+
+class BlockHandler(ContentHandler):
+    """Handler for block definitions (Python classes)."""
+
+    @property
+    def content_type(self) -> ContentType:
+        return ContentType.BLOCK
+
+    async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+        """Fetch blocks without embeddings."""
+        from backend.data.block import get_blocks
+
+        # Get all available blocks
+        all_blocks = get_blocks()
+
+        # Check which ones have embeddings
+        if not all_blocks:
+            return []
+
+        block_ids = list(all_blocks.keys())
+
+        # Query for existing embeddings
+        placeholders = ",".join([f"${i+1}" for i in range(len(block_ids))])
+        existing_result = await query_raw_with_schema(
+            f"""
+            SELECT "contentId"
+            FROM {{schema_prefix}}"UnifiedContentEmbedding"
+            WHERE "contentType" = 'BLOCK'::{{schema_prefix}}"ContentType"
+            AND "contentId" = ANY(ARRAY[{placeholders}])
+            """,
+            *block_ids,
+        )
+
+        existing_ids = {row["contentId"] for row in existing_result}
+        missing_blocks = [
+            (block_id, block_cls)
+            for block_id, block_cls in all_blocks.items()
+            if block_id not in existing_ids
+        ]
+
+        # Convert to ContentItem
+        items = []
+        for block_id, block_cls in missing_blocks[:batch_size]:
+            try:
+                block_instance = block_cls()
+
+                # Build searchable text from block metadata
+                parts = []
+                if hasattr(block_instance, "name") and block_instance.name:
+                    parts.append(block_instance.name)
+                if (
+                    hasattr(block_instance, "description")
+                    and block_instance.description
+                ):
+                    parts.append(block_instance.description)
+                if hasattr(block_instance, "categories") and block_instance.categories:
+                    # Convert BlockCategory enum to strings
+                    parts.append(
+                        " ".join(str(cat.value) for cat in block_instance.categories)
+                    )
+
+                # Add input/output schema info
+                if hasattr(block_instance, "input_schema"):
+                    schema = block_instance.input_schema
+                    if hasattr(schema, "model_json_schema"):
+                        schema_dict = schema.model_json_schema()
+                        if "properties" in schema_dict:
+                            for prop_name, prop_info in schema_dict[
+                                "properties"
+                            ].items():
+                                if "description" in prop_info:
+                                    parts.append(
+                                        f"{prop_name}: {prop_info['description']}"
+                                    )
+
+                searchable_text = " ".join(parts)
+
+                items.append(
+                    ContentItem(
+                        content_id=block_id,
+                        content_type=ContentType.BLOCK,
+                        searchable_text=searchable_text,
+                        metadata={
+                            "name": getattr(block_instance, "name", ""),
+                            "categories": getattr(block_instance, "categories", []),
+                        },
+                        user_id=None,  # Blocks are public
+                    )
+                )
+            except Exception as e:
+                logger.warning(f"Failed to process block {block_id}: {e}")
+                continue
+
+        return items
+
+    async def get_stats(self) -> dict[str, int]:
+        """Get statistics about block embedding coverage."""
+        from backend.data.block import get_blocks
+
+        all_blocks = get_blocks()
+        total_blocks = len(all_blocks)
+
+        if total_blocks == 0:
+            return {"total": 0, "with_embeddings": 0, "without_embeddings": 0}
+
+        block_ids = list(all_blocks.keys())
+        placeholders = ",".join([f"${i+1}" for i in range(len(block_ids))])
+
+        embedded_result = await query_raw_with_schema(
+            f"""
+            SELECT COUNT(*) as count
+            FROM {{schema_prefix}}"UnifiedContentEmbedding"
+            WHERE "contentType" = 'BLOCK'::{{schema_prefix}}"ContentType"
+            AND "contentId" = ANY(ARRAY[{placeholders}])
+            """,
+            *block_ids,
+        )
+
+        with_embeddings = embedded_result[0]["count"] if embedded_result else 0
+
+        return {
+            "total": total_blocks,
+            "with_embeddings": with_embeddings,
+            "without_embeddings": total_blocks - with_embeddings,
+        }
+
+
+class DocumentationHandler(ContentHandler):
+    """Handler for documentation files (.md/.mdx)."""
+
+    @property
+    def content_type(self) -> ContentType:
+        return ContentType.DOCUMENTATION
+
+    def _get_docs_root(self) -> Path:
+        """Get the documentation root directory."""
+        # Assuming docs are in /docs relative to project root
+        backend_root = Path(__file__).parent.parent.parent.parent
+        docs_root = backend_root.parent.parent / "docs"
+        return docs_root
+
+    def _extract_title_and_content(self, file_path: Path) -> tuple[str, str]:
+        """Extract title and content from markdown file."""
+        try:
+            content = file_path.read_text(encoding="utf-8")
+
+            # Try to extract title from first # heading
+            lines = content.split("\n")
+            title = ""
+            body_lines = []
+
+            for line in lines:
+                if line.startswith("# ") and not title:
+                    title = line[2:].strip()
+                else:
+                    body_lines.append(line)
+
+            # If no title found, use filename
+            if not title:
+                title = file_path.stem.replace("-", " ").replace("_", " ").title()
+
+            body = "\n".join(body_lines)
+
+            return title, body
+        except Exception as e:
+            logger.warning(f"Failed to read {file_path}: {e}")
+            return file_path.stem, ""
+
+    async def get_missing_items(self, batch_size: int) -> list[ContentItem]:
+        """Fetch documentation files without embeddings."""
+        docs_root = self._get_docs_root()
+
+        if not docs_root.exists():
+            logger.warning(f"Documentation root not found: {docs_root}")
+            return []
+
+        # Find all .md and .mdx files
+        all_docs = list(docs_root.rglob("*.md")) + list(docs_root.rglob("*.mdx"))
+
+        # Get relative paths for content IDs
+        doc_paths = [str(doc.relative_to(docs_root)) for doc in all_docs]
+
+        if not doc_paths:
+            return []
+
+        # Check which ones have embeddings
+        placeholders = ",".join([f"${i+1}" for i in range(len(doc_paths))])
+        existing_result = await query_raw_with_schema(
+            f"""
+            SELECT "contentId"
+            FROM {{schema_prefix}}"UnifiedContentEmbedding"
+            WHERE "contentType" = 'DOCUMENTATION'::{{schema_prefix}}"ContentType"
+            AND "contentId" = ANY(ARRAY[{placeholders}])
+            """,
+            *doc_paths,
+        )
+
+        existing_ids = {row["contentId"] for row in existing_result}
+        missing_docs = [
+            (doc_path, doc_file)
+            for doc_path, doc_file in zip(doc_paths, all_docs)
+            if doc_path not in existing_ids
+        ]
+
+        # Convert to ContentItem
+        items = []
+        for doc_path, doc_file in missing_docs[:batch_size]:
+            try:
+                title, content = self._extract_title_and_content(doc_file)
+
+                # Build searchable text
+                searchable_text = f"{title} {content}"
+
+                items.append(
+                    ContentItem(
+                        content_id=doc_path,
+                        content_type=ContentType.DOCUMENTATION,
+                        searchable_text=searchable_text,
+                        metadata={
+                            "title": title,
+                            "path": doc_path,
+                        },
+                        user_id=None,  # Documentation is public
+                    )
+                )
+            except Exception as e:
+                logger.warning(f"Failed to process doc {doc_path}: {e}")
+                continue
+
+        return items
+
+    async def get_stats(self) -> dict[str, int]:
+        """Get statistics about documentation embedding coverage."""
+        docs_root = self._get_docs_root()
+
+        if not docs_root.exists():
+            return {"total": 0, "with_embeddings": 0, "without_embeddings": 0}
+
+        # Count all .md and .mdx files
+        all_docs = list(docs_root.rglob("*.md")) + list(docs_root.rglob("*.mdx"))
+        total_docs = len(all_docs)
+
+        if total_docs == 0:
+            return {"total": 0, "with_embeddings": 0, "without_embeddings": 0}
+
+        doc_paths = [str(doc.relative_to(docs_root)) for doc in all_docs]
+        placeholders = ",".join([f"${i+1}" for i in range(len(doc_paths))])
+
+        embedded_result = await query_raw_with_schema(
+            f"""
+            SELECT COUNT(*) as count
+            FROM {{schema_prefix}}"UnifiedContentEmbedding"
+            WHERE "contentType" = 'DOCUMENTATION'::{{schema_prefix}}"ContentType"
+            AND "contentId" = ANY(ARRAY[{placeholders}])
+            """,
+            *doc_paths,
+        )
+
+        with_embeddings = embedded_result[0]["count"] if embedded_result else 0
+
+        return {
+            "total": total_docs,
+            "with_embeddings": with_embeddings,
+            "without_embeddings": total_docs - with_embeddings,
+        }
+
+
+# Content handler registry
+CONTENT_HANDLERS: dict[ContentType, ContentHandler] = {
+    ContentType.STORE_AGENT: StoreAgentHandler(),
+    ContentType.BLOCK: BlockHandler(),
+    ContentType.DOCUMENTATION: DocumentationHandler(),
+}

autogpt_platform/backend/backend/api/features/store/content_handlers_integration_test.py

@@ -0,0 +1,214 @@
+"""
+Integration tests for content handlers using real DB.
+
+Run with: poetry run pytest backend/api/features/store/content_handlers_integration_test.py -xvs
+
+These tests use the real database but mock OpenAI calls.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from backend.api.features.store.content_handlers import (
+    CONTENT_HANDLERS,
+    BlockHandler,
+    DocumentationHandler,
+    StoreAgentHandler,
+)
+from backend.api.features.store.embeddings import (
+    backfill_all_content_types,
+    ensure_content_embedding,
+    get_embedding_stats,
+)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_agent_handler_real_db():
+    """Test StoreAgentHandler with real database queries."""
+    handler = StoreAgentHandler()
+
+    # Get stats from real DB
+    stats = await handler.get_stats()
+
+    # Stats should have correct structure
+    assert "total" in stats
+    assert "with_embeddings" in stats
+    assert "without_embeddings" in stats
+    assert stats["total"] >= 0
+    assert stats["with_embeddings"] >= 0
+    assert stats["without_embeddings"] >= 0
+
+    # Get missing items (max 1 to keep test fast)
+    items = await handler.get_missing_items(batch_size=1)
+
+    # Items should be list (may be empty if all have embeddings)
+    assert isinstance(items, list)
+
+    if items:
+        item = items[0]
+        assert item.content_id is not None
+        assert item.content_type.value == "STORE_AGENT"
+        assert item.searchable_text != ""
+        assert item.user_id is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_block_handler_real_db():
+    """Test BlockHandler with real database queries."""
+    handler = BlockHandler()
+
+    # Get stats from real DB
+    stats = await handler.get_stats()
+
+    # Stats should have correct structure
+    assert "total" in stats
+    assert "with_embeddings" in stats
+    assert "without_embeddings" in stats
+    assert stats["total"] >= 0  # Should have at least some blocks
+    assert stats["with_embeddings"] >= 0
+    assert stats["without_embeddings"] >= 0
+
+    # Get missing items (max 1 to keep test fast)
+    items = await handler.get_missing_items(batch_size=1)
+
+    # Items should be list
+    assert isinstance(items, list)
+
+    if items:
+        item = items[0]
+        assert item.content_id is not None  # Should be block UUID
+        assert item.content_type.value == "BLOCK"
+        assert item.searchable_text != ""
+        assert item.user_id is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_documentation_handler_real_fs():
+    """Test DocumentationHandler with real filesystem."""
+    handler = DocumentationHandler()
+
+    # Get stats from real filesystem
+    stats = await handler.get_stats()
+
+    # Stats should have correct structure
+    assert "total" in stats
+    assert "with_embeddings" in stats
+    assert "without_embeddings" in stats
+    assert stats["total"] >= 0
+    assert stats["with_embeddings"] >= 0
+    assert stats["without_embeddings"] >= 0
+
+    # Get missing items (max 1 to keep test fast)
+    items = await handler.get_missing_items(batch_size=1)
+
+    # Items should be list
+    assert isinstance(items, list)
+
+    if items:
+        item = items[0]
+        assert item.content_id is not None  # Should be relative path
+        assert item.content_type.value == "DOCUMENTATION"
+        assert item.searchable_text != ""
+        assert item.user_id is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_stats_all_types():
+    """Test get_embedding_stats aggregates all content types."""
+    stats = await get_embedding_stats()
+
+    # Should have structure with by_type and totals
+    assert "by_type" in stats
+    assert "totals" in stats
+
+    # Check each content type is present
+    by_type = stats["by_type"]
+    assert "STORE_AGENT" in by_type
+    assert "BLOCK" in by_type
+    assert "DOCUMENTATION" in by_type
+
+    # Check totals are aggregated
+    totals = stats["totals"]
+    assert totals["total"] >= 0
+    assert totals["with_embeddings"] >= 0
+    assert totals["without_embeddings"] >= 0
+    assert "coverage_percent" in totals
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+async def test_ensure_content_embedding_blocks(mock_generate):
+    """Test creating embeddings for blocks (mocked OpenAI)."""
+    # Mock OpenAI to return fake embedding
+    mock_generate.return_value = [0.1] * 1536
+
+    # Get one block without embedding
+    handler = BlockHandler()
+    items = await handler.get_missing_items(batch_size=1)
+
+    if not items:
+        pytest.skip("No blocks without embeddings")
+
+    item = items[0]
+
+    # Try to create embedding (OpenAI mocked)
+    result = await ensure_content_embedding(
+        content_type=item.content_type,
+        content_id=item.content_id,
+        searchable_text=item.searchable_text,
+        metadata=item.metadata,
+        user_id=item.user_id,
+    )
+
+    # Should succeed with mocked OpenAI
+    assert result is True
+    mock_generate.assert_called_once()
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+async def test_backfill_all_content_types_dry_run(mock_generate):
+    """Test backfill_all_content_types processes all handlers in order."""
+    # Mock OpenAI to return fake embedding
+    mock_generate.return_value = [0.1] * 1536
+
+    # Run backfill with batch_size=1 to process max 1 per type
+    result = await backfill_all_content_types(batch_size=1)
+
+    # Should have results for all content types
+    assert "by_type" in result
+    assert "totals" in result
+
+    by_type = result["by_type"]
+    assert "BLOCK" in by_type
+    assert "STORE_AGENT" in by_type
+    assert "DOCUMENTATION" in by_type
+
+    # Each type should have correct structure
+    for content_type, type_result in by_type.items():
+        assert "processed" in type_result
+        assert "success" in type_result
+        assert "failed" in type_result
+
+    # Totals should aggregate
+    totals = result["totals"]
+    assert totals["processed"] >= 0
+    assert totals["success"] >= 0
+    assert totals["failed"] >= 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_content_handler_registry():
+    """Test all handlers are registered in correct order."""
+    from prisma.enums import ContentType
+
+    # All three types should be registered
+    assert ContentType.STORE_AGENT in CONTENT_HANDLERS
+    assert ContentType.BLOCK in CONTENT_HANDLERS
+    assert ContentType.DOCUMENTATION in CONTENT_HANDLERS
+
+    # Check handler types
+    assert isinstance(CONTENT_HANDLERS[ContentType.STORE_AGENT], StoreAgentHandler)
+    assert isinstance(CONTENT_HANDLERS[ContentType.BLOCK], BlockHandler)
+    assert isinstance(CONTENT_HANDLERS[ContentType.DOCUMENTATION], DocumentationHandler)

autogpt_platform/backend/backend/api/features/store/content_handlers_test.py

@@ -0,0 +1,324 @@
+"""
+E2E tests for content handlers (blocks, store agents, documentation).
+
+Tests the full flow: discovering content → generating embeddings → storing.
+"""
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.api.features.store.content_handlers import (
+    CONTENT_HANDLERS,
+    BlockHandler,
+    DocumentationHandler,
+    StoreAgentHandler,
+)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_agent_handler_get_missing_items(mocker):
+    """Test StoreAgentHandler fetches approved agents without embeddings."""
+    handler = StoreAgentHandler()
+
+    # Mock database query
+    mock_missing = [
+        {
+            "id": "agent-1",
+            "name": "Test Agent",
+            "description": "A test agent",
+            "subHeading": "Test heading",
+            "categories": ["AI", "Testing"],
+        }
+    ]
+
+    with patch(
+        "backend.api.features.store.content_handlers.query_raw_with_schema",
+        return_value=mock_missing,
+    ):
+        items = await handler.get_missing_items(batch_size=10)
+
+        assert len(items) == 1
+        assert items[0].content_id == "agent-1"
+        assert items[0].content_type == ContentType.STORE_AGENT
+        assert "Test Agent" in items[0].searchable_text
+        assert "A test agent" in items[0].searchable_text
+        assert items[0].metadata["name"] == "Test Agent"
+        assert items[0].user_id is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_agent_handler_get_stats(mocker):
+    """Test StoreAgentHandler returns correct stats."""
+    handler = StoreAgentHandler()
+
+    # Mock approved count query
+    mock_approved = [{"count": 50}]
+    # Mock embedded count query
+    mock_embedded = [{"count": 30}]
+
+    with patch(
+        "backend.api.features.store.content_handlers.query_raw_with_schema",
+        side_effect=[mock_approved, mock_embedded],
+    ):
+        stats = await handler.get_stats()
+
+        assert stats["total"] == 50
+        assert stats["with_embeddings"] == 30
+        assert stats["without_embeddings"] == 20
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_block_handler_get_missing_items(mocker):
+    """Test BlockHandler discovers blocks without embeddings."""
+    handler = BlockHandler()
+
+    # Mock get_blocks to return test blocks
+    mock_block_class = MagicMock()
+    mock_block_instance = MagicMock()
+    mock_block_instance.name = "Calculator Block"
+    mock_block_instance.description = "Performs calculations"
+    mock_block_instance.categories = [MagicMock(value="MATH")]
+    mock_block_instance.input_schema.model_json_schema.return_value = {
+        "properties": {"expression": {"description": "Math expression to evaluate"}}
+    }
+    mock_block_class.return_value = mock_block_instance
+
+    mock_blocks = {"block-uuid-1": mock_block_class}
+
+    # Mock existing embeddings query (no embeddings exist)
+    mock_existing = []
+
+    with patch(
+        "backend.api.features.store.content_handlers.get_blocks",
+        return_value=mock_blocks,
+    ):
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=mock_existing,
+        ):
+            items = await handler.get_missing_items(batch_size=10)
+
+            assert len(items) == 1
+            assert items[0].content_id == "block-uuid-1"
+            assert items[0].content_type == ContentType.BLOCK
+            assert "Calculator Block" in items[0].searchable_text
+            assert "Performs calculations" in items[0].searchable_text
+            assert "MATH" in items[0].searchable_text
+            assert "expression: Math expression" in items[0].searchable_text
+            assert items[0].user_id is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_block_handler_get_stats(mocker):
+    """Test BlockHandler returns correct stats."""
+    handler = BlockHandler()
+
+    # Mock get_blocks
+    mock_blocks = {
+        "block-1": MagicMock(),
+        "block-2": MagicMock(),
+        "block-3": MagicMock(),
+    }
+
+    # Mock embedded count query (2 blocks have embeddings)
+    mock_embedded = [{"count": 2}]
+
+    with patch(
+        "backend.api.features.store.content_handlers.get_blocks",
+        return_value=mock_blocks,
+    ):
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=mock_embedded,
+        ):
+            stats = await handler.get_stats()
+
+            assert stats["total"] == 3
+            assert stats["with_embeddings"] == 2
+            assert stats["without_embeddings"] == 1
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_documentation_handler_get_missing_items(tmp_path, mocker):
+    """Test DocumentationHandler discovers docs without embeddings."""
+    handler = DocumentationHandler()
+
+    # Create temporary docs directory with test files
+    docs_root = tmp_path / "docs"
+    docs_root.mkdir()
+
+    (docs_root / "guide.md").write_text("# Getting Started\n\nThis is a guide.")
+    (docs_root / "api.mdx").write_text("# API Reference\n\nAPI documentation.")
+
+    # Mock _get_docs_root to return temp dir
+    with patch.object(handler, "_get_docs_root", return_value=docs_root):
+        # Mock existing embeddings query (no embeddings exist)
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=[],
+        ):
+            items = await handler.get_missing_items(batch_size=10)
+
+            assert len(items) == 2
+
+            # Check guide.md
+            guide_item = next(
+                (item for item in items if item.content_id == "guide.md"), None
+            )
+            assert guide_item is not None
+            assert guide_item.content_type == ContentType.DOCUMENTATION
+            assert "Getting Started" in guide_item.searchable_text
+            assert "This is a guide" in guide_item.searchable_text
+            assert guide_item.metadata["title"] == "Getting Started"
+            assert guide_item.user_id is None
+
+            # Check api.mdx
+            api_item = next(
+                (item for item in items if item.content_id == "api.mdx"), None
+            )
+            assert api_item is not None
+            assert "API Reference" in api_item.searchable_text
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_documentation_handler_get_stats(tmp_path, mocker):
+    """Test DocumentationHandler returns correct stats."""
+    handler = DocumentationHandler()
+
+    # Create temporary docs directory
+    docs_root = tmp_path / "docs"
+    docs_root.mkdir()
+    (docs_root / "doc1.md").write_text("# Doc 1")
+    (docs_root / "doc2.md").write_text("# Doc 2")
+    (docs_root / "doc3.mdx").write_text("# Doc 3")
+
+    # Mock embedded count query (1 doc has embedding)
+    mock_embedded = [{"count": 1}]
+
+    with patch.object(handler, "_get_docs_root", return_value=docs_root):
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=mock_embedded,
+        ):
+            stats = await handler.get_stats()
+
+            assert stats["total"] == 3
+            assert stats["with_embeddings"] == 1
+            assert stats["without_embeddings"] == 2
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_documentation_handler_title_extraction(tmp_path):
+    """Test DocumentationHandler extracts title from markdown heading."""
+    handler = DocumentationHandler()
+
+    # Test with heading
+    doc_with_heading = tmp_path / "with_heading.md"
+    doc_with_heading.write_text("# My Title\n\nContent here")
+    title, content = handler._extract_title_and_content(doc_with_heading)
+    assert title == "My Title"
+    assert "# My Title" not in content
+    assert "Content here" in content
+
+    # Test without heading
+    doc_without_heading = tmp_path / "no-heading.md"
+    doc_without_heading.write_text("Just content, no heading")
+    title, content = handler._extract_title_and_content(doc_without_heading)
+    assert title == "No Heading"  # Uses filename
+    assert "Just content" in content
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_content_handlers_registry():
+    """Test all content types are registered."""
+    assert ContentType.STORE_AGENT in CONTENT_HANDLERS
+    assert ContentType.BLOCK in CONTENT_HANDLERS
+    assert ContentType.DOCUMENTATION in CONTENT_HANDLERS
+
+    assert isinstance(CONTENT_HANDLERS[ContentType.STORE_AGENT], StoreAgentHandler)
+    assert isinstance(CONTENT_HANDLERS[ContentType.BLOCK], BlockHandler)
+    assert isinstance(CONTENT_HANDLERS[ContentType.DOCUMENTATION], DocumentationHandler)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_block_handler_handles_missing_attributes():
+    """Test BlockHandler gracefully handles blocks with missing attributes."""
+    handler = BlockHandler()
+
+    # Mock block with minimal attributes
+    mock_block_class = MagicMock()
+    mock_block_instance = MagicMock()
+    mock_block_instance.name = "Minimal Block"
+    # No description, categories, or schema
+    del mock_block_instance.description
+    del mock_block_instance.categories
+    del mock_block_instance.input_schema
+    mock_block_class.return_value = mock_block_instance
+
+    mock_blocks = {"block-minimal": mock_block_class}
+
+    with patch(
+        "backend.api.features.store.content_handlers.get_blocks",
+        return_value=mock_blocks,
+    ):
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=[],
+        ):
+            items = await handler.get_missing_items(batch_size=10)
+
+            assert len(items) == 1
+            assert items[0].searchable_text == "Minimal Block"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_block_handler_skips_failed_blocks():
+    """Test BlockHandler skips blocks that fail to instantiate."""
+    handler = BlockHandler()
+
+    # Mock one good block and one bad block
+    good_block = MagicMock()
+    good_instance = MagicMock()
+    good_instance.name = "Good Block"
+    good_instance.description = "Works fine"
+    good_instance.categories = []
+    good_block.return_value = good_instance
+
+    bad_block = MagicMock()
+    bad_block.side_effect = Exception("Instantiation failed")
+
+    mock_blocks = {"good-block": good_block, "bad-block": bad_block}
+
+    with patch(
+        "backend.api.features.store.content_handlers.get_blocks",
+        return_value=mock_blocks,
+    ):
+        with patch(
+            "backend.api.features.store.content_handlers.query_raw_with_schema",
+            return_value=[],
+        ):
+            items = await handler.get_missing_items(batch_size=10)
+
+            # Should only get the good block
+            assert len(items) == 1
+            assert items[0].content_id == "good-block"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_documentation_handler_missing_docs_directory():
+    """Test DocumentationHandler handles missing docs directory gracefully."""
+    handler = DocumentationHandler()
+
+    # Mock _get_docs_root to return non-existent path
+    fake_path = Path("/nonexistent/docs")
+    with patch.object(handler, "_get_docs_root", return_value=fake_path):
+        items = await handler.get_missing_items(batch_size=10)
+        assert items == []
+
+        stats = await handler.get_stats()
+        assert stats["total"] == 0
+        assert stats["with_embeddings"] == 0
+        assert stats["without_embeddings"] == 0

autogpt_platform/backend/backend/api/features/store/db.py

@@ -1,8 +1,7 @@
 import asyncio
 import logging
-import typing
 from datetime import datetime, timezone
-from typing import Literal
+from typing import Any, Literal
 
 import fastapi
 import prisma.enums
@@ -10,7 +9,7 @@ import prisma.errors
 import prisma.models
 import prisma.types
 
-from backend.data.db import query_raw_with_schema, transaction
+from backend.data.db import transaction
 from backend.data.graph import (
     GraphMeta,
     GraphModel,
@@ -30,6 +29,8 @@ from backend.util.settings import Settings
 
 from . import exceptions as store_exceptions
 from . import model as store_model
+from .embeddings import ensure_embedding
+from .hybrid_search import hybrid_search
 
 logger = logging.getLogger(__name__)
 settings = Settings()
@@ -50,128 +51,77 @@ async def get_store_agents(
     page_size: int = 20,
 ) -> store_model.StoreAgentsResponse:
     """
-    Get PUBLIC store agents from the StoreAgent view
+    Get PUBLIC store agents from the StoreAgent view.
+
+    Search behavior:
+    - With search_query: Uses hybrid search (semantic + lexical)
+    - Fallback: If embeddings unavailable, gracefully degrades to lexical-only
+    - Rationale: User-facing endpoint prioritizes availability over accuracy
+
+    Note: Admin operations (approval) use fail-fast to prevent inconsistent state.
     """
     logger.debug(
         f"Getting store agents. featured={featured}, creators={creators}, sorted_by={sorted_by}, search={search_query}, category={category}, page={page}"
     )
 
+    search_used_hybrid = False
+    store_agents: list[store_model.StoreAgent] = []
+    agents: list[dict[str, Any]] = []
+    total = 0
+    total_pages = 0
+
     try:
-        # If search_query is provided, use full-text search
+        # If search_query is provided, use hybrid search (embeddings + tsvector)
         if search_query:
-            offset = (page - 1) * page_size
+            # Try hybrid search combining semantic and lexical signals
+            # Falls back to lexical-only if OpenAI unavailable (user-facing, high SLA)
+            try:
+                agents, total = await hybrid_search(
+                    query=search_query,
+                    featured=featured,
+                    creators=creators,
+                    category=category,
+                    sorted_by="relevance",  # Use hybrid scoring for relevance
+                    page=page,
+                    page_size=page_size,
+                )
+                search_used_hybrid = True
+            except Exception as e:
+                # Log error but fall back to lexical search for better UX
+                logger.error(
+                    f"Hybrid search failed (likely OpenAI unavailable), "
+                    f"falling back to lexical search: {e}"
+                )
+                # search_used_hybrid remains False, will use fallback path below
 
-            # Whitelist allowed order_by columns
-            ALLOWED_ORDER_BY = {
-                "rating": "rating DESC, rank DESC",
-                "runs": "runs DESC, rank DESC",
-                "name": "agent_name ASC, rank ASC",
-                "updated_at": "updated_at DESC, rank DESC",
-            }
+            # Convert hybrid search results (dict format) if hybrid succeeded
+            if search_used_hybrid:
+                total_pages = (total + page_size - 1) // page_size
+                store_agents: list[store_model.StoreAgent] = []
+                for agent in agents:
+                    try:
+                        store_agent = store_model.StoreAgent(
+                            slug=agent["slug"],
+                            agent_name=agent["agent_name"],
+                            agent_image=(
+                                agent["agent_image"][0] if agent["agent_image"] else ""
+                            ),
+                            creator=agent["creator_username"] or "Needs Profile",
+                            creator_avatar=agent["creator_avatar"] or "",
+                            sub_heading=agent["sub_heading"],
+                            description=agent["description"],
+                            runs=agent["runs"],
+                            rating=agent["rating"],
+                        )
+                        store_agents.append(store_agent)
+                    except Exception as e:
+                        logger.error(
+                            f"Error parsing Store agent from hybrid search results: {e}"
+                        )
+                        continue
 
-            # Validate and get order clause
-            if sorted_by and sorted_by in ALLOWED_ORDER_BY:
-                order_by_clause = ALLOWED_ORDER_BY[sorted_by]
-            else:
-                order_by_clause = "updated_at DESC, rank DESC"
-
-            # Build WHERE conditions and parameters list
-            where_parts: list[str] = []
-            params: list[typing.Any] = [search_query]  # $1 - search term
-            param_index = 2  # Start at $2 for next parameter
-
-            # Always filter for available agents
-            where_parts.append("is_available = true")
-
-            if featured:
-                where_parts.append("featured = true")
-
-            if creators and creators:
-                # Use ANY with array parameter
-                where_parts.append(f"creator_username = ANY(${param_index})")
-                params.append(creators)
-                param_index += 1
-
-            if category and category:
-                where_parts.append(f"${param_index} = ANY(categories)")
-                params.append(category)
-                param_index += 1
-
-            sql_where_clause: str = " AND ".join(where_parts) if where_parts else "1=1"
-
-            # Add pagination params
-            params.extend([page_size, offset])
-            limit_param = f"${param_index}"
-            offset_param = f"${param_index + 1}"
-
-            # Execute full-text search query with parameterized values
-            sql_query = f"""
-                SELECT
-                    slug,
-                    agent_name,
-                    agent_image,
-                    creator_username,
-                    creator_avatar,
-                    sub_heading,
-                    description,
-                    runs,
-                    rating,
-                    categories,
-                    featured,
-                    is_available,
-                    updated_at,
-                    ts_rank_cd(search, query) AS rank
-                FROM {{schema_prefix}}"StoreAgent",
-                    plainto_tsquery('english', $1) AS query
-                WHERE {sql_where_clause}
-                    AND search @@ query
-                ORDER BY {order_by_clause}
-                LIMIT {limit_param} OFFSET {offset_param}
-            """
-
-            # Count query for pagination - only uses search term parameter
-            count_query = f"""
-                SELECT COUNT(*) as count
-                FROM {{schema_prefix}}"StoreAgent",
-                    plainto_tsquery('english', $1) AS query
-                WHERE {sql_where_clause}
-                    AND search @@ query
-            """
-
-            # Execute both queries with parameters
-            agents = await query_raw_with_schema(sql_query, *params)
-
-            # For count, use params without pagination (last 2 params)
-            count_params = params[:-2]
-            count_result = await query_raw_with_schema(count_query, *count_params)
-
-            total = count_result[0]["count"] if count_result else 0
-            total_pages = (total + page_size - 1) // page_size
-
-            # Convert raw results to StoreAgent models
-            store_agents: list[store_model.StoreAgent] = []
-            for agent in agents:
-                try:
-                    store_agent = store_model.StoreAgent(
-                        slug=agent["slug"],
-                        agent_name=agent["agent_name"],
-                        agent_image=(
-                            agent["agent_image"][0] if agent["agent_image"] else ""
-                        ),
-                        creator=agent["creator_username"] or "Needs Profile",
-                        creator_avatar=agent["creator_avatar"] or "",
-                        sub_heading=agent["sub_heading"],
-                        description=agent["description"],
-                        runs=agent["runs"],
-                        rating=agent["rating"],
-                    )
-                    store_agents.append(store_agent)
-                except Exception as e:
-                    logger.error(f"Error parsing Store agent from search results: {e}")
-                    continue
-
-        else:
-            # Non-search query path (original logic)
+        if not search_used_hybrid:
+            # Fallback path - use basic search or no search
             where_clause: prisma.types.StoreAgentWhereInput = {"is_available": True}
             if featured:
                 where_clause["featured"] = featured
@@ -180,6 +130,14 @@ async def get_store_agents(
             if category:
                 where_clause["categories"] = {"has": category}
 
+            # Add basic text search if search_query provided but hybrid failed
+            if search_query:
+                where_clause["OR"] = [
+                    {"agent_name": {"contains": search_query, "mode": "insensitive"}},
+                    {"sub_heading": {"contains": search_query, "mode": "insensitive"}},
+                    {"description": {"contains": search_query, "mode": "insensitive"}},
+                ]
+
             order_by = []
             if sorted_by == "rating":
                 order_by.append({"rating": "desc"})
@@ -188,7 +146,7 @@ async def get_store_agents(
             elif sorted_by == "name":
                 order_by.append({"agent_name": "asc"})
 
-            agents = await prisma.models.StoreAgent.prisma().find_many(
+            db_agents = await prisma.models.StoreAgent.prisma().find_many(
                 where=where_clause,
                 order=order_by,
                 skip=(page - 1) * page_size,
@@ -199,7 +157,7 @@ async def get_store_agents(
             total_pages = (total + page_size - 1) // page_size
 
             store_agents: list[store_model.StoreAgent] = []
-            for agent in agents:
+            for agent in db_agents:
                 try:
                     # Create the StoreAgent object safely
                     store_agent = store_model.StoreAgent(
@@ -614,6 +572,7 @@ async def get_store_submissions(
         submission_models = []
         for sub in submissions:
             submission_model = store_model.StoreSubmission(
+                listing_id=sub.listing_id,
                 agent_id=sub.agent_id,
                 agent_version=sub.agent_version,
                 name=sub.name,
@@ -667,35 +626,48 @@ async def delete_store_submission(
     submission_id: str,
 ) -> bool:
     """
-    Delete a store listing submission as the submitting user.
+    Delete a store submission version as the submitting user.
 
     Args:
         user_id: ID of the authenticated user
-        submission_id: ID of the submission to be deleted
+        submission_id: StoreListingVersion ID to delete
 
     Returns:
-        bool: True if the submission was successfully deleted, False otherwise
+        bool: True if successfully deleted
     """
-    logger.debug(f"Deleting store submission {submission_id} for user {user_id}")
-
     try:
-        # Verify the submission belongs to this user
-        submission = await prisma.models.StoreListing.prisma().find_first(
-            where={"agentGraphId": submission_id, "owningUserId": user_id}
+        # Find the submission version with ownership check
+        version = await prisma.models.StoreListingVersion.prisma().find_first(
+            where={"id": submission_id}, include={"StoreListing": True}
         )
 
-        if not submission:
-            logger.warning(f"Submission not found for user {user_id}: {submission_id}")
-            raise store_exceptions.SubmissionNotFoundError(
-                f"Submission not found for this user. User ID: {user_id}, Submission ID: {submission_id}"
+        if (
+            not version
+            or not version.StoreListing
+            or version.StoreListing.owningUserId != user_id
+        ):
+            raise store_exceptions.SubmissionNotFoundError("Submission not found")
+
+        # Prevent deletion of approved submissions
+        if version.submissionStatus == prisma.enums.SubmissionStatus.APPROVED:
+            raise store_exceptions.InvalidOperationError(
+                "Cannot delete approved submissions"
             )
 
-        # Delete the submission
-        await prisma.models.StoreListing.prisma().delete(where={"id": submission.id})
-
-        logger.debug(
-            f"Successfully deleted submission {submission_id} for user {user_id}"
+        # Delete the version
+        await prisma.models.StoreListingVersion.prisma().delete(
+            where={"id": version.id}
         )
+
+        # Clean up empty listing if this was the last version
+        remaining = await prisma.models.StoreListingVersion.prisma().count(
+            where={"storeListingId": version.storeListingId}
+        )
+        if remaining == 0:
+            await prisma.models.StoreListing.prisma().delete(
+                where={"id": version.storeListingId}
+            )
+
         return True
 
     except Exception as e:
@@ -759,9 +731,15 @@ async def create_store_submission(
             logger.warning(
                 f"Agent not found for user {user_id}: {agent_id} v{agent_version}"
             )
-            raise store_exceptions.AgentNotFoundError(
-                f"Agent not found for this user. User ID: {user_id}, Agent ID: {agent_id}, Version: {agent_version}"
-            )
+            # Provide more user-friendly error message when agent_id is empty
+            if not agent_id or agent_id.strip() == "":
+                raise store_exceptions.AgentNotFoundError(
+                    "No agent selected. Please select an agent before submitting to the store."
+                )
+            else:
+                raise store_exceptions.AgentNotFoundError(
+                    f"Agent not found for this user. User ID: {user_id}, Agent ID: {agent_id}, Version: {agent_version}"
+                )
 
         # Check if listing already exists for this agent
         existing_listing = await prisma.models.StoreListing.prisma().find_first(
@@ -833,6 +811,7 @@ async def create_store_submission(
         logger.debug(f"Created store listing for agent {agent_id}")
         # Return submission details
         return store_model.StoreSubmission(
+            listing_id=listing.id,
             agent_id=agent_id,
             agent_version=agent_version,
             name=name,
@@ -944,81 +923,56 @@ async def edit_store_submission(
         # Currently we are not allowing user to update the agent associated with a submission
         # If we allow it in future, then we need a check here to verify the agent belongs to this user.
 
-        # Check if we can edit this submission
-        if current_version.submissionStatus == prisma.enums.SubmissionStatus.REJECTED:
+        # Only allow editing of PENDING submissions
+        if current_version.submissionStatus != prisma.enums.SubmissionStatus.PENDING:
             raise store_exceptions.InvalidOperationError(
-                "Cannot edit a rejected submission"
-            )
-
-        # For APPROVED submissions, we need to create a new version
-        if current_version.submissionStatus == prisma.enums.SubmissionStatus.APPROVED:
-            # Create a new version for the existing listing
-            return await create_store_version(
-                user_id=user_id,
-                agent_id=current_version.agentGraphId,
-                agent_version=current_version.agentGraphVersion,
-                store_listing_id=current_version.storeListingId,
-                name=name,
-                video_url=video_url,
-                agent_output_demo_url=agent_output_demo_url,
-                image_urls=image_urls,
-                description=description,
-                sub_heading=sub_heading,
-                categories=categories,
-                changes_summary=changes_summary,
-                recommended_schedule_cron=recommended_schedule_cron,
-                instructions=instructions,
+                f"Cannot edit a {current_version.submissionStatus.value.lower()} submission. Only pending submissions can be edited."
             )
 
         # For PENDING submissions, we can update the existing version
-        elif current_version.submissionStatus == prisma.enums.SubmissionStatus.PENDING:
-            # Update the existing version
-            updated_version = await prisma.models.StoreListingVersion.prisma().update(
-                where={"id": store_listing_version_id},
-                data=prisma.types.StoreListingVersionUpdateInput(
-                    name=name,
-                    videoUrl=video_url,
-                    agentOutputDemoUrl=agent_output_demo_url,
-                    imageUrls=image_urls,
-                    description=description,
-                    categories=categories,
-                    subHeading=sub_heading,
-                    changesSummary=changes_summary,
-                    recommendedScheduleCron=recommended_schedule_cron,
-                    instructions=instructions,
-                ),
-            )
-
-            logger.debug(
-                f"Updated existing version {store_listing_version_id} for agent {current_version.agentGraphId}"
-            )
-
-            if not updated_version:
-                raise DatabaseError("Failed to update store listing version")
-            return store_model.StoreSubmission(
-                agent_id=current_version.agentGraphId,
-                agent_version=current_version.agentGraphVersion,
+        # Update the existing version
+        updated_version = await prisma.models.StoreListingVersion.prisma().update(
+            where={"id": store_listing_version_id},
+            data=prisma.types.StoreListingVersionUpdateInput(
                 name=name,
-                sub_heading=sub_heading,
-                slug=current_version.StoreListing.slug,
+                videoUrl=video_url,
+                agentOutputDemoUrl=agent_output_demo_url,
+                imageUrls=image_urls,
                 description=description,
-                instructions=instructions,
-                image_urls=image_urls,
-                date_submitted=updated_version.submittedAt or updated_version.createdAt,
-                status=updated_version.submissionStatus,
-                runs=0,
-                rating=0.0,
-                store_listing_version_id=updated_version.id,
-                changes_summary=changes_summary,
-                video_url=video_url,
                 categories=categories,
-                version=updated_version.version,
-            )
+                subHeading=sub_heading,
+                changesSummary=changes_summary,
+                recommendedScheduleCron=recommended_schedule_cron,
+                instructions=instructions,
+            ),
+        )
 
-        else:
-            raise store_exceptions.InvalidOperationError(
-                f"Cannot edit submission with status: {current_version.submissionStatus}"
-            )
+        logger.debug(
+            f"Updated existing version {store_listing_version_id} for agent {current_version.agentGraphId}"
+        )
+
+        if not updated_version:
+            raise DatabaseError("Failed to update store listing version")
+        return store_model.StoreSubmission(
+            listing_id=current_version.StoreListing.id,
+            agent_id=current_version.agentGraphId,
+            agent_version=current_version.agentGraphVersion,
+            name=name,
+            sub_heading=sub_heading,
+            slug=current_version.StoreListing.slug,
+            description=description,
+            instructions=instructions,
+            image_urls=image_urls,
+            date_submitted=updated_version.submittedAt or updated_version.createdAt,
+            status=updated_version.submissionStatus,
+            runs=0,
+            rating=0.0,
+            store_listing_version_id=updated_version.id,
+            changes_summary=changes_summary,
+            video_url=video_url,
+            categories=categories,
+            version=updated_version.version,
+        )
 
     except (
         store_exceptions.SubmissionNotFoundError,
@@ -1097,38 +1051,78 @@ async def create_store_version(
                 f"Agent not found for this user. User ID: {user_id}, Agent ID: {agent_id}, Version: {agent_version}"
             )
 
-        # Get the latest version number
-        latest_version = listing.Versions[0] if listing.Versions else None
-
-        next_version = (latest_version.version + 1) if latest_version else 1
-
-        # Create a new version for the existing listing
-        new_version = await prisma.models.StoreListingVersion.prisma().create(
-            data=prisma.types.StoreListingVersionCreateInput(
-                version=next_version,
-                agentGraphId=agent_id,
-                agentGraphVersion=agent_version,
-                name=name,
-                videoUrl=video_url,
-                agentOutputDemoUrl=agent_output_demo_url,
-                imageUrls=image_urls,
-                description=description,
-                instructions=instructions,
-                categories=categories,
-                subHeading=sub_heading,
-                submissionStatus=prisma.enums.SubmissionStatus.PENDING,
-                submittedAt=datetime.now(),
-                changesSummary=changes_summary,
-                recommendedScheduleCron=recommended_schedule_cron,
-                storeListingId=store_listing_id,
+        # Check if there's already a PENDING submission for this agent (any version)
+        existing_pending_submission = (
+            await prisma.models.StoreListingVersion.prisma().find_first(
+                where=prisma.types.StoreListingVersionWhereInput(
+                    storeListingId=store_listing_id,
+                    agentGraphId=agent_id,
+                    submissionStatus=prisma.enums.SubmissionStatus.PENDING,
+                    isDeleted=False,
+                )
             )
         )
 
+        # Handle existing pending submission and create new one atomically
+        async with transaction() as tx:
+            # Get the latest version number first
+            latest_listing = await prisma.models.StoreListing.prisma(tx).find_first(
+                where=prisma.types.StoreListingWhereInput(
+                    id=store_listing_id, owningUserId=user_id
+                ),
+                include={"Versions": {"order_by": {"version": "desc"}, "take": 1}},
+            )
+
+            if not latest_listing:
+                raise store_exceptions.ListingNotFoundError(
+                    f"Store listing not found. User ID: {user_id}, Listing ID: {store_listing_id}"
+                )
+
+            latest_version = (
+                latest_listing.Versions[0] if latest_listing.Versions else None
+            )
+            next_version = (latest_version.version + 1) if latest_version else 1
+
+            # If there's an existing pending submission, delete it atomically before creating new one
+            if existing_pending_submission:
+                logger.info(
+                    f"Found existing PENDING submission for agent {agent_id} (was v{existing_pending_submission.agentGraphVersion}, now v{agent_version}), replacing existing submission instead of creating duplicate"
+                )
+                await prisma.models.StoreListingVersion.prisma(tx).delete(
+                    where={"id": existing_pending_submission.id}
+                )
+                logger.debug(
+                    f"Deleted existing pending submission {existing_pending_submission.id}"
+                )
+
+            # Create a new version for the existing listing
+            new_version = await prisma.models.StoreListingVersion.prisma(tx).create(
+                data=prisma.types.StoreListingVersionCreateInput(
+                    version=next_version,
+                    agentGraphId=agent_id,
+                    agentGraphVersion=agent_version,
+                    name=name,
+                    videoUrl=video_url,
+                    agentOutputDemoUrl=agent_output_demo_url,
+                    imageUrls=image_urls,
+                    description=description,
+                    instructions=instructions,
+                    categories=categories,
+                    subHeading=sub_heading,
+                    submissionStatus=prisma.enums.SubmissionStatus.PENDING,
+                    submittedAt=datetime.now(),
+                    changesSummary=changes_summary,
+                    recommendedScheduleCron=recommended_schedule_cron,
+                    storeListingId=store_listing_id,
+                )
+            )
+
         logger.debug(
             f"Created new version for listing {store_listing_id} of agent {agent_id}"
         )
         # Return submission details
         return store_model.StoreSubmission(
+            listing_id=listing.id,
             agent_id=agent_id,
             agent_version=agent_version,
             name=name,
@@ -1541,7 +1535,7 @@ async def review_store_submission(
                 )
 
                 # Update the AgentGraph with store listing data
-                await prisma.models.AgentGraph.prisma().update(
+                await prisma.models.AgentGraph.prisma(tx).update(
                     where={
                         "graphVersionId": {
                             "id": store_listing_version.agentGraphId,
@@ -1556,6 +1550,23 @@ async def review_store_submission(
                     },
                 )
 
+                # Generate embedding for approved listing (blocking - admin operation)
+                # Inside transaction: if embedding fails, entire transaction rolls back
+                embedding_success = await ensure_embedding(
+                    version_id=store_listing_version_id,
+                    name=store_listing_version.name,
+                    description=store_listing_version.description,
+                    sub_heading=store_listing_version.subHeading,
+                    categories=store_listing_version.categories or [],
+                    tx=tx,
+                )
+                if not embedding_success:
+                    raise ValueError(
+                        f"Failed to generate embedding for listing {store_listing_version_id}. "
+                        "This is likely due to OpenAI API being unavailable. "
+                        "Please try again later or contact support if the issue persists."
+                    )
+
                 await prisma.models.StoreListing.prisma(tx).update(
                     where={"id": store_listing_version.StoreListing.id},
                     data={
@@ -1708,15 +1719,12 @@ async def review_store_submission(
 
         # Convert to Pydantic model for consistency
         return store_model.StoreSubmission(
+            listing_id=(submission.StoreListing.id if submission.StoreListing else ""),
             agent_id=submission.agentGraphId,
             agent_version=submission.agentGraphVersion,
             name=submission.name,
             sub_heading=submission.subHeading,
-            slug=(
-                submission.StoreListing.slug
-                if hasattr(submission, "storeListing") and submission.StoreListing
-                else ""
-            ),
+            slug=(submission.StoreListing.slug if submission.StoreListing else ""),
             description=submission.description,
             instructions=submission.instructions,
             image_urls=submission.imageUrls or [],
@@ -1818,9 +1826,7 @@ async def get_admin_listings_with_versions(
         where = prisma.types.StoreListingWhereInput(**where_dict)
         include = prisma.types.StoreListingInclude(
             Versions=prisma.types.FindManyStoreListingVersionArgsFromStoreListing(
-                order_by=prisma.types._StoreListingVersion_version_OrderByInput(
-                    version="desc"
-                )
+                order_by={"version": "desc"}
             ),
             OwningUser=True,
         )
@@ -1845,6 +1851,7 @@ async def get_admin_listings_with_versions(
             # If we have versions, turn them into StoreSubmission models
             for version in listing.Versions or []:
                 version_model = store_model.StoreSubmission(
+                    listing_id=listing.id,
                     agent_id=version.agentGraphId,
                     agent_version=version.agentGraphVersion,
                     name=version.name,

autogpt_platform/backend/backend/api/features/store/embeddings.py

@@ -0,0 +1,628 @@
+"""
+Unified Content Embeddings Service
+
+Handles generation and storage of OpenAI embeddings for all content types
+(store listings, blocks, documentation, library agents) to enable semantic/hybrid search.
+"""
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import prisma
+from prisma.enums import ContentType
+from tiktoken import encoding_for_model
+
+from backend.api.features.store.content_handlers import CONTENT_HANDLERS
+from backend.data.db import execute_raw_with_schema, query_raw_with_schema
+from backend.util.clients import get_openai_client
+from backend.util.json import dumps
+
+logger = logging.getLogger(__name__)
+
+
+# OpenAI embedding model configuration
+EMBEDDING_MODEL = "text-embedding-3-small"
+# OpenAI embedding token limit (8,191 with 1 token buffer for safety)
+EMBEDDING_MAX_TOKENS = 8191
+
+
+def build_searchable_text(
+    name: str,
+    description: str,
+    sub_heading: str,
+    categories: list[str],
+) -> str:
+    """
+    Build searchable text from listing version fields.
+
+    Combines relevant fields into a single string for embedding.
+    """
+    parts = []
+
+    # Name is important - include it
+    if name:
+        parts.append(name)
+
+    # Sub-heading provides context
+    if sub_heading:
+        parts.append(sub_heading)
+
+    # Description is the main content
+    if description:
+        parts.append(description)
+
+    # Categories help with semantic matching
+    if categories:
+        parts.append(" ".join(categories))
+
+    return " ".join(parts)
+
+
+async def generate_embedding(text: str) -> list[float] | None:
+    """
+    Generate embedding for text using OpenAI API.
+
+    Returns None if embedding generation fails.
+    Fail-fast: no retries to maintain consistency with approval flow.
+    """
+    try:
+        client = get_openai_client()
+        if not client:
+            logger.error("openai_internal_api_key not set, cannot generate embedding")
+            return None
+
+        # Truncate text to token limit using tiktoken
+        # Character-based truncation is insufficient because token ratios vary by content type
+        enc = encoding_for_model(EMBEDDING_MODEL)
+        tokens = enc.encode(text)
+        if len(tokens) > EMBEDDING_MAX_TOKENS:
+            tokens = tokens[:EMBEDDING_MAX_TOKENS]
+            truncated_text = enc.decode(tokens)
+            logger.info(
+                f"Truncated text from {len(enc.encode(text))} to {len(tokens)} tokens"
+            )
+        else:
+            truncated_text = text
+
+        start_time = time.time()
+        response = await client.embeddings.create(
+            model=EMBEDDING_MODEL,
+            input=truncated_text,
+        )
+        latency_ms = (time.time() - start_time) * 1000
+
+        embedding = response.data[0].embedding
+        logger.info(
+            f"Generated embedding: {len(embedding)} dims, "
+            f"{len(tokens)} tokens, {latency_ms:.0f}ms"
+        )
+        return embedding
+
+    except Exception as e:
+        logger.error(f"Failed to generate embedding: {e}")
+        return None
+
+
+async def store_embedding(
+    version_id: str,
+    embedding: list[float],
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Store embedding in the database.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    DEPRECATED: Use ensure_embedding() instead (includes searchable_text).
+    """
+    return await store_content_embedding(
+        content_type=ContentType.STORE_AGENT,
+        content_id=version_id,
+        embedding=embedding,
+        searchable_text="",  # Empty for backward compat; ensure_embedding() populates this
+        metadata=None,
+        user_id=None,  # Store agents are public
+        tx=tx,
+    )
+
+
+async def store_content_embedding(
+    content_type: ContentType,
+    content_id: str,
+    embedding: list[float],
+    searchable_text: str,
+    metadata: dict | None = None,
+    user_id: str | None = None,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Store embedding in the unified content embeddings table.
+
+    New function for unified content embedding storage.
+    Uses raw SQL since Prisma doesn't natively support pgvector.
+    """
+    try:
+        client = tx if tx else prisma.get_client()
+
+        # Convert embedding to PostgreSQL vector format
+        embedding_str = embedding_to_vector_string(embedding)
+        metadata_json = dumps(metadata or {})
+
+        # Upsert the embedding
+        # WHERE clause in DO UPDATE prevents PostgreSQL 15 bug with NULLS NOT DISTINCT
+        await execute_raw_with_schema(
+            """
+            INSERT INTO {schema_prefix}"UnifiedContentEmbedding" (
+                "id", "contentType", "contentId", "userId", "embedding", "searchableText", "metadata", "createdAt", "updatedAt"
+            )
+            VALUES (gen_random_uuid()::text, $1::{schema_prefix}"ContentType", $2, $3, $4::vector, $5, $6::jsonb, NOW(), NOW())
+            ON CONFLICT ("contentType", "contentId", "userId")
+            DO UPDATE SET
+                "embedding" = $4::vector,
+                "searchableText" = $5,
+                "metadata" = $6::jsonb,
+                "updatedAt" = NOW()
+            WHERE {schema_prefix}"UnifiedContentEmbedding"."contentType" = $1::{schema_prefix}"ContentType"
+                AND {schema_prefix}"UnifiedContentEmbedding"."contentId" = $2
+                AND ({schema_prefix}"UnifiedContentEmbedding"."userId" = $3 OR ($3 IS NULL AND {schema_prefix}"UnifiedContentEmbedding"."userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+            embedding_str,
+            searchable_text,
+            metadata_json,
+            client=client,
+            set_public_search_path=True,
+        )
+
+        logger.info(f"Stored embedding for {content_type}:{content_id}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to store embedding for {content_type}:{content_id}: {e}")
+        return False
+
+
+async def get_embedding(version_id: str) -> dict[str, Any] | None:
+    """
+    Retrieve embedding record for a listing version.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    Returns dict with storeListingVersionId, embedding, timestamps or None if not found.
+    """
+    result = await get_content_embedding(
+        ContentType.STORE_AGENT, version_id, user_id=None
+    )
+    if result:
+        # Transform to old format for backward compatibility
+        return {
+            "storeListingVersionId": result["contentId"],
+            "embedding": result["embedding"],
+            "createdAt": result["createdAt"],
+            "updatedAt": result["updatedAt"],
+        }
+    return None
+
+
+async def get_content_embedding(
+    content_type: ContentType, content_id: str, user_id: str | None = None
+) -> dict[str, Any] | None:
+    """
+    Retrieve embedding record for any content type.
+
+    New function for unified content embedding retrieval.
+    Returns dict with contentType, contentId, embedding, timestamps or None if not found.
+    """
+    try:
+        result = await query_raw_with_schema(
+            """
+            SELECT
+                "contentType",
+                "contentId",
+                "userId",
+                "embedding"::text as "embedding",
+                "searchableText",
+                "metadata",
+                "createdAt",
+                "updatedAt"
+            FROM {schema_prefix}"UnifiedContentEmbedding"
+            WHERE "contentType" = $1::{schema_prefix}"ContentType" AND "contentId" = $2 AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+            set_public_search_path=True,
+        )
+
+        if result and len(result) > 0:
+            return result[0]
+        return None
+
+    except Exception as e:
+        logger.error(f"Failed to get embedding for {content_type}:{content_id}: {e}")
+        return None
+
+
+async def ensure_embedding(
+    version_id: str,
+    name: str,
+    description: str,
+    sub_heading: str,
+    categories: list[str],
+    force: bool = False,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Ensure an embedding exists for the listing version.
+
+    Creates embedding if missing. Use force=True to regenerate.
+    Backward-compatible wrapper for store listings.
+
+    Args:
+        version_id: The StoreListingVersion ID
+        name: Agent name
+        description: Agent description
+        sub_heading: Agent sub-heading
+        categories: Agent categories
+        force: Force regeneration even if embedding exists
+        tx: Optional transaction client
+
+    Returns:
+        True if embedding exists/was created, False on failure
+    """
+    try:
+        # Check if embedding already exists
+        if not force:
+            existing = await get_embedding(version_id)
+            if existing and existing.get("embedding"):
+                logger.debug(f"Embedding for version {version_id} already exists")
+                return True
+
+        # Build searchable text for embedding
+        searchable_text = build_searchable_text(
+            name, description, sub_heading, categories
+        )
+
+        # Generate new embedding
+        embedding = await generate_embedding(searchable_text)
+        if embedding is None:
+            logger.warning(f"Could not generate embedding for version {version_id}")
+            return False
+
+        # Store the embedding with metadata using new function
+        metadata = {
+            "name": name,
+            "subHeading": sub_heading,
+            "categories": categories,
+        }
+        return await store_content_embedding(
+            content_type=ContentType.STORE_AGENT,
+            content_id=version_id,
+            embedding=embedding,
+            searchable_text=searchable_text,
+            metadata=metadata,
+            user_id=None,  # Store agents are public
+            tx=tx,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to ensure embedding for version {version_id}: {e}")
+        return False
+
+
+async def delete_embedding(version_id: str) -> bool:
+    """
+    Delete embedding for a listing version.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    Note: This is usually handled automatically by CASCADE delete,
+    but provided for manual cleanup if needed.
+    """
+    return await delete_content_embedding(ContentType.STORE_AGENT, version_id)
+
+
+async def delete_content_embedding(
+    content_type: ContentType, content_id: str, user_id: str | None = None
+) -> bool:
+    """
+    Delete embedding for any content type.
+
+    New function for unified content embedding deletion.
+    Note: This is usually handled automatically by CASCADE delete,
+    but provided for manual cleanup if needed.
+
+    Args:
+        content_type: The type of content (STORE_AGENT, LIBRARY_AGENT, etc.)
+        content_id: The unique identifier for the content
+        user_id: Optional user ID. For public content (STORE_AGENT, BLOCK), pass None.
+                 For user-scoped content (LIBRARY_AGENT), pass the user's ID to avoid
+                 deleting embeddings belonging to other users.
+
+    Returns:
+        True if deletion succeeded, False otherwise
+    """
+    try:
+        client = prisma.get_client()
+
+        await execute_raw_with_schema(
+            """
+            DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
+            WHERE "contentType" = $1::{schema_prefix}"ContentType"
+              AND "contentId" = $2
+              AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+            client=client,
+        )
+
+        user_str = f" (user: {user_id})" if user_id else ""
+        logger.info(f"Deleted embedding for {content_type}:{content_id}{user_str}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to delete embedding for {content_type}:{content_id}: {e}")
+        return False
+
+
+async def get_embedding_stats() -> dict[str, Any]:
+    """
+    Get statistics about embedding coverage for all content types.
+
+    Returns stats per content type and overall totals.
+    """
+    try:
+        stats_by_type = {}
+        total_items = 0
+        total_with_embeddings = 0
+        total_without_embeddings = 0
+
+        # Aggregate stats from all handlers
+        for content_type, handler in CONTENT_HANDLERS.items():
+            try:
+                stats = await handler.get_stats()
+                stats_by_type[content_type.value] = {
+                    "total": stats["total"],
+                    "with_embeddings": stats["with_embeddings"],
+                    "without_embeddings": stats["without_embeddings"],
+                    "coverage_percent": (
+                        round(stats["with_embeddings"] / stats["total"] * 100, 1)
+                        if stats["total"] > 0
+                        else 0
+                    ),
+                }
+
+                total_items += stats["total"]
+                total_with_embeddings += stats["with_embeddings"]
+                total_without_embeddings += stats["without_embeddings"]
+
+            except Exception as e:
+                logger.error(f"Failed to get stats for {content_type.value}: {e}")
+                stats_by_type[content_type.value] = {
+                    "total": 0,
+                    "with_embeddings": 0,
+                    "without_embeddings": 0,
+                    "coverage_percent": 0,
+                    "error": str(e),
+                }
+
+        return {
+            "by_type": stats_by_type,
+            "totals": {
+                "total": total_items,
+                "with_embeddings": total_with_embeddings,
+                "without_embeddings": total_without_embeddings,
+                "coverage_percent": (
+                    round(total_with_embeddings / total_items * 100, 1)
+                    if total_items > 0
+                    else 0
+                ),
+            },
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to get embedding stats: {e}")
+        return {
+            "by_type": {},
+            "totals": {
+                "total": 0,
+                "with_embeddings": 0,
+                "without_embeddings": 0,
+                "coverage_percent": 0,
+            },
+            "error": str(e),
+        }
+
+
+async def backfill_missing_embeddings(batch_size: int = 10) -> dict[str, Any]:
+    """
+    Generate embeddings for approved listings that don't have them.
+
+    BACKWARD COMPATIBILITY: Maintained for existing usage.
+    This now delegates to backfill_all_content_types() to process all content types.
+
+    Args:
+        batch_size: Number of embeddings to generate per content type
+
+    Returns:
+        Dict with success/failure counts aggregated across all content types
+    """
+    # Delegate to the new generic backfill system
+    result = await backfill_all_content_types(batch_size)
+
+    # Return in the old format for backward compatibility
+    return result["totals"]
+
+
+async def backfill_all_content_types(batch_size: int = 10) -> dict[str, Any]:
+    """
+    Generate embeddings for all content types using registered handlers.
+
+    Processes content types in order: BLOCK → STORE_AGENT → DOCUMENTATION.
+    This ensures foundational content (blocks) are searchable first.
+
+    Args:
+        batch_size: Number of embeddings to generate per content type
+
+    Returns:
+        Dict with stats per content type and overall totals
+    """
+    results_by_type = {}
+    total_processed = 0
+    total_success = 0
+    total_failed = 0
+
+    # Process content types in explicit order
+    processing_order = [
+        ContentType.BLOCK,
+        ContentType.STORE_AGENT,
+        ContentType.DOCUMENTATION,
+    ]
+
+    for content_type in processing_order:
+        handler = CONTENT_HANDLERS.get(content_type)
+        if not handler:
+            logger.warning(f"No handler registered for {content_type.value}")
+            continue
+        try:
+            logger.info(f"Processing {content_type.value} content type...")
+
+            # Get missing items from handler
+            missing_items = await handler.get_missing_items(batch_size)
+
+            if not missing_items:
+                results_by_type[content_type.value] = {
+                    "processed": 0,
+                    "success": 0,
+                    "failed": 0,
+                    "message": "No missing embeddings",
+                }
+                continue
+
+            # Process embeddings concurrently for better performance
+            embedding_tasks = [
+                ensure_content_embedding(
+                    content_type=item.content_type,
+                    content_id=item.content_id,
+                    searchable_text=item.searchable_text,
+                    metadata=item.metadata,
+                    user_id=item.user_id,
+                )
+                for item in missing_items
+            ]
+
+            results = await asyncio.gather(*embedding_tasks, return_exceptions=True)
+
+            success = sum(1 for result in results if result is True)
+            failed = len(results) - success
+
+            results_by_type[content_type.value] = {
+                "processed": len(missing_items),
+                "success": success,
+                "failed": failed,
+                "message": f"Backfilled {success} embeddings, {failed} failed",
+            }
+
+            total_processed += len(missing_items)
+            total_success += success
+            total_failed += failed
+
+            logger.info(
+                f"{content_type.value}: processed {len(missing_items)}, "
+                f"success {success}, failed {failed}"
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to process {content_type.value}: {e}")
+            results_by_type[content_type.value] = {
+                "processed": 0,
+                "success": 0,
+                "failed": 0,
+                "error": str(e),
+            }
+
+    return {
+        "by_type": results_by_type,
+        "totals": {
+            "processed": total_processed,
+            "success": total_success,
+            "failed": total_failed,
+            "message": f"Overall: {total_success} succeeded, {total_failed} failed",
+        },
+    }
+
+
+async def embed_query(query: str) -> list[float] | None:
+    """
+    Generate embedding for a search query.
+
+    Same as generate_embedding but with clearer intent.
+    """
+    return await generate_embedding(query)
+
+
+def embedding_to_vector_string(embedding: list[float]) -> str:
+    """Convert embedding list to PostgreSQL vector string format."""
+    return "[" + ",".join(str(x) for x in embedding) + "]"
+
+
+async def ensure_content_embedding(
+    content_type: ContentType,
+    content_id: str,
+    searchable_text: str,
+    metadata: dict | None = None,
+    user_id: str | None = None,
+    force: bool = False,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Ensure an embedding exists for any content type.
+
+    Generic function for creating embeddings for store agents, blocks, docs, etc.
+
+    Args:
+        content_type: ContentType enum value (STORE_AGENT, BLOCK, etc.)
+        content_id: Unique identifier for the content
+        searchable_text: Combined text for embedding generation
+        metadata: Optional metadata to store with embedding
+        force: Force regeneration even if embedding exists
+        tx: Optional transaction client
+
+    Returns:
+        True if embedding exists/was created, False on failure
+    """
+    try:
+        # Check if embedding already exists
+        if not force:
+            existing = await get_content_embedding(content_type, content_id, user_id)
+            if existing and existing.get("embedding"):
+                logger.debug(
+                    f"Embedding for {content_type}:{content_id} already exists"
+                )
+                return True
+
+        # Generate new embedding
+        embedding = await generate_embedding(searchable_text)
+        if embedding is None:
+            logger.warning(
+                f"Could not generate embedding for {content_type}:{content_id}"
+            )
+            return False
+
+        # Store the embedding
+        return await store_content_embedding(
+            content_type=content_type,
+            content_id=content_id,
+            embedding=embedding,
+            searchable_text=searchable_text,
+            metadata=metadata or {},
+            user_id=user_id,
+            tx=tx,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to ensure embedding for {content_type}:{content_id}: {e}")
+        return False

autogpt_platform/backend/backend/api/features/store/embeddings_schema_test.py

@@ -0,0 +1,329 @@
+"""
+Integration tests for embeddings with schema handling.
+
+These tests verify that embeddings operations work correctly across different database schemas.
+"""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.api.features.store import embeddings
+
+# Schema prefix tests removed - functionality moved to db.raw_with_schema() helper
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_store_content_embedding_with_schema():
+    """Test storing embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.store_content_embedding(
+                content_type=ContentType.STORE_AGENT,
+                content_id="test-id",
+                embedding=[0.1] * 1536,
+                searchable_text="test text",
+                metadata={"test": "data"},
+                user_id=None,
+            )
+
+            # Verify the query was called
+            assert mock_client.execute_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.execute_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_get_content_embedding_with_schema():
+    """Test retrieving embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_client.query_raw.return_value = [
+                {
+                    "contentType": "STORE_AGENT",
+                    "contentId": "test-id",
+                    "userId": None,
+                    "embedding": "[0.1, 0.2]",
+                    "searchableText": "test",
+                    "metadata": {},
+                    "createdAt": "2024-01-01",
+                    "updatedAt": "2024-01-01",
+                }
+            ]
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.get_content_embedding(
+                ContentType.STORE_AGENT,
+                "test-id",
+                user_id=None,
+            )
+
+            # Verify the query was called
+            assert mock_client.query_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.query_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is not None
+            assert result["contentId"] == "test-id"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_delete_content_embedding_with_schema():
+    """Test deleting embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.delete_content_embedding(
+                ContentType.STORE_AGENT,
+                "test-id",
+            )
+
+            # Verify the query was called
+            assert mock_client.execute_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.execute_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_get_embedding_stats_with_schema():
+    """Test embedding statistics with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            # Mock both query results
+            mock_client.query_raw.side_effect = [
+                [{"count": 100}],  # total_approved
+                [{"count": 80}],  # with_embeddings
+            ]
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.get_embedding_stats()
+
+            # Verify both queries were called
+            assert mock_client.query_raw.call_count == 2
+
+            # Get both SQL queries
+            first_call = mock_client.query_raw.call_args_list[0]
+            second_call = mock_client.query_raw.call_args_list[1]
+
+            first_sql = first_call[0][0]
+            second_sql = second_call[0][0]
+
+            # Verify schema prefix in both queries
+            assert '"platform"."StoreListingVersion"' in first_sql
+            assert '"platform"."StoreListingVersion"' in second_sql
+            assert '"platform"."UnifiedContentEmbedding"' in second_sql
+
+            # Verify results
+            assert result["total_approved"] == 100
+            assert result["with_embeddings"] == 80
+            assert result["without_embeddings"] == 20
+            assert result["coverage_percent"] == 80.0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backfill_missing_embeddings_with_schema():
+    """Test backfilling embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            # Mock missing embeddings query
+            mock_client.query_raw.return_value = [
+                {
+                    "id": "version-1",
+                    "name": "Test Agent",
+                    "description": "Test description",
+                    "subHeading": "Test heading",
+                    "categories": ["test"],
+                }
+            ]
+            mock_get_client.return_value = mock_client
+
+            with patch(
+                "backend.api.features.store.embeddings.ensure_embedding"
+            ) as mock_ensure:
+                mock_ensure.return_value = True
+
+                result = await embeddings.backfill_missing_embeddings(batch_size=10)
+
+                # Verify the query was called
+                assert mock_client.query_raw.called
+
+                # Get the SQL query
+                call_args = mock_client.query_raw.call_args
+                sql_query = call_args[0][0]
+
+                # Verify schema prefix in query
+                assert '"platform"."StoreListingVersion"' in sql_query
+                assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+                # Verify ensure_embedding was called
+                assert mock_ensure.called
+
+                # Verify results
+                assert result["processed"] == 1
+                assert result["success"] == 1
+                assert result["failed"] == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_ensure_content_embedding_with_schema():
+    """Test ensuring embeddings exist with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch(
+            "backend.api.features.store.embeddings.get_content_embedding"
+        ) as mock_get:
+            # Simulate no existing embedding
+            mock_get.return_value = None
+
+            with patch(
+                "backend.api.features.store.embeddings.generate_embedding"
+            ) as mock_generate:
+                mock_generate.return_value = [0.1] * 1536
+
+                with patch(
+                    "backend.api.features.store.embeddings.store_content_embedding"
+                ) as mock_store:
+                    mock_store.return_value = True
+
+                    result = await embeddings.ensure_content_embedding(
+                        content_type=ContentType.STORE_AGENT,
+                        content_id="test-id",
+                        searchable_text="test text",
+                        metadata={"test": "data"},
+                        user_id=None,
+                        force=False,
+                    )
+
+                    # Verify the flow
+                    assert mock_get.called
+                    assert mock_generate.called
+                    assert mock_store.called
+                    assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backward_compatibility_store_embedding():
+    """Test backward compatibility wrapper for store_embedding."""
+    with patch(
+        "backend.api.features.store.embeddings.store_content_embedding"
+    ) as mock_store:
+        mock_store.return_value = True
+
+        result = await embeddings.store_embedding(
+            version_id="test-version-id",
+            embedding=[0.1] * 1536,
+            tx=None,
+        )
+
+        # Verify it calls the new function with correct parameters
+        assert mock_store.called
+        call_args = mock_store.call_args
+
+        assert call_args[1]["content_type"] == ContentType.STORE_AGENT
+        assert call_args[1]["content_id"] == "test-version-id"
+        assert call_args[1]["user_id"] is None
+        assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backward_compatibility_get_embedding():
+    """Test backward compatibility wrapper for get_embedding."""
+    with patch(
+        "backend.api.features.store.embeddings.get_content_embedding"
+    ) as mock_get:
+        mock_get.return_value = {
+            "contentType": "STORE_AGENT",
+            "contentId": "test-version-id",
+            "embedding": "[0.1, 0.2]",
+            "createdAt": "2024-01-01",
+            "updatedAt": "2024-01-01",
+        }
+
+        result = await embeddings.get_embedding("test-version-id")
+
+        # Verify it calls the new function
+        assert mock_get.called
+
+        # Verify it transforms to old format
+        assert result is not None
+        assert result["storeListingVersionId"] == "test-version-id"
+        assert "embedding" in result
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_schema_handling_error_cases():
+    """Test error handling in schema-aware operations."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_client.execute_raw.side_effect = Exception("Database error")
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.store_content_embedding(
+                content_type=ContentType.STORE_AGENT,
+                content_id="test-id",
+                embedding=[0.1] * 1536,
+                searchable_text="test",
+                metadata=None,
+                user_id=None,
+            )
+
+            # Should return False on error, not raise
+            assert result is False
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "-s"])

autogpt_platform/backend/backend/api/features/store/embeddings_test.py

@@ -0,0 +1,387 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import prisma
+import pytest
+from prisma import Prisma
+from prisma.enums import ContentType
+
+from backend.api.features.store import embeddings
+
+
+@pytest.fixture(autouse=True)
+async def setup_prisma():
+    """Setup Prisma client for tests."""
+    try:
+        Prisma()
+    except prisma.errors.ClientAlreadyRegisteredError:
+        pass
+    yield
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_build_searchable_text():
+    """Test searchable text building from listing fields."""
+    result = embeddings.build_searchable_text(
+        name="AI Assistant",
+        description="A helpful AI assistant for productivity",
+        sub_heading="Boost your productivity",
+        categories=["AI", "Productivity"],
+    )
+
+    expected = "AI Assistant Boost your productivity A helpful AI assistant for productivity AI Productivity"
+    assert result == expected
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_build_searchable_text_empty_fields():
+    """Test searchable text building with empty fields."""
+    result = embeddings.build_searchable_text(
+        name="", description="Test description", sub_heading="", categories=[]
+    )
+
+    assert result == "Test description"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_success():
+    """Test successful embedding generation."""
+    # Mock OpenAI response
+    mock_client = MagicMock()
+    mock_response = MagicMock()
+    mock_response.data = [MagicMock()]
+    mock_response.data[0].embedding = [0.1, 0.2, 0.3] * 512  # 1536 dimensions
+
+    # Use AsyncMock for async embeddings.create method
+    mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is not None
+        assert len(result) == 1536
+        assert result[0] == 0.1
+
+        mock_client.embeddings.create.assert_called_once_with(
+            model="text-embedding-3-small", input="test text"
+        )
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_no_api_key():
+    """Test embedding generation without API key."""
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = None
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_api_error():
+    """Test embedding generation with API error."""
+    mock_client = MagicMock()
+    mock_client.embeddings.create = AsyncMock(side_effect=Exception("API Error"))
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_text_truncation():
+    """Test that long text is properly truncated using tiktoken."""
+    from tiktoken import encoding_for_model
+
+    mock_client = MagicMock()
+    mock_response = MagicMock()
+    mock_response.data = [MagicMock()]
+    mock_response.data[0].embedding = [0.1] * 1536
+
+    # Use AsyncMock for async embeddings.create method
+    mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        # Create text that will exceed 8191 tokens
+        # Use varied characters to ensure token-heavy text: each word is ~1 token
+        words = [f"word{i}" for i in range(10000)]
+        long_text = " ".join(words)  # ~10000 tokens
+
+        await embeddings.generate_embedding(long_text)
+
+        # Verify text was truncated to 8191 tokens
+        call_args = mock_client.embeddings.create.call_args
+        truncated_text = call_args.kwargs["input"]
+
+        # Count actual tokens in truncated text
+        enc = encoding_for_model("text-embedding-3-small")
+        actual_tokens = len(enc.encode(truncated_text))
+
+        # Should be at or just under 8191 tokens
+        assert actual_tokens <= 8191
+        # Should be close to the limit (not over-truncated)
+        assert actual_tokens >= 8100
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_embedding_success(mocker):
+    """Test successful embedding storage."""
+    mock_client = mocker.AsyncMock()
+    mock_client.execute_raw = mocker.AsyncMock()
+
+    embedding = [0.1, 0.2, 0.3]
+
+    result = await embeddings.store_embedding(
+        version_id="test-version-id", embedding=embedding, tx=mock_client
+    )
+
+    assert result is True
+    # execute_raw is called twice: once for SET search_path, once for INSERT
+    assert mock_client.execute_raw.call_count == 2
+
+    # First call: SET search_path
+    first_call_args = mock_client.execute_raw.call_args_list[0][0]
+    assert "SET search_path" in first_call_args[0]
+
+    # Second call: INSERT query with the actual data
+    second_call_args = mock_client.execute_raw.call_args_list[1][0]
+    assert "test-version-id" in second_call_args
+    assert "[0.1,0.2,0.3]" in second_call_args
+    assert None in second_call_args  # userId should be None for store agents
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_embedding_database_error(mocker):
+    """Test embedding storage with database error."""
+    mock_client = mocker.AsyncMock()
+    mock_client.execute_raw.side_effect = Exception("Database error")
+
+    embedding = [0.1, 0.2, 0.3]
+
+    result = await embeddings.store_embedding(
+        version_id="test-version-id", embedding=embedding, tx=mock_client
+    )
+
+    assert result is False
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_success():
+    """Test successful embedding retrieval."""
+    mock_result = [
+        {
+            "contentType": "STORE_AGENT",
+            "contentId": "test-version-id",
+            "userId": None,
+            "embedding": "[0.1,0.2,0.3]",
+            "searchableText": "Test text",
+            "metadata": {},
+            "createdAt": "2024-01-01T00:00:00Z",
+            "updatedAt": "2024-01-01T00:00:00Z",
+        }
+    ]
+
+    with patch(
+        "backend.api.features.store.embeddings.query_raw_with_schema",
+        return_value=mock_result,
+    ):
+        result = await embeddings.get_embedding("test-version-id")
+
+        assert result is not None
+        assert result["storeListingVersionId"] == "test-version-id"
+        assert result["embedding"] == "[0.1,0.2,0.3]"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_not_found():
+    """Test embedding retrieval when not found."""
+    with patch(
+        "backend.api.features.store.embeddings.query_raw_with_schema",
+        return_value=[],
+    ):
+        result = await embeddings.get_embedding("test-version-id")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.store_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_already_exists(mock_get, mock_store, mock_generate):
+    """Test ensure_embedding when embedding already exists."""
+    mock_get.return_value = {"embedding": "[0.1,0.2,0.3]"}
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is True
+    mock_generate.assert_not_called()
+    mock_store.assert_not_called()
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.store_content_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_create_new(mock_get, mock_store, mock_generate):
+    """Test ensure_embedding creating new embedding."""
+    mock_get.return_value = None
+    mock_generate.return_value = [0.1, 0.2, 0.3]
+    mock_store.return_value = True
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is True
+    mock_generate.assert_called_once_with("Test Test heading Test description test")
+    mock_store.assert_called_once_with(
+        content_type=ContentType.STORE_AGENT,
+        content_id="test-id",
+        embedding=[0.1, 0.2, 0.3],
+        searchable_text="Test Test heading Test description test",
+        metadata={"name": "Test", "subHeading": "Test heading", "categories": ["test"]},
+        user_id=None,
+        tx=None,
+    )
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_generation_fails(mock_get, mock_generate):
+    """Test ensure_embedding when generation fails."""
+    mock_get.return_value = None
+    mock_generate.return_value = None
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is False
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_stats():
+    """Test embedding statistics retrieval."""
+    # Mock approved count query and embedded count query
+    mock_approved_result = [{"count": 100}]
+    mock_embedded_result = [{"count": 75}]
+
+    with patch(
+        "backend.api.features.store.embeddings.query_raw_with_schema",
+        side_effect=[mock_approved_result, mock_embedded_result],
+    ):
+        result = await embeddings.get_embedding_stats()
+
+        assert result["total_approved"] == 100
+        assert result["with_embeddings"] == 75
+        assert result["without_embeddings"] == 25
+        assert result["coverage_percent"] == 75.0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.ensure_embedding")
+async def test_backfill_missing_embeddings_success(mock_ensure):
+    """Test backfill with successful embedding generation."""
+    # Mock missing embeddings query
+    mock_missing = [
+        {
+            "id": "version-1",
+            "name": "Agent 1",
+            "description": "Description 1",
+            "subHeading": "Heading 1",
+            "categories": ["AI"],
+        },
+        {
+            "id": "version-2",
+            "name": "Agent 2",
+            "description": "Description 2",
+            "subHeading": "Heading 2",
+            "categories": ["Productivity"],
+        },
+    ]
+
+    # Mock ensure_embedding to succeed for first, fail for second
+    mock_ensure.side_effect = [True, False]
+
+    with patch(
+        "backend.api.features.store.embeddings.query_raw_with_schema",
+        return_value=mock_missing,
+    ):
+        result = await embeddings.backfill_missing_embeddings(batch_size=5)
+
+        assert result["processed"] == 2
+        assert result["success"] == 1
+        assert result["failed"] == 1
+        assert mock_ensure.call_count == 2
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_backfill_missing_embeddings_no_missing():
+    """Test backfill when no embeddings are missing."""
+    with patch(
+        "backend.api.features.store.embeddings.query_raw_with_schema",
+        return_value=[],
+    ):
+        result = await embeddings.backfill_missing_embeddings(batch_size=5)
+
+        assert result["processed"] == 0
+        assert result["success"] == 0
+        assert result["failed"] == 0
+        assert result["message"] == "No missing embeddings"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_embedding_to_vector_string():
+    """Test embedding to PostgreSQL vector string conversion."""
+    embedding = [0.1, 0.2, 0.3, -0.4]
+    result = embeddings.embedding_to_vector_string(embedding)
+    assert result == "[0.1,0.2,0.3,-0.4]"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_embed_query():
+    """Test embed_query function (alias for generate_embedding)."""
+    with patch(
+        "backend.api.features.store.embeddings.generate_embedding"
+    ) as mock_generate:
+        mock_generate.return_value = [0.1, 0.2, 0.3]
+
+        result = await embeddings.embed_query("test query")
+
+        assert result == [0.1, 0.2, 0.3]
+        mock_generate.assert_called_once_with("test query")

autogpt_platform/backend/backend/api/features/store/hybrid_search.py

@@ -0,0 +1,393 @@
+"""
+Hybrid Search for Store Agents
+
+Combines semantic (embedding) search with lexical (tsvector) search
+for improved relevance in marketplace agent discovery.
+"""
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+from backend.api.features.store.embeddings import (
+    embed_query,
+    embedding_to_vector_string,
+)
+from backend.data.db import query_raw_with_schema
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class HybridSearchWeights:
+    """Weights for combining search signals."""
+
+    semantic: float = 0.30  # Embedding cosine similarity
+    lexical: float = 0.30  # tsvector ts_rank_cd score
+    category: float = 0.20  # Category match boost
+    recency: float = 0.10  # Newer agents ranked higher
+    popularity: float = 0.10  # Agent usage/runs (PageRank-like)
+
+    def __post_init__(self):
+        """Validate weights are non-negative and sum to approximately 1.0."""
+        total = (
+            self.semantic
+            + self.lexical
+            + self.category
+            + self.recency
+            + self.popularity
+        )
+
+        if any(
+            w < 0
+            for w in [
+                self.semantic,
+                self.lexical,
+                self.category,
+                self.recency,
+                self.popularity,
+            ]
+        ):
+            raise ValueError("All weights must be non-negative")
+
+        if not (0.99 <= total <= 1.01):
+            raise ValueError(f"Weights must sum to ~1.0, got {total:.3f}")
+
+
+DEFAULT_WEIGHTS = HybridSearchWeights()
+
+# Minimum relevance score threshold - agents below this are filtered out
+# With weights (0.30 semantic + 0.30 lexical + 0.20 category + 0.10 recency + 0.10 popularity):
+# - 0.20 means at least ~60% semantic match OR strong lexical match required
+# - Ensures only genuinely relevant results are returned
+# - Recency/popularity alone (0.10 each) won't pass the threshold
+DEFAULT_MIN_SCORE = 0.20
+
+
+@dataclass
+class HybridSearchResult:
+    """A single search result with score breakdown."""
+
+    slug: str
+    agent_name: str
+    agent_image: str
+    creator_username: str
+    creator_avatar: str
+    sub_heading: str
+    description: str
+    runs: int
+    rating: float
+    categories: list[str]
+    featured: bool
+    is_available: bool
+    updated_at: datetime
+
+    # Score breakdown (for debugging/tuning)
+    combined_score: float
+    semantic_score: float = 0.0
+    lexical_score: float = 0.0
+    category_score: float = 0.0
+    recency_score: float = 0.0
+    popularity_score: float = 0.0
+
+
+async def hybrid_search(
+    query: str,
+    featured: bool = False,
+    creators: list[str] | None = None,
+    category: str | None = None,
+    sorted_by: (
+        Literal["relevance", "rating", "runs", "name", "updated_at"] | None
+    ) = None,
+    page: int = 1,
+    page_size: int = 20,
+    weights: HybridSearchWeights | None = None,
+    min_score: float | None = None,
+) -> tuple[list[dict[str, Any]], int]:
+    """
+    Perform hybrid search combining semantic and lexical signals.
+
+    Args:
+        query: Search query string
+        featured: Filter for featured agents only
+        creators: Filter by creator usernames
+        category: Filter by category
+        sorted_by: Sort order (relevance uses hybrid scoring)
+        page: Page number (1-indexed)
+        page_size: Results per page
+        weights: Custom weights for search signals
+        min_score: Minimum relevance score threshold (0-1). Results below
+                   this score are filtered out. Defaults to DEFAULT_MIN_SCORE.
+
+    Returns:
+        Tuple of (results list, total count). Returns empty list if no
+        results meet the minimum relevance threshold.
+    """
+    # Validate inputs
+    query = query.strip()
+    if not query:
+        return [], 0  # Empty query returns no results
+
+    if page < 1:
+        page = 1
+    if page_size < 1:
+        page_size = 1
+    if page_size > 100:  # Cap at reasonable limit to prevent performance issues
+        page_size = 100
+
+    if weights is None:
+        weights = DEFAULT_WEIGHTS
+    if min_score is None:
+        min_score = DEFAULT_MIN_SCORE
+
+    offset = (page - 1) * page_size
+
+    # Generate query embedding
+    query_embedding = await embed_query(query)
+
+    # Build WHERE clause conditions
+    where_parts: list[str] = ["sa.is_available = true"]
+    params: list[Any] = []
+    param_index = 1
+
+    # Add search query for lexical matching
+    params.append(query)
+    query_param = f"${param_index}"
+    param_index += 1
+
+    # Add lowercased query for category matching
+    params.append(query.lower())
+    query_lower_param = f"${param_index}"
+    param_index += 1
+
+    if featured:
+        where_parts.append("sa.featured = true")
+
+    if creators:
+        where_parts.append(f"sa.creator_username = ANY(${param_index})")
+        params.append(creators)
+        param_index += 1
+
+    if category:
+        where_parts.append(f"${param_index} = ANY(sa.categories)")
+        params.append(category)
+        param_index += 1
+
+    # Safe: where_parts only contains hardcoded strings with $N parameter placeholders
+    # No user input is concatenated directly into the SQL string
+    where_clause = " AND ".join(where_parts)
+
+    # Embedding is required for hybrid search - fail fast if unavailable
+    if query_embedding is None or not query_embedding:
+        # Log detailed error server-side
+        logger.error(
+            "Failed to generate query embedding. "
+            "Check that openai_internal_api_key is configured and OpenAI API is accessible."
+        )
+        # Raise generic error to client
+        raise ValueError("Search service temporarily unavailable")
+
+    # Add embedding parameter
+    embedding_str = embedding_to_vector_string(query_embedding)
+    params.append(embedding_str)
+    embedding_param = f"${param_index}"
+    param_index += 1
+
+    # Add weight parameters for SQL calculation
+    params.append(weights.semantic)
+    weight_semantic_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.lexical)
+    weight_lexical_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.category)
+    weight_category_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.recency)
+    weight_recency_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.popularity)
+    weight_popularity_param = f"${param_index}"
+    param_index += 1
+
+    # Add min_score parameter
+    params.append(min_score)
+    min_score_param = f"${param_index}"
+    param_index += 1
+
+    # Optimized hybrid search query:
+    # 1. Direct join to UnifiedContentEmbedding via contentId=storeListingVersionId (no redundant JOINs)
+    # 2. UNION approach (deduplicates agents matching both branches)
+    # 3. COUNT(*) OVER() to get total count in single query
+    # 4. Optimized category matching with EXISTS + unnest
+    # 5. Pre-calculated max values for lexical and popularity normalization
+    # 6. Simplified recency calculation with linear decay
+    # 7. Logarithmic popularity scaling to prevent viral agents from dominating
+    sql_query = f"""
+            WITH candidates AS (
+                -- Lexical matches (uses GIN index on search column)
+                SELECT sa."storeListingVersionId"
+                FROM {{schema_prefix}}"StoreAgent" sa
+                WHERE {where_clause}
+                AND sa.search @@ plainto_tsquery('english', {query_param})
+
+                UNION
+
+                -- Semantic matches (uses HNSW index on embedding with KNN)
+                SELECT "storeListingVersionId"
+                FROM (
+                    SELECT sa."storeListingVersionId", uce.embedding
+                    FROM {{schema_prefix}}"StoreAgent" sa
+                    INNER JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce
+                        ON sa."storeListingVersionId" = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{{schema_prefix}}"ContentType"
+                    WHERE {where_clause}
+                    ORDER BY uce.embedding <=> {embedding_param}::vector
+                    LIMIT 200
+                ) semantic_results
+            ),
+            search_scores AS (
+                SELECT
+                    sa.slug,
+                    sa.agent_name,
+                    sa.agent_image,
+                    sa.creator_username,
+                    sa.creator_avatar,
+                    sa.sub_heading,
+                    sa.description,
+                    sa.runs,
+                    sa.rating,
+                    sa.categories,
+                    sa.featured,
+                    sa.is_available,
+                    sa.updated_at,
+                    -- Semantic score: cosine similarity (1 - distance)
+                    COALESCE(1 - (uce.embedding <=> {embedding_param}::vector), 0) as semantic_score,
+                    -- Lexical score: ts_rank_cd (will be normalized later)
+                    COALESCE(ts_rank_cd(sa.search, plainto_tsquery('english', {query_param})), 0) as lexical_raw,
+                    -- Category match: optimized with unnest for better performance
+                    CASE
+                        WHEN EXISTS (
+                            SELECT 1 FROM unnest(sa.categories) cat
+                            WHERE LOWER(cat) LIKE '%' || {query_lower_param} || '%'
+                        )
+                        THEN 1.0
+                        ELSE 0.0
+                    END as category_score,
+                    -- Recency score: linear decay over 90 days (simpler than exponential)
+                    GREATEST(0, 1 - EXTRACT(EPOCH FROM (NOW() - sa.updated_at)) / (90 * 24 * 3600)) as recency_score,
+                    -- Popularity raw: agent runs count (will be normalized with log scaling)
+                    sa.runs as popularity_raw
+                FROM candidates c
+                INNER JOIN {{schema_prefix}}"StoreAgent" sa
+                    ON c."storeListingVersionId" = sa."storeListingVersionId"
+                LEFT JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce
+                    ON sa."storeListingVersionId" = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{{schema_prefix}}"ContentType"
+            ),
+            max_lexical AS (
+                SELECT MAX(lexical_raw) as max_val FROM search_scores
+            ),
+            max_popularity AS (
+                SELECT MAX(popularity_raw) as max_val FROM search_scores
+            ),
+            normalized AS (
+                SELECT
+                    ss.*,
+                    -- Normalize lexical score by pre-calculated max
+                    CASE
+                        WHEN ml.max_val > 0
+                        THEN ss.lexical_raw / ml.max_val
+                        ELSE 0
+                    END as lexical_score,
+                    -- Normalize popularity with logarithmic scaling to prevent viral agents from dominating
+                    -- LOG(1 + runs) / LOG(1 + max_runs) ensures score is 0-1 range
+                    CASE
+                        WHEN mp.max_val > 0 AND ss.popularity_raw > 0
+                        THEN LN(1 + ss.popularity_raw) / LN(1 + mp.max_val)
+                        ELSE 0
+                    END as popularity_score
+                FROM search_scores ss
+                CROSS JOIN max_lexical ml
+                CROSS JOIN max_popularity mp
+            ),
+            scored AS (
+                SELECT
+                    slug,
+                    agent_name,
+                    agent_image,
+                    creator_username,
+                    creator_avatar,
+                    sub_heading,
+                    description,
+                    runs,
+                    rating,
+                    categories,
+                    featured,
+                    is_available,
+                    updated_at,
+                    semantic_score,
+                    lexical_score,
+                    category_score,
+                    recency_score,
+                    popularity_score,
+                    (
+                        {weight_semantic_param} * semantic_score +
+                        {weight_lexical_param} * lexical_score +
+                        {weight_category_param} * category_score +
+                        {weight_recency_param} * recency_score +
+                        {weight_popularity_param} * popularity_score
+                    ) as combined_score
+                FROM normalized
+            ),
+            filtered AS (
+                SELECT
+                    *,
+                    COUNT(*) OVER () as total_count
+                FROM scored
+                WHERE combined_score >= {min_score_param}
+            )
+            SELECT * FROM filtered
+            ORDER BY combined_score DESC
+            LIMIT ${param_index} OFFSET ${param_index + 1}
+    """
+
+    # Add pagination params
+    params.extend([page_size, offset])
+
+    # Execute search query - includes total_count via window function
+    results = await query_raw_with_schema(
+        sql_query, *params, set_public_search_path=True
+    )
+
+    # Extract total count from first result (all rows have same count)
+    total = results[0]["total_count"] if results else 0
+
+    # Remove total_count from results before returning
+    for result in results:
+        result.pop("total_count", None)
+
+    # Log without sensitive query content
+    logger.info(f"Hybrid search: {len(results)} results, {total} total")
+
+    return results, total
+
+
+async def hybrid_search_simple(
+    query: str,
+    page: int = 1,
+    page_size: int = 20,
+) -> tuple[list[dict[str, Any]], int]:
+    """
+    Simplified hybrid search for common use cases.
+
+    Uses default weights and no filters.
+    """
+    return await hybrid_search(
+        query=query,
+        page=page,
+        page_size=page_size,
+    )

autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py

@@ -0,0 +1,334 @@
+"""
+Integration tests for hybrid search with schema handling.
+
+These tests verify that hybrid search works correctly across different database schemas.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from backend.api.features.store.hybrid_search import HybridSearchWeights, hybrid_search
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_schema_handling():
+    """Test that hybrid search correctly handles database schema prefixes."""
+    # Test with a mock query to ensure schema handling works
+    query = "test agent"
+
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Mock the query result
+        mock_query.return_value = [
+            {
+                "slug": "test/agent",
+                "agent_name": "Test Agent",
+                "agent_image": "test.png",
+                "creator_username": "test",
+                "creator_avatar": "avatar.png",
+                "sub_heading": "Test sub-heading",
+                "description": "Test description",
+                "runs": 10,
+                "rating": 4.5,
+                "categories": ["test"],
+                "featured": False,
+                "is_available": True,
+                "updated_at": "2024-01-01T00:00:00Z",
+                "combined_score": 0.8,
+                "semantic_score": 0.7,
+                "lexical_score": 0.6,
+                "category_score": 0.5,
+                "recency_score": 0.4,
+                "total_count": 1,
+            }
+        ]
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536  # Mock embedding
+
+            results, total = await hybrid_search(
+                query=query,
+                page=1,
+                page_size=20,
+            )
+
+            # Verify the query was called
+            assert mock_query.called
+            # Verify the SQL template uses schema_prefix placeholder
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            assert "{schema_prefix}" in sql_template
+
+            # Verify results
+            assert len(results) == 1
+            assert total == 1
+            assert results[0]["slug"] == "test/agent"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_public_schema():
+    """Test hybrid search when using public schema (no prefix needed)."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "public"
+
+        with patch(
+            "backend.api.features.store.hybrid_search.query_raw_with_schema"
+        ) as mock_query:
+            mock_query.return_value = []
+
+            with patch(
+                "backend.api.features.store.hybrid_search.embed_query"
+            ) as mock_embed:
+                mock_embed.return_value = [0.1] * 1536
+
+                results, total = await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+                # Verify the mock was set up correctly
+                assert mock_schema.return_value == "public"
+
+                # Results should work even with empty results
+                assert results == []
+                assert total == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_custom_schema():
+    """Test hybrid search when using custom schema (e.g., 'platform')."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch(
+            "backend.api.features.store.hybrid_search.query_raw_with_schema"
+        ) as mock_query:
+            mock_query.return_value = []
+
+            with patch(
+                "backend.api.features.store.hybrid_search.embed_query"
+            ) as mock_embed:
+                mock_embed.return_value = [0.1] * 1536
+
+                results, total = await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+                # Verify the mock was set up correctly
+                assert mock_schema.return_value == "platform"
+
+                assert results == []
+                assert total == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_without_embeddings():
+    """Test hybrid search fails fast when embeddings are unavailable."""
+    # Patch where the function is used, not where it's defined
+    with patch("backend.api.features.store.hybrid_search.embed_query") as mock_embed:
+        # Simulate embedding failure
+        mock_embed.return_value = None
+
+        # Should raise ValueError with helpful message
+        with pytest.raises(ValueError) as exc_info:
+            await hybrid_search(
+                query="test",
+                page=1,
+                page_size=20,
+            )
+
+        # Verify error message is generic (doesn't leak implementation details)
+        assert "Search service temporarily unavailable" in str(exc_info.value)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_filters():
+    """Test hybrid search with various filters."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test with featured filter
+            results, total = await hybrid_search(
+                query="test",
+                featured=True,
+                creators=["user1", "user2"],
+                category="productivity",
+                page=1,
+                page_size=10,
+            )
+
+            # Verify filters were applied in the query
+            call_args = mock_query.call_args
+            params = call_args[0][1:]  # Skip SQL template
+
+            # Should have query, query_lower, creators array, category
+            assert len(params) >= 4
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_weights():
+    """Test hybrid search with custom weights."""
+    custom_weights = HybridSearchWeights(
+        semantic=0.5,
+        lexical=0.3,
+        category=0.1,
+        recency=0.1,
+        popularity=0.0,
+    )
+
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            results, total = await hybrid_search(
+                query="test",
+                weights=custom_weights,
+                page=1,
+                page_size=20,
+            )
+
+            # Verify custom weights were used in the query
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            params = call_args[0][1:]  # Get all parameters passed
+
+            # Check that SQL uses parameterized weights (not f-string interpolation)
+            assert "$" in sql_template  # Verify parameterization is used
+
+            # Check that custom weights are in the params
+            assert 0.5 in params  # semantic weight
+            assert 0.3 in params  # lexical weight
+            assert 0.1 in params  # category and recency weights
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_min_score_filtering():
+    """Test hybrid search minimum score threshold."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Return results with varying scores
+        mock_query.return_value = [
+            {
+                "slug": "high-score/agent",
+                "agent_name": "High Score Agent",
+                "combined_score": 0.8,
+                "total_count": 1,
+                # ... other fields
+            }
+        ]
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test with custom min_score
+            results, total = await hybrid_search(
+                query="test",
+                min_score=0.5,  # High threshold
+                page=1,
+                page_size=20,
+            )
+
+            # Verify min_score was applied in query
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            params = call_args[0][1:]  # Get all parameters
+
+            # Check that SQL uses parameterized min_score
+            assert "combined_score >=" in sql_template
+            assert "$" in sql_template  # Verify parameterization
+
+            # Check that custom min_score is in the params
+            assert 0.5 in params
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_pagination():
+    """Test hybrid search pagination."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test page 2 with page_size 10
+            results, total = await hybrid_search(
+                query="test",
+                page=2,
+                page_size=10,
+            )
+
+            # Verify pagination parameters
+            call_args = mock_query.call_args
+            params = call_args[0]
+
+            # Last two params should be LIMIT and OFFSET
+            limit = params[-2]
+            offset = params[-1]
+
+            assert limit == 10  # page_size
+            assert offset == 10  # (page - 1) * page_size = (2 - 1) * 10
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_error_handling():
+    """Test hybrid search error handling."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Simulate database error
+        mock_query.side_effect = Exception("Database connection error")
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Should raise exception
+            with pytest.raises(Exception) as exc_info:
+                await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+            assert "Database connection error" in str(exc_info.value)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "-s"])

autogpt_platform/backend/backend/api/features/store/model.py

@@ -110,6 +110,7 @@ class Profile(pydantic.BaseModel):
 
 
 class StoreSubmission(pydantic.BaseModel):
+    listing_id: str
     agent_id: str
     agent_version: int
     name: str
@@ -164,8 +165,12 @@ class StoreListingsWithVersionsResponse(pydantic.BaseModel):
 
 
 class StoreSubmissionRequest(pydantic.BaseModel):
-    agent_id: str
-    agent_version: int
+    agent_id: str = pydantic.Field(
+        ..., min_length=1, description="Agent ID cannot be empty"
+    )
+    agent_version: int = pydantic.Field(
+        ..., gt=0, description="Agent version must be greater than 0"
+    )
     slug: str
     name: str
     sub_heading: str

autogpt_platform/backend/backend/api/features/store/model_test.py

@@ -138,6 +138,7 @@ def test_creator_details():
 
 def test_store_submission():
     submission = store_model.StoreSubmission(
+        listing_id="listing123",
         agent_id="agent123",
         agent_version=1,
         sub_heading="Test subheading",
@@ -159,6 +160,7 @@ def test_store_submissions_response():
     response = store_model.StoreSubmissionsResponse(
         submissions=[
             store_model.StoreSubmission(
+                listing_id="listing123",
                 agent_id="agent123",
                 agent_version=1,
                 sub_heading="Test subheading",

autogpt_platform/backend/backend/api/features/store/routes.py

@@ -294,7 +294,6 @@ async def get_creators(
 @router.get(
     "/creator/{username}",
     summary="Get creator details",
-    operation_id="getV2GetCreatorDetails",
     tags=["store", "public"],
     response_model=store_model.CreatorDetails,
 )

autogpt_platform/backend/backend/api/features/store/routes_test.py

@@ -521,6 +521,7 @@ def test_get_submissions_success(
     mocked_value = store_model.StoreSubmissionsResponse(
         submissions=[
             store_model.StoreSubmission(
+                listing_id="test-listing-id",
                 name="Test Agent",
                 description="Test agent description",
                 image_urls=["test.jpg"],

autogpt_platform/backend/backend/api/rest_api.py

@@ -18,7 +18,6 @@ from prisma.errors import PrismaError
 
 import backend.api.features.admin.credit_admin_routes
 import backend.api.features.admin.execution_analytics_routes
-import backend.api.features.admin.llm_routes
 import backend.api.features.admin.store_admin_routes
 import backend.api.features.builder
 import backend.api.features.builder.routes
@@ -38,11 +37,9 @@ import backend.data.db
 import backend.data.graph
 import backend.data.user
 import backend.integrations.webhooks.utils
-import backend.server.v2.llm.routes as public_llm_routes
 import backend.util.service
 import backend.util.settings
-from backend.data import llm_registry
-from backend.data.block_cost_config import refresh_llm_costs
+from backend.blocks.llm import DEFAULT_LLM_MODEL
 from backend.data.model import Credentials
 from backend.integrations.providers import ProviderName
 from backend.monitoring.instrumentation import instrument_fastapi
@@ -112,27 +109,11 @@ async def lifespan_context(app: fastapi.FastAPI):
 
     AutoRegistry.patch_integrations()
 
-    # Refresh LLM registry before initializing blocks so blocks can use registry data
-    await llm_registry.refresh_llm_registry()
-    refresh_llm_costs()
-
-    # Clear block schema caches so they're regenerated with updated discriminator_mapping
-    from backend.data.block import BlockSchema
-
-    BlockSchema.clear_all_schema_caches()
-
     await backend.data.block.initialize_blocks()
 
     await backend.data.user.migrate_and_encrypt_user_integrations()
     await backend.data.graph.fix_llm_provider_credentials()
-    # migrate_llm_models uses registry default model
-    from backend.blocks.llm import LlmModel
-
-    default_model_slug = llm_registry.get_default_model_slug()
-    if default_model_slug:
-        await backend.data.graph.migrate_llm_models(LlmModel(default_model_slug))
-    else:
-        logger.warning("Skipping LLM model migration: no default model available")
+    await backend.data.graph.migrate_llm_models(DEFAULT_LLM_MODEL)
     await backend.integrations.webhooks.utils.migrate_legacy_triggered_graphs()
 
     with launch_darkly_context():
@@ -317,16 +298,6 @@ app.include_router(
     tags=["v2", "executions", "review"],
     prefix="/api/review",
 )
-app.include_router(
-    backend.api.features.admin.llm_routes.router,
-    tags=["v2", "admin", "llm"],
-    prefix="/api/llm/admin",
-)
-app.include_router(
-    public_llm_routes.router,
-    tags=["v2", "llm"],
-    prefix="/api",
-)
 app.include_router(
     backend.api.features.library.routes.router, tags=["v2"], prefix="/api/library"
 )

autogpt_platform/backend/backend/api/ws_api.py

@@ -77,39 +77,7 @@ async def event_broadcaster(manager: ConnectionManager):
                 payload=notification.payload,
             )
 
-    async def registry_refresh_worker():
-        """Listen for LLM registry refresh notifications and broadcast to all clients."""
-        from backend.data.llm_registry import REGISTRY_REFRESH_CHANNEL
-        from backend.data.redis_client import connect_async
-
-        redis = await connect_async()
-        pubsub = redis.pubsub()
-        await pubsub.subscribe(REGISTRY_REFRESH_CHANNEL)
-        logger.info(
-            "Subscribed to LLM registry refresh notifications for WebSocket broadcast"
-        )
-
-        async for message in pubsub.listen():
-            if (
-                message["type"] == "message"
-                and message["channel"] == REGISTRY_REFRESH_CHANNEL
-            ):
-                logger.info(
-                    "Broadcasting LLM registry refresh to all WebSocket clients"
-                )
-                await manager.broadcast_to_all(
-                    method=WSMethod.NOTIFICATION,
-                    data={
-                        "type": "LLM_REGISTRY_REFRESH",
-                        "event": "registry_updated",
-                    },
-                )
-
-    await asyncio.gather(
-        execution_worker(),
-        notification_worker(),
-        registry_refresh_worker(),
-    )
+    await asyncio.gather(execution_worker(), notification_worker())
 
 
 async def authenticate_websocket(websocket: WebSocket) -> str:

autogpt_platform/backend/backend/blocks/ai_condition.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from backend.blocks.llm import (
+    DEFAULT_LLM_MODEL,
     TEST_CREDENTIALS,
     TEST_CREDENTIALS_INPUT,
     AIBlockBase,
@@ -9,7 +10,6 @@ from backend.blocks.llm import (
     LlmModel,
     LLMResponse,
     llm_call,
-    llm_model_schema_extra,
 )
 from backend.data.block import (
     BlockCategory,
@@ -50,10 +50,9 @@ class AIConditionBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for evaluating the condition.",
             advanced=False,
-            json_schema_extra=llm_model_schema_extra(),
         )
         credentials: AICredentials = AICredentialsField()
 
@@ -83,7 +82,7 @@ class AIConditionBlock(AIBlockBase):
                 "condition": "the input is an email address",
                 "yes_value": "Valid email",
                 "no_value": "Not an email",
-                "model": "gpt-4o",  # Using string value - enum accepts any model slug dynamically
+                "model": DEFAULT_LLM_MODEL,
                 "credentials": TEST_CREDENTIALS_INPUT,
             },
             test_credentials=TEST_CREDENTIALS,

autogpt_platform/backend/backend/blocks/airtable/_webhook.py

@@ -6,6 +6,9 @@ import hashlib
 import hmac
 import logging
 from enum import Enum
+from typing import cast
+
+from prisma.types import Serializable
 
 from backend.sdk import (
     BaseWebhooksManager,
@@ -84,7 +87,9 @@ class AirtableWebhookManager(BaseWebhooksManager):
         # update webhook config
         await update_webhook(
             webhook.id,
-            config={"base_id": base_id, "cursor": response.cursor},
+            config=cast(
+                dict[str, Serializable], {"base_id": base_id, "cursor": response.cursor}
+            ),
         )
 
         event_type = "notification"

autogpt_platform/backend/backend/blocks/helpers/review.py

@@ -0,0 +1,184 @@
+"""
+Shared helpers for Human-In-The-Loop (HITL) review functionality.
+Used by both the dedicated HumanInTheLoopBlock and blocks that require human review.
+"""
+
+import logging
+from typing import Any, Optional
+
+from prisma.enums import ReviewStatus
+from pydantic import BaseModel
+
+from backend.data.execution import ExecutionContext, ExecutionStatus
+from backend.data.human_review import ReviewResult
+from backend.executor.manager import async_update_node_execution_status
+from backend.util.clients import get_database_manager_async_client
+
+logger = logging.getLogger(__name__)
+
+
+class ReviewDecision(BaseModel):
+    """Result of a review decision."""
+
+    should_proceed: bool
+    message: str
+    review_result: ReviewResult
+
+
+class HITLReviewHelper:
+    """Helper class for Human-In-The-Loop review operations."""
+
+    @staticmethod
+    async def get_or_create_human_review(**kwargs) -> Optional[ReviewResult]:
+        """Create or retrieve a human review from the database."""
+        return await get_database_manager_async_client().get_or_create_human_review(
+            **kwargs
+        )
+
+    @staticmethod
+    async def update_node_execution_status(**kwargs) -> None:
+        """Update the execution status of a node."""
+        await async_update_node_execution_status(
+            db_client=get_database_manager_async_client(), **kwargs
+        )
+
+    @staticmethod
+    async def update_review_processed_status(
+        node_exec_id: str, processed: bool
+    ) -> None:
+        """Update the processed status of a review."""
+        return await get_database_manager_async_client().update_review_processed_status(
+            node_exec_id, processed
+        )
+
+    @staticmethod
+    async def _handle_review_request(
+        input_data: Any,
+        user_id: str,
+        node_exec_id: str,
+        graph_exec_id: str,
+        graph_id: str,
+        graph_version: int,
+        execution_context: ExecutionContext,
+        block_name: str = "Block",
+        editable: bool = False,
+    ) -> Optional[ReviewResult]:
+        """
+        Handle a review request for a block that requires human review.
+
+        Args:
+            input_data: The input data to be reviewed
+            user_id: ID of the user requesting the review
+            node_exec_id: ID of the node execution
+            graph_exec_id: ID of the graph execution
+            graph_id: ID of the graph
+            graph_version: Version of the graph
+            execution_context: Current execution context
+            block_name: Name of the block requesting review
+            editable: Whether the reviewer can edit the data
+
+        Returns:
+            ReviewResult if review is complete, None if waiting for human input
+
+        Raises:
+            Exception: If review creation or status update fails
+        """
+        # Skip review if safe mode is disabled - return auto-approved result
+        if not execution_context.safe_mode:
+            logger.info(
+                f"Block {block_name} skipping review for node {node_exec_id} - safe mode disabled"
+            )
+            return ReviewResult(
+                data=input_data,
+                status=ReviewStatus.APPROVED,
+                message="Auto-approved (safe mode disabled)",
+                processed=True,
+                node_exec_id=node_exec_id,
+            )
+
+        result = await HITLReviewHelper.get_or_create_human_review(
+            user_id=user_id,
+            node_exec_id=node_exec_id,
+            graph_exec_id=graph_exec_id,
+            graph_id=graph_id,
+            graph_version=graph_version,
+            input_data=input_data,
+            message=f"Review required for {block_name} execution",
+            editable=editable,
+        )
+
+        if result is None:
+            logger.info(
+                f"Block {block_name} pausing execution for node {node_exec_id} - awaiting human review"
+            )
+            await HITLReviewHelper.update_node_execution_status(
+                exec_id=node_exec_id,
+                status=ExecutionStatus.REVIEW,
+            )
+            return None  # Signal that execution should pause
+
+        # Mark review as processed if not already done
+        if not result.processed:
+            await HITLReviewHelper.update_review_processed_status(
+                node_exec_id=node_exec_id, processed=True
+            )
+
+        return result
+
+    @staticmethod
+    async def handle_review_decision(
+        input_data: Any,
+        user_id: str,
+        node_exec_id: str,
+        graph_exec_id: str,
+        graph_id: str,
+        graph_version: int,
+        execution_context: ExecutionContext,
+        block_name: str = "Block",
+        editable: bool = False,
+    ) -> Optional[ReviewDecision]:
+        """
+        Handle a review request and return the decision in a single call.
+
+        Args:
+            input_data: The input data to be reviewed
+            user_id: ID of the user requesting the review
+            node_exec_id: ID of the node execution
+            graph_exec_id: ID of the graph execution
+            graph_id: ID of the graph
+            graph_version: Version of the graph
+            execution_context: Current execution context
+            block_name: Name of the block requesting review
+            editable: Whether the reviewer can edit the data
+
+        Returns:
+            ReviewDecision if review is complete (approved/rejected),
+            None if execution should pause (awaiting review)
+        """
+        review_result = await HITLReviewHelper._handle_review_request(
+            input_data=input_data,
+            user_id=user_id,
+            node_exec_id=node_exec_id,
+            graph_exec_id=graph_exec_id,
+            graph_id=graph_id,
+            graph_version=graph_version,
+            execution_context=execution_context,
+            block_name=block_name,
+            editable=editable,
+        )
+
+        if review_result is None:
+            # Still awaiting review - return None to pause execution
+            return None
+
+        # Review is complete, determine outcome
+        should_proceed = review_result.status == ReviewStatus.APPROVED
+        message = review_result.message or (
+            "Execution approved by reviewer"
+            if should_proceed
+            else "Execution rejected by reviewer"
+        )
+
+        return ReviewDecision(
+            should_proceed=should_proceed, message=message, review_result=review_result
+        )

autogpt_platform/backend/backend/blocks/human_in_the_loop.py

@@ -3,6 +3,7 @@ from typing import Any
 
 from prisma.enums import ReviewStatus
 
+from backend.blocks.helpers.review import HITLReviewHelper
 from backend.data.block import (
     Block,
     BlockCategory,
@@ -11,11 +12,9 @@ from backend.data.block import (
     BlockSchemaOutput,
     BlockType,
 )
-from backend.data.execution import ExecutionContext, ExecutionStatus
+from backend.data.execution import ExecutionContext
 from backend.data.human_review import ReviewResult
 from backend.data.model import SchemaField
-from backend.executor.manager import async_update_node_execution_status
-from backend.util.clients import get_database_manager_async_client
 
 logger = logging.getLogger(__name__)
 
@@ -72,32 +71,26 @@ class HumanInTheLoopBlock(Block):
                 ("approved_data", {"name": "John Doe", "age": 30}),
             ],
             test_mock={
-                "get_or_create_human_review": lambda *_args, **_kwargs: ReviewResult(
-                    data={"name": "John Doe", "age": 30},
-                    status=ReviewStatus.APPROVED,
-                    message="",
-                    processed=False,
-                    node_exec_id="test-node-exec-id",
-                ),
-                "update_node_execution_status": lambda *_args, **_kwargs: None,
-                "update_review_processed_status": lambda *_args, **_kwargs: None,
+                "handle_review_decision": lambda **kwargs: type(
+                    "ReviewDecision",
+                    (),
+                    {
+                        "should_proceed": True,
+                        "message": "Test approval message",
+                        "review_result": ReviewResult(
+                            data={"name": "John Doe", "age": 30},
+                            status=ReviewStatus.APPROVED,
+                            message="",
+                            processed=False,
+                            node_exec_id="test-node-exec-id",
+                        ),
+                    },
+                )(),
             },
         )
 
-    async def get_or_create_human_review(self, **kwargs):
-        return await get_database_manager_async_client().get_or_create_human_review(
-            **kwargs
-        )
-
-    async def update_node_execution_status(self, **kwargs):
-        return await async_update_node_execution_status(
-            db_client=get_database_manager_async_client(), **kwargs
-        )
-
-    async def update_review_processed_status(self, node_exec_id: str, processed: bool):
-        return await get_database_manager_async_client().update_review_processed_status(
-            node_exec_id, processed
-        )
+    async def handle_review_decision(self, **kwargs):
+        return await HITLReviewHelper.handle_review_decision(**kwargs)
 
     async def run(
         self,
@@ -109,7 +102,7 @@ class HumanInTheLoopBlock(Block):
         graph_id: str,
         graph_version: int,
         execution_context: ExecutionContext,
-        **kwargs,
+        **_kwargs,
     ) -> BlockOutput:
         if not execution_context.safe_mode:
             logger.info(
@@ -119,48 +112,28 @@ class HumanInTheLoopBlock(Block):
             yield "review_message", "Auto-approved (safe mode disabled)"
             return
 
-        try:
-            result = await self.get_or_create_human_review(
-                user_id=user_id,
-                node_exec_id=node_exec_id,
-                graph_exec_id=graph_exec_id,
-                graph_id=graph_id,
-                graph_version=graph_version,
-                input_data=input_data.data,
-                message=input_data.name,
-                editable=input_data.editable,
-            )
-        except Exception as e:
-            logger.error(f"Error in HITL block for node {node_exec_id}: {str(e)}")
-            raise
+        decision = await self.handle_review_decision(
+            input_data=input_data.data,
+            user_id=user_id,
+            node_exec_id=node_exec_id,
+            graph_exec_id=graph_exec_id,
+            graph_id=graph_id,
+            graph_version=graph_version,
+            execution_context=execution_context,
+            block_name=self.name,
+            editable=input_data.editable,
+        )
 
-        if result is None:
-            logger.info(
-                f"HITL block pausing execution for node {node_exec_id} - awaiting human review"
-            )
-            try:
-                await self.update_node_execution_status(
-                    exec_id=node_exec_id,
-                    status=ExecutionStatus.REVIEW,
-                )
-                return
-            except Exception as e:
-                logger.error(
-                    f"Failed to update node status for HITL block {node_exec_id}: {str(e)}"
-                )
-                raise
+        if decision is None:
+            return
 
-        if not result.processed:
-            await self.update_review_processed_status(
-                node_exec_id=node_exec_id, processed=True
-            )
+        status = decision.review_result.status
+        if status == ReviewStatus.APPROVED:
+            yield "approved_data", decision.review_result.data
+        elif status == ReviewStatus.REJECTED:
+            yield "rejected_data", decision.review_result.data
+        else:
+            raise RuntimeError(f"Unexpected review status: {status}")
 
-            if result.status == ReviewStatus.APPROVED:
-                yield "approved_data", result.data
-                if result.message:
-                    yield "review_message", result.message
-
-            elif result.status == ReviewStatus.REJECTED:
-                yield "rejected_data", result.data
-                if result.message:
-                    yield "review_message", result.message
+        if decision.message:
+            yield "review_message", decision.message

autogpt_platform/backend/backend/blocks/llm.py

@@ -4,19 +4,17 @@ import logging
 import re
 import secrets
 from abc import ABC
-from enum import Enum
+from enum import Enum, EnumMeta
 from json import JSONDecodeError
-from typing import Any, Iterable, List, Literal, Optional
+from typing import Any, Iterable, List, Literal, NamedTuple, Optional
 
 import anthropic
 import ollama
 import openai
 from anthropic.types import ToolParam
 from groq import AsyncGroq
-from pydantic import BaseModel, GetCoreSchemaHandler, SecretStr
-from pydantic_core import CoreSchema, core_schema
+from pydantic import BaseModel, SecretStr
 
-from backend.data import llm_registry
 from backend.data.block import (
     Block,
     BlockCategory,
@@ -24,7 +22,6 @@ from backend.data.block import (
     BlockSchemaInput,
     BlockSchemaOutput,
 )
-from backend.data.llm_registry import ModelMetadata
 from backend.data.model import (
     APIKeyCredentials,
     CredentialsField,
@@ -69,117 +66,114 @@ TEST_CREDENTIALS_INPUT = {
 
 
 def AICredentialsField() -> AICredentials:
-    """
-    Returns a CredentialsField for LLM providers.
-    The discriminator_mapping will be refreshed when the schema is generated
-    if it's empty, ensuring the LLM registry is loaded.
-    """
-    # Get the mapping now - it may be empty initially, but will be refreshed
-    # when the schema is generated via CredentialsMetaInput._add_json_schema_extra
-    mapping = llm_registry.get_llm_discriminator_mapping()
-
     return CredentialsField(
         description="API key for the LLM provider.",
         discriminator="model",
-        discriminator_mapping=mapping,  # May be empty initially, refreshed later
+        discriminator_mapping={
+            model.value: model.metadata.provider for model in LlmModel
+        },
     )
 
 
-def llm_model_schema_extra() -> dict[str, Any]:
-    return {"options": llm_registry.get_llm_model_schema_options()}
+class ModelMetadata(NamedTuple):
+    provider: str
+    context_window: int
+    max_output_tokens: int | None
 
 
-class LlmModelMeta(type):
-    """
-    Metaclass for LlmModel that enables attribute-style access to dynamic models.
-
-    This allows code like `LlmModel.GPT4O` to work by converting the attribute
-    name to a slug format:
-    - GPT4O -> gpt-4o
-    - GPT4O_MINI -> gpt-4o-mini
-    - CLAUDE_3_5_SONNET -> claude-3-5-sonnet
-    """
-
-    def __getattr__(cls, name: str):
-        # Don't intercept private/dunder attributes
-        if name.startswith("_"):
-            raise AttributeError(f"type object 'LlmModel' has no attribute '{name}'")
-
-        # Convert attribute name to slug format:
-        # 1. Lowercase: GPT4O -> gpt4o
-        # 2. Underscores to hyphens: GPT4O_MINI -> gpt4o-mini
-        # 3. Insert hyphen between letter and digit: gpt4o -> gpt-4o
-        slug = name.lower().replace("_", "-")
-        slug = re.sub(r"([a-z])(\d)", r"\1-\2", slug)
-
-        return cls(slug)
+class LlmModelMeta(EnumMeta):
+    pass
 
 
-class LlmModel(str, metaclass=LlmModelMeta):
-    """
-    Dynamic LLM model type that accepts any model slug from the registry.
-
-    This is a string subclass (not an Enum) that allows any model slug value.
-    All models are managed via the LLM Registry in the database.
-
-    Usage:
-        model = LlmModel("gpt-4o")  # Direct construction
-        model = LlmModel.GPT4O      # Attribute access (converted to "gpt-4o")
-        model.value                  # Returns the slug string
-        model.provider               # Returns the provider from registry
-    """
-
-    def __new__(cls, value: str):
-        if isinstance(value, LlmModel):
-            return value
-        return str.__new__(cls, value)
-
-    @classmethod
-    def __get_pydantic_core_schema__(
-        cls, source_type: Any, handler: GetCoreSchemaHandler
-    ) -> CoreSchema:
-        """
-        Tell Pydantic how to validate LlmModel.
-
-        Accepts strings and converts them to LlmModel instances.
-        """
-        return core_schema.no_info_after_validator_function(
-            cls,  # The validator function (LlmModel constructor)
-            core_schema.str_schema(),  # Accept string input
-            serialization=core_schema.to_string_ser_schema(),  # Serialize as string
-        )
-
-    @property
-    def value(self) -> str:
-        """Return the model slug (for compatibility with enum-style access)."""
-        return str(self)
-
-    @classmethod
-    def default(cls) -> "LlmModel":
-        """
-        Get the default model from the registry.
-
-        Returns the recommended model if set, otherwise gpt-4o if available
-        and enabled, otherwise the first enabled model from the registry.
-        Falls back to "gpt-4o" if registry is empty (e.g., at module import time).
-        """
-        from backend.data.llm_registry import get_default_model_slug
-
-        slug = get_default_model_slug()
-        if slug is None:
-            # Registry is empty (e.g., at module import time before DB connection).
-            # Fall back to gpt-4o for backward compatibility.
-            slug = "gpt-4o"
-        return cls(slug)
+class LlmModel(str, Enum, metaclass=LlmModelMeta):
+    # OpenAI models
+    O3_MINI = "o3-mini"
+    O3 = "o3-2025-04-16"
+    O1 = "o1"
+    O1_MINI = "o1-mini"
+    # GPT-5 models
+    GPT5_2 = "gpt-5.2-2025-12-11"
+    GPT5_1 = "gpt-5.1-2025-11-13"
+    GPT5 = "gpt-5-2025-08-07"
+    GPT5_MINI = "gpt-5-mini-2025-08-07"
+    GPT5_NANO = "gpt-5-nano-2025-08-07"
+    GPT5_CHAT = "gpt-5-chat-latest"
+    GPT41 = "gpt-4.1-2025-04-14"
+    GPT41_MINI = "gpt-4.1-mini-2025-04-14"
+    GPT4O_MINI = "gpt-4o-mini"
+    GPT4O = "gpt-4o"
+    GPT4_TURBO = "gpt-4-turbo"
+    GPT3_5_TURBO = "gpt-3.5-turbo"
+    # Anthropic models
+    CLAUDE_4_1_OPUS = "claude-opus-4-1-20250805"
+    CLAUDE_4_OPUS = "claude-opus-4-20250514"
+    CLAUDE_4_SONNET = "claude-sonnet-4-20250514"
+    CLAUDE_4_5_OPUS = "claude-opus-4-5-20251101"
+    CLAUDE_4_5_SONNET = "claude-sonnet-4-5-20250929"
+    CLAUDE_4_5_HAIKU = "claude-haiku-4-5-20251001"
+    CLAUDE_3_7_SONNET = "claude-3-7-sonnet-20250219"
+    CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
+    # AI/ML API models
+    AIML_API_QWEN2_5_72B = "Qwen/Qwen2.5-72B-Instruct-Turbo"
+    AIML_API_LLAMA3_1_70B = "nvidia/llama-3.1-nemotron-70b-instruct"
+    AIML_API_LLAMA3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
+    AIML_API_META_LLAMA_3_1_70B = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"
+    AIML_API_LLAMA_3_2_3B = "meta-llama/Llama-3.2-3B-Instruct-Turbo"
+    # Groq models
+    LLAMA3_3_70B = "llama-3.3-70b-versatile"
+    LLAMA3_1_8B = "llama-3.1-8b-instant"
+    # Ollama models
+    OLLAMA_LLAMA3_3 = "llama3.3"
+    OLLAMA_LLAMA3_2 = "llama3.2"
+    OLLAMA_LLAMA3_8B = "llama3"
+    OLLAMA_LLAMA3_405B = "llama3.1:405b"
+    OLLAMA_DOLPHIN = "dolphin-mistral:latest"
+    # OpenRouter models
+    OPENAI_GPT_OSS_120B = "openai/gpt-oss-120b"
+    OPENAI_GPT_OSS_20B = "openai/gpt-oss-20b"
+    GEMINI_2_5_PRO = "google/gemini-2.5-pro-preview-03-25"
+    GEMINI_3_PRO_PREVIEW = "google/gemini-3-pro-preview"
+    GEMINI_2_5_FLASH = "google/gemini-2.5-flash"
+    GEMINI_2_0_FLASH = "google/gemini-2.0-flash-001"
+    GEMINI_2_5_FLASH_LITE_PREVIEW = "google/gemini-2.5-flash-lite-preview-06-17"
+    GEMINI_2_0_FLASH_LITE = "google/gemini-2.0-flash-lite-001"
+    MISTRAL_NEMO = "mistralai/mistral-nemo"
+    COHERE_COMMAND_R_08_2024 = "cohere/command-r-08-2024"
+    COHERE_COMMAND_R_PLUS_08_2024 = "cohere/command-r-plus-08-2024"
+    DEEPSEEK_CHAT = "deepseek/deepseek-chat"  # Actually: DeepSeek V3
+    DEEPSEEK_R1_0528 = "deepseek/deepseek-r1-0528"
+    PERPLEXITY_SONAR = "perplexity/sonar"
+    PERPLEXITY_SONAR_PRO = "perplexity/sonar-pro"
+    PERPLEXITY_SONAR_DEEP_RESEARCH = "perplexity/sonar-deep-research"
+    NOUSRESEARCH_HERMES_3_LLAMA_3_1_405B = "nousresearch/hermes-3-llama-3.1-405b"
+    NOUSRESEARCH_HERMES_3_LLAMA_3_1_70B = "nousresearch/hermes-3-llama-3.1-70b"
+    AMAZON_NOVA_LITE_V1 = "amazon/nova-lite-v1"
+    AMAZON_NOVA_MICRO_V1 = "amazon/nova-micro-v1"
+    AMAZON_NOVA_PRO_V1 = "amazon/nova-pro-v1"
+    MICROSOFT_WIZARDLM_2_8X22B = "microsoft/wizardlm-2-8x22b"
+    GRYPHE_MYTHOMAX_L2_13B = "gryphe/mythomax-l2-13b"
+    META_LLAMA_4_SCOUT = "meta-llama/llama-4-scout"
+    META_LLAMA_4_MAVERICK = "meta-llama/llama-4-maverick"
+    GROK_4 = "x-ai/grok-4"
+    GROK_4_FAST = "x-ai/grok-4-fast"
+    GROK_4_1_FAST = "x-ai/grok-4.1-fast"
+    GROK_CODE_FAST_1 = "x-ai/grok-code-fast-1"
+    KIMI_K2 = "moonshotai/kimi-k2"
+    QWEN3_235B_A22B_THINKING = "qwen/qwen3-235b-a22b-thinking-2507"
+    QWEN3_CODER = "qwen/qwen3-coder"
+    # Llama API models
+    LLAMA_API_LLAMA_4_SCOUT = "Llama-4-Scout-17B-16E-Instruct-FP8"
+    LLAMA_API_LLAMA4_MAVERICK = "Llama-4-Maverick-17B-128E-Instruct-FP8"
+    LLAMA_API_LLAMA3_3_8B = "Llama-3.3-8B-Instruct"
+    LLAMA_API_LLAMA3_3_70B = "Llama-3.3-70B-Instruct"
+    # v0 by Vercel models
+    V0_1_5_MD = "v0-1.5-md"
+    V0_1_5_LG = "v0-1.5-lg"
+    V0_1_0_MD = "v0-1.0-md"
 
     @property
     def metadata(self) -> ModelMetadata:
-        metadata = llm_registry.get_llm_model_metadata(self.value)
-        if metadata:
-            return metadata
-        raise ValueError(
-            f"Missing metadata for model: {self.value}. Model not found in LLM registry."
-        )
+        return MODEL_METADATA[self]
 
     @property
     def provider(self) -> str:
@@ -194,11 +188,128 @@ class LlmModel(str, metaclass=LlmModelMeta):
         return self.metadata.max_output_tokens
 
 
-# MODEL_METADATA removed - all models now come from the database via llm_registry
+MODEL_METADATA = {
+    # https://platform.openai.com/docs/models
+    LlmModel.O3: ModelMetadata("openai", 200000, 100000),
+    LlmModel.O3_MINI: ModelMetadata("openai", 200000, 100000),  # o3-mini-2025-01-31
+    LlmModel.O1: ModelMetadata("openai", 200000, 100000),  # o1-2024-12-17
+    LlmModel.O1_MINI: ModelMetadata("openai", 128000, 65536),  # o1-mini-2024-09-12
+    # GPT-5 models
+    LlmModel.GPT5_2: ModelMetadata("openai", 400000, 128000),
+    LlmModel.GPT5_1: ModelMetadata("openai", 400000, 128000),
+    LlmModel.GPT5: ModelMetadata("openai", 400000, 128000),
+    LlmModel.GPT5_MINI: ModelMetadata("openai", 400000, 128000),
+    LlmModel.GPT5_NANO: ModelMetadata("openai", 400000, 128000),
+    LlmModel.GPT5_CHAT: ModelMetadata("openai", 400000, 16384),
+    LlmModel.GPT41: ModelMetadata("openai", 1047576, 32768),
+    LlmModel.GPT41_MINI: ModelMetadata("openai", 1047576, 32768),
+    LlmModel.GPT4O_MINI: ModelMetadata(
+        "openai", 128000, 16384
+    ),  # gpt-4o-mini-2024-07-18
+    LlmModel.GPT4O: ModelMetadata("openai", 128000, 16384),  # gpt-4o-2024-08-06
+    LlmModel.GPT4_TURBO: ModelMetadata(
+        "openai", 128000, 4096
+    ),  # gpt-4-turbo-2024-04-09
+    LlmModel.GPT3_5_TURBO: ModelMetadata("openai", 16385, 4096),  # gpt-3.5-turbo-0125
+    # https://docs.anthropic.com/en/docs/about-claude/models
+    LlmModel.CLAUDE_4_1_OPUS: ModelMetadata(
+        "anthropic", 200000, 32000
+    ),  # claude-opus-4-1-20250805
+    LlmModel.CLAUDE_4_OPUS: ModelMetadata(
+        "anthropic", 200000, 32000
+    ),  # claude-4-opus-20250514
+    LlmModel.CLAUDE_4_SONNET: ModelMetadata(
+        "anthropic", 200000, 64000
+    ),  # claude-4-sonnet-20250514
+    LlmModel.CLAUDE_4_5_OPUS: ModelMetadata(
+        "anthropic", 200000, 64000
+    ),  # claude-opus-4-5-20251101
+    LlmModel.CLAUDE_4_5_SONNET: ModelMetadata(
+        "anthropic", 200000, 64000
+    ),  # claude-sonnet-4-5-20250929
+    LlmModel.CLAUDE_4_5_HAIKU: ModelMetadata(
+        "anthropic", 200000, 64000
+    ),  # claude-haiku-4-5-20251001
+    LlmModel.CLAUDE_3_7_SONNET: ModelMetadata(
+        "anthropic", 200000, 64000
+    ),  # claude-3-7-sonnet-20250219
+    LlmModel.CLAUDE_3_HAIKU: ModelMetadata(
+        "anthropic", 200000, 4096
+    ),  # claude-3-haiku-20240307
+    # https://docs.aimlapi.com/api-overview/model-database/text-models
+    LlmModel.AIML_API_QWEN2_5_72B: ModelMetadata("aiml_api", 32000, 8000),
+    LlmModel.AIML_API_LLAMA3_1_70B: ModelMetadata("aiml_api", 128000, 40000),
+    LlmModel.AIML_API_LLAMA3_3_70B: ModelMetadata("aiml_api", 128000, None),
+    LlmModel.AIML_API_META_LLAMA_3_1_70B: ModelMetadata("aiml_api", 131000, 2000),
+    LlmModel.AIML_API_LLAMA_3_2_3B: ModelMetadata("aiml_api", 128000, None),
+    # https://console.groq.com/docs/models
+    LlmModel.LLAMA3_3_70B: ModelMetadata("groq", 128000, 32768),
+    LlmModel.LLAMA3_1_8B: ModelMetadata("groq", 128000, 8192),
+    # https://ollama.com/library
+    LlmModel.OLLAMA_LLAMA3_3: ModelMetadata("ollama", 8192, None),
+    LlmModel.OLLAMA_LLAMA3_2: ModelMetadata("ollama", 8192, None),
+    LlmModel.OLLAMA_LLAMA3_8B: ModelMetadata("ollama", 8192, None),
+    LlmModel.OLLAMA_LLAMA3_405B: ModelMetadata("ollama", 8192, None),
+    LlmModel.OLLAMA_DOLPHIN: ModelMetadata("ollama", 32768, None),
+    # https://openrouter.ai/models
+    LlmModel.GEMINI_2_5_PRO: ModelMetadata("open_router", 1050000, 8192),
+    LlmModel.GEMINI_3_PRO_PREVIEW: ModelMetadata("open_router", 1048576, 65535),
+    LlmModel.GEMINI_2_5_FLASH: ModelMetadata("open_router", 1048576, 65535),
+    LlmModel.GEMINI_2_0_FLASH: ModelMetadata("open_router", 1048576, 8192),
+    LlmModel.GEMINI_2_5_FLASH_LITE_PREVIEW: ModelMetadata(
+        "open_router", 1048576, 65535
+    ),
+    LlmModel.GEMINI_2_0_FLASH_LITE: ModelMetadata("open_router", 1048576, 8192),
+    LlmModel.MISTRAL_NEMO: ModelMetadata("open_router", 128000, 4096),
+    LlmModel.COHERE_COMMAND_R_08_2024: ModelMetadata("open_router", 128000, 4096),
+    LlmModel.COHERE_COMMAND_R_PLUS_08_2024: ModelMetadata("open_router", 128000, 4096),
+    LlmModel.DEEPSEEK_CHAT: ModelMetadata("open_router", 64000, 2048),
+    LlmModel.DEEPSEEK_R1_0528: ModelMetadata("open_router", 163840, 163840),
+    LlmModel.PERPLEXITY_SONAR: ModelMetadata("open_router", 127000, 8000),
+    LlmModel.PERPLEXITY_SONAR_PRO: ModelMetadata("open_router", 200000, 8000),
+    LlmModel.PERPLEXITY_SONAR_DEEP_RESEARCH: ModelMetadata(
+        "open_router",
+        128000,
+        16000,
+    ),
+    LlmModel.NOUSRESEARCH_HERMES_3_LLAMA_3_1_405B: ModelMetadata(
+        "open_router", 131000, 4096
+    ),
+    LlmModel.NOUSRESEARCH_HERMES_3_LLAMA_3_1_70B: ModelMetadata(
+        "open_router", 12288, 12288
+    ),
+    LlmModel.OPENAI_GPT_OSS_120B: ModelMetadata("open_router", 131072, 131072),
+    LlmModel.OPENAI_GPT_OSS_20B: ModelMetadata("open_router", 131072, 32768),
+    LlmModel.AMAZON_NOVA_LITE_V1: ModelMetadata("open_router", 300000, 5120),
+    LlmModel.AMAZON_NOVA_MICRO_V1: ModelMetadata("open_router", 128000, 5120),
+    LlmModel.AMAZON_NOVA_PRO_V1: ModelMetadata("open_router", 300000, 5120),
+    LlmModel.MICROSOFT_WIZARDLM_2_8X22B: ModelMetadata("open_router", 65536, 4096),
+    LlmModel.GRYPHE_MYTHOMAX_L2_13B: ModelMetadata("open_router", 4096, 4096),
+    LlmModel.META_LLAMA_4_SCOUT: ModelMetadata("open_router", 131072, 131072),
+    LlmModel.META_LLAMA_4_MAVERICK: ModelMetadata("open_router", 1048576, 1000000),
+    LlmModel.GROK_4: ModelMetadata("open_router", 256000, 256000),
+    LlmModel.GROK_4_FAST: ModelMetadata("open_router", 2000000, 30000),
+    LlmModel.GROK_4_1_FAST: ModelMetadata("open_router", 2000000, 30000),
+    LlmModel.GROK_CODE_FAST_1: ModelMetadata("open_router", 256000, 10000),
+    LlmModel.KIMI_K2: ModelMetadata("open_router", 131000, 131000),
+    LlmModel.QWEN3_235B_A22B_THINKING: ModelMetadata("open_router", 262144, 262144),
+    LlmModel.QWEN3_CODER: ModelMetadata("open_router", 262144, 262144),
+    # Llama API models
+    LlmModel.LLAMA_API_LLAMA_4_SCOUT: ModelMetadata("llama_api", 128000, 4028),
+    LlmModel.LLAMA_API_LLAMA4_MAVERICK: ModelMetadata("llama_api", 128000, 4028),
+    LlmModel.LLAMA_API_LLAMA3_3_8B: ModelMetadata("llama_api", 128000, 4028),
+    LlmModel.LLAMA_API_LLAMA3_3_70B: ModelMetadata("llama_api", 128000, 4028),
+    # v0 by Vercel models
+    LlmModel.V0_1_5_MD: ModelMetadata("v0", 128000, 64000),
+    LlmModel.V0_1_5_LG: ModelMetadata("v0", 512000, 64000),
+    LlmModel.V0_1_0_MD: ModelMetadata("v0", 128000, 64000),
+}
 
-# Default model constant for backward compatibility
-# Uses the dynamic registry to get the default model
-DEFAULT_LLM_MODEL = LlmModel.default()
+DEFAULT_LLM_MODEL = LlmModel.GPT5_2
+
+for model in LlmModel:
+    if model not in MODEL_METADATA:
+        raise ValueError(f"Missing MODEL_METADATA metadata for model: {model}")
 
 
 class ToolCall(BaseModel):
@@ -327,94 +438,19 @@ async def llm_call(
             - prompt_tokens: The number of tokens used in the prompt.
             - completion_tokens: The number of tokens used in the completion.
     """
-    # Get model metadata and check if enabled - with fallback support
-    # The model we'll actually use (may differ if original is disabled)
-    model_to_use = llm_model.value
-
-    # Check if model is in registry and if it's enabled
-    from backend.data.llm_registry import (
-        get_fallback_model_for_disabled,
-        get_model_info,
-    )
-
-    model_info = get_model_info(llm_model.value)
-
-    if model_info and not model_info.is_enabled:
-        # Model is disabled - try to find a fallback from the same provider
-        fallback = get_fallback_model_for_disabled(llm_model.value)
-        if fallback:
-            logger.warning(
-                f"Model '{llm_model.value}' is disabled. Using fallback model '{fallback.slug}' from the same provider ({fallback.metadata.provider})."
-            )
-            model_to_use = fallback.slug
-            # Use fallback model's metadata
-            provider = fallback.metadata.provider
-            context_window = fallback.metadata.context_window
-            model_max_output = fallback.metadata.max_output_tokens or int(2**15)
-        else:
-            # No fallback available - raise error
-            raise ValueError(
-                f"LLM model '{llm_model.value}' is disabled and no fallback model "
-                f"from the same provider is available. Please enable the model or "
-                f"select a different model in the block configuration."
-            )
-    else:
-        # Model is enabled or not in registry (legacy/static model)
-        try:
-            provider = llm_model.metadata.provider
-            context_window = llm_model.context_window
-            model_max_output = llm_model.max_output_tokens or int(2**15)
-        except ValueError:
-            # Model not in cache - try refreshing the registry once if we have DB access
-            logger.warning(f"Model {llm_model.value} not found in registry cache")
-
-            # Try refreshing the registry if we have database access
-            from backend.data.db import is_connected
-
-            if is_connected():
-                try:
-                    logger.info(
-                        f"Refreshing LLM registry and retrying lookup for {llm_model.value}"
-                    )
-                    await llm_registry.refresh_llm_registry()
-                    # Try again after refresh
-                    try:
-                        provider = llm_model.metadata.provider
-                        context_window = llm_model.context_window
-                        model_max_output = llm_model.max_output_tokens or int(2**15)
-                        logger.info(
-                            f"Successfully loaded model {llm_model.value} metadata after registry refresh"
-                        )
-                    except ValueError:
-                        # Still not found after refresh
-                        raise ValueError(
-                            f"LLM model '{llm_model.value}' not found in registry after refresh. "
-                            "Please ensure the model is added and enabled in the LLM registry via the admin UI."
-                        )
-                except Exception as refresh_exc:
-                    logger.error(f"Failed to refresh LLM registry: {refresh_exc}")
-                    raise ValueError(
-                        f"LLM model '{llm_model.value}' not found in registry and failed to refresh. "
-                        "Please ensure the model is added to the LLM registry via the admin UI."
-                    ) from refresh_exc
-            else:
-                # No DB access (e.g., in executor without direct DB connection)
-                # The registry should have been loaded on startup
-                raise ValueError(
-                    f"LLM model '{llm_model.value}' not found in registry cache. "
-                    "The registry may need to be refreshed. Please contact support or try again later."
-                )
+    provider = llm_model.metadata.provider
+    context_window = llm_model.context_window
 
     if compress_prompt_to_fit:
         prompt = compress_prompt(
             messages=prompt,
-            target_tokens=context_window // 2,
+            target_tokens=llm_model.context_window // 2,
             lossy_ok=True,
         )
 
     # Calculate available tokens based on context window and input length
     estimated_input_tokens = estimate_token_count(prompt)
-    # model_max_output already set above
+    model_max_output = llm_model.max_output_tokens or int(2**15)
     user_max = max_tokens or model_max_output
     available_tokens = max(context_window - estimated_input_tokens, 0)
     max_tokens = max(min(available_tokens, model_max_output, user_max), 1)
@@ -432,7 +468,7 @@ async def llm_call(
             response_format = {"type": "json_object"}
 
         response = await oai_client.chat.completions.create(
-            model=model_to_use,
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             response_format=response_format,  # type: ignore
             max_completion_tokens=max_tokens,
@@ -479,7 +515,7 @@ async def llm_call(
         )
         try:
             resp = await client.messages.create(
-                model=model_to_use,
+                model=llm_model.value,
                 system=sysprompt,
                 messages=messages,
                 max_tokens=max_tokens,
@@ -543,7 +579,7 @@ async def llm_call(
         client = AsyncGroq(api_key=credentials.api_key.get_secret_value())
         response_format = {"type": "json_object"} if force_json_output else None
         response = await client.chat.completions.create(
-            model=model_to_use,
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             response_format=response_format,  # type: ignore
             max_tokens=max_tokens,
@@ -565,7 +601,7 @@ async def llm_call(
         sys_messages = [p["content"] for p in prompt if p["role"] == "system"]
         usr_messages = [p["content"] for p in prompt if p["role"] != "system"]
         response = await client.generate(
-            model=model_to_use,
+            model=llm_model.value,
             prompt=f"{sys_messages}\n\n{usr_messages}",
             stream=False,
             options={"num_ctx": max_tokens},
@@ -595,7 +631,7 @@ async def llm_call(
                 "HTTP-Referer": "https://agpt.co",
                 "X-Title": "AutoGPT",
             },
-            model=model_to_use,
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             max_tokens=max_tokens,
             tools=tools_param,  # type: ignore
@@ -637,7 +673,7 @@ async def llm_call(
                 "HTTP-Referer": "https://agpt.co",
                 "X-Title": "AutoGPT",
             },
-            model=model_to_use,
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             max_tokens=max_tokens,
             tools=tools_param,  # type: ignore
@@ -664,7 +700,7 @@ async def llm_call(
             reasoning=reasoning,
         )
     elif provider == "aiml_api":
-        client = openai.AsyncOpenAI(
+        client = openai.OpenAI(
             base_url="https://api.aimlapi.com/v2",
             api_key=credentials.api_key.get_secret_value(),
             default_headers={
@@ -674,8 +710,8 @@ async def llm_call(
             },
         )
 
-        completion = await client.chat.completions.create(
-            model=model_to_use,
+        completion = client.chat.completions.create(
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             max_tokens=max_tokens,
         )
@@ -707,7 +743,7 @@ async def llm_call(
         )
 
         response = await client.chat.completions.create(
-            model=model_to_use,
+            model=llm_model.value,
             messages=prompt,  # type: ignore
             response_format=response_format,  # type: ignore
             max_tokens=max_tokens,
@@ -758,10 +794,9 @@ class AIStructuredResponseGeneratorBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for answering the prompt.",
             advanced=False,
-            json_schema_extra=llm_model_schema_extra(),
         )
         force_json_output: bool = SchemaField(
             title="Restrict LLM to pure JSON output",
@@ -824,7 +859,7 @@ class AIStructuredResponseGeneratorBlock(AIBlockBase):
             input_schema=AIStructuredResponseGeneratorBlock.Input,
             output_schema=AIStructuredResponseGeneratorBlock.Output,
             test_input={
-                "model": "gpt-4o",  # Using string value - enum accepts any model slug dynamically
+                "model": DEFAULT_LLM_MODEL,
                 "credentials": TEST_CREDENTIALS_INPUT,
                 "expected_format": {
                     "key1": "value1",
@@ -1190,10 +1225,9 @@ class AITextGeneratorBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for answering the prompt.",
             advanced=False,
-            json_schema_extra=llm_model_schema_extra(),
         )
         credentials: AICredentials = AICredentialsField()
         sys_prompt: str = SchemaField(
@@ -1287,9 +1321,8 @@ class AITextSummarizerBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for summarizing the text.",
-            json_schema_extra=llm_model_schema_extra(),
         )
         focus: str = SchemaField(
             title="Focus",
@@ -1505,9 +1538,8 @@ class AIConversationBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for the conversation.",
-            json_schema_extra=llm_model_schema_extra(),
         )
         credentials: AICredentials = AICredentialsField()
         max_tokens: int | None = SchemaField(
@@ -1544,7 +1576,7 @@ class AIConversationBlock(AIBlockBase):
                     },
                     {"role": "user", "content": "Where was it played?"},
                 ],
-                "model": "gpt-4o",  # Using string value - enum accepts any model slug dynamically
+                "model": DEFAULT_LLM_MODEL,
                 "credentials": TEST_CREDENTIALS_INPUT,
             },
             test_credentials=TEST_CREDENTIALS,
@@ -1607,10 +1639,9 @@ class AIListGeneratorBlock(AIBlockBase):
         )
         model: LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=LlmModel.default,
+            default=DEFAULT_LLM_MODEL,
             description="The language model to use for generating the list.",
             advanced=True,
-            json_schema_extra=llm_model_schema_extra(),
         )
         credentials: AICredentials = AICredentialsField()
         max_retries: int = SchemaField(
@@ -1665,7 +1696,7 @@ class AIListGeneratorBlock(AIBlockBase):
                     "drawing explorers to uncover its mysteries. Each planet showcases the limitless possibilities of "
                     "fictional worlds."
                 ),
-                "model": "gpt-4o",  # Using string value - enum accepts any model slug dynamically
+                "model": DEFAULT_LLM_MODEL,
                 "credentials": TEST_CREDENTIALS_INPUT,
                 "max_retries": 3,
                 "force_json_output": False,

autogpt_platform/backend/backend/blocks/search.py

@@ -18,6 +18,7 @@ from backend.data.model import (
     SchemaField,
 )
 from backend.integrations.providers import ProviderName
+from backend.util.request import DEFAULT_USER_AGENT
 
 
 class GetWikipediaSummaryBlock(Block, GetRequest):
@@ -39,17 +40,27 @@ class GetWikipediaSummaryBlock(Block, GetRequest):
             output_schema=GetWikipediaSummaryBlock.Output,
             test_input={"topic": "Artificial Intelligence"},
             test_output=("summary", "summary content"),
-            test_mock={"get_request": lambda url, json: {"extract": "summary content"}},
+            test_mock={
+                "get_request": lambda url, headers, json: {"extract": "summary content"}
+            },
         )
 
     async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         topic = input_data.topic
-        url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{topic}"
+        # URL-encode the topic to handle spaces and special characters
+        encoded_topic = quote(topic, safe="")
+        url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{encoded_topic}"
+
+        # Set headers per Wikimedia robot policy (https://w.wiki/4wJS)
+        # - User-Agent: Required, must identify the bot
+        # - Accept-Encoding: gzip recommended to reduce bandwidth
+        headers = {
+            "User-Agent": DEFAULT_USER_AGENT,
+            "Accept-Encoding": "gzip, deflate",
+        }
 
-        # Note: User-Agent is now automatically set by the request library
-        # to comply with Wikimedia's robot policy (https://w.wiki/4wJS)
         try:
-            response = await self.get_request(url, json=True)
+            response = await self.get_request(url, headers=headers, json=True)
             if "extract" not in response:
                 raise ValueError(f"Unable to parse Wikipedia response: {response}")
             yield "summary", response["extract"]

autogpt_platform/backend/backend/blocks/smart_decision_maker.py

@@ -226,10 +226,9 @@ class SmartDecisionMakerBlock(Block):
         )
         model: llm.LlmModel = SchemaField(
             title="LLM Model",
-            default_factory=llm.LlmModel.default,
+            default=llm.DEFAULT_LLM_MODEL,
             description="The language model to use for answering the prompt.",
             advanced=False,
-            json_schema_extra=llm.llm_model_schema_extra(),
         )
         credentials: llm.AICredentials = llm.AICredentialsField()
         multiple_tool_calls: bool = SchemaField(
@@ -392,8 +391,12 @@ class SmartDecisionMakerBlock(Block):
         """
         block = sink_node.block
 
+        # Use custom name from node metadata if set, otherwise fall back to block.name
+        custom_name = sink_node.metadata.get("customized_name")
+        tool_name = custom_name if custom_name else block.name
+
         tool_function: dict[str, Any] = {
-            "name": SmartDecisionMakerBlock.cleanup(block.name),
+            "name": SmartDecisionMakerBlock.cleanup(tool_name),
             "description": block.description,
         }
         sink_block_input_schema = block.input_schema
@@ -490,14 +493,24 @@ class SmartDecisionMakerBlock(Block):
                 f"Sink graph metadata not found: {graph_id} {graph_version}"
             )
 
+        # Use custom name from node metadata if set, otherwise fall back to graph name
+        custom_name = sink_node.metadata.get("customized_name")
+        tool_name = custom_name if custom_name else sink_graph_meta.name
+
         tool_function: dict[str, Any] = {
-            "name": SmartDecisionMakerBlock.cleanup(sink_graph_meta.name),
+            "name": SmartDecisionMakerBlock.cleanup(tool_name),
             "description": sink_graph_meta.description,
         }
 
         properties = {}
+        field_mapping = {}
 
         for link in links:
+            field_name = link.sink_name
+
+            clean_field_name = SmartDecisionMakerBlock.cleanup(field_name)
+            field_mapping[clean_field_name] = field_name
+
             sink_block_input_schema = sink_node.input_default["input_schema"]
             sink_block_properties = sink_block_input_schema.get("properties", {}).get(
                 link.sink_name, {}
@@ -507,7 +520,7 @@ class SmartDecisionMakerBlock(Block):
                 if "description" in sink_block_properties
                 else f"The {link.sink_name} of the tool"
             )
-            properties[link.sink_name] = {
+            properties[clean_field_name] = {
                 "type": "string",
                 "description": description,
                 "default": json.dumps(sink_block_properties.get("default", None)),
@@ -520,7 +533,7 @@ class SmartDecisionMakerBlock(Block):
             "strict": True,
         }
 
-        # Store node info for later use in output processing
+        tool_function["_field_mapping"] = field_mapping
         tool_function["_sink_node_id"] = sink_node.id
 
         return {"type": "function", "function": tool_function}
@@ -976,10 +989,28 @@ class SmartDecisionMakerBlock(Block):
         graph_version: int,
         execution_context: ExecutionContext,
         execution_processor: "ExecutionProcessor",
+        nodes_to_skip: set[str] | None = None,
         **kwargs,
     ) -> BlockOutput:
 
         tool_functions = await self._create_tool_node_signatures(node_id)
+        original_tool_count = len(tool_functions)
+
+        # Filter out tools for nodes that should be skipped (e.g., missing optional credentials)
+        if nodes_to_skip:
+            tool_functions = [
+                tf
+                for tf in tool_functions
+                if tf.get("function", {}).get("_sink_node_id") not in nodes_to_skip
+            ]
+
+            # Only raise error if we had tools but they were all filtered out
+            if original_tool_count > 0 and not tool_functions:
+                raise ValueError(
+                    "No available tools to execute - all downstream nodes are unavailable "
+                    "(possibly due to missing optional credentials)"
+                )
+
         yield "tool_functions", json.dumps(tool_functions)
 
         conversation_history = input_data.conversation_history or []
@@ -1130,8 +1161,9 @@ class SmartDecisionMakerBlock(Block):
                 original_field_name = field_mapping.get(clean_arg_name, clean_arg_name)
                 arg_value = tool_args.get(clean_arg_name)
 
-                sanitized_arg_name = self.cleanup(original_field_name)
-                emit_key = f"tools_^_{sink_node_id}_~_{sanitized_arg_name}"
+                # Use original_field_name directly (not sanitized) to match link sink_name
+                # The field_mapping already translates from LLM's cleaned names to original names
+                emit_key = f"tools_^_{sink_node_id}_~_{original_field_name}"
 
                 logger.debug(
                     "[SmartDecisionMakerBlock|geid:%s|neid:%s] emit %s",

autogpt_platform/backend/backend/blocks/stagehand/blocks.py

@@ -10,13 +10,13 @@ import stagehand.main
 from stagehand import Stagehand
 
 from backend.blocks.llm import (
+    MODEL_METADATA,
     AICredentials,
     AICredentialsField,
     LlmModel,
     ModelMetadata,
 )
 from backend.blocks.stagehand._config import stagehand as stagehand_provider
-from backend.data import llm_registry
 from backend.sdk import (
     APIKeyCredentials,
     Block,
@@ -91,7 +91,7 @@ class StagehandRecommendedLlmModel(str, Enum):
         Returns the provider name for the model in the required format for Stagehand:
         provider/model_name
         """
-        model_metadata = self.metadata
+        model_metadata = MODEL_METADATA[LlmModel(self.value)]
         model_name = self.value
 
         if len(model_name.split("/")) == 1 and not self.value.startswith(
@@ -107,23 +107,19 @@ class StagehandRecommendedLlmModel(str, Enum):
 
     @property
     def provider(self) -> str:
-        return self.metadata.provider
+        return MODEL_METADATA[LlmModel(self.value)].provider
 
     @property
     def metadata(self) -> ModelMetadata:
-        metadata = llm_registry.get_llm_model_metadata(self.value)
-        if metadata:
-            return metadata
-        # Fallback to LlmModel enum if registry lookup fails
-        return LlmModel(self.value).metadata
+        return MODEL_METADATA[LlmModel(self.value)]
 
     @property
     def context_window(self) -> int:
-        return self.metadata.context_window
+        return MODEL_METADATA[LlmModel(self.value)].context_window
 
     @property
     def max_output_tokens(self) -> int | None:
-        return self.metadata.max_output_tokens
+        return MODEL_METADATA[LlmModel(self.value)].max_output_tokens
 
 
 class StagehandObserveBlock(Block):

autogpt_platform/backend/backend/blocks/test/test_blocks_dos_vulnerability.py

@@ -196,6 +196,15 @@ class TestXMLParserBlockSecurity:
             async for _ in block.run(XMLParserBlock.Input(input_xml=large_xml)):
                 pass
 
+    async def test_rejects_text_outside_root(self):
+        """Ensure parser surfaces readable errors for invalid root text."""
+        block = XMLParserBlock()
+        invalid_xml = "<root><child>value</child></root> trailing"
+
+        with pytest.raises(ValueError, match="text outside the root element"):
+            async for _ in block.run(XMLParserBlock.Input(input_xml=invalid_xml)):
+                pass
+
 
 class TestStoreMediaFileSecurity:
     """Test file storage security limits."""

autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py

@@ -1057,3 +1057,153 @@ async def test_smart_decision_maker_traditional_mode_default():
         )  # Should yield individual tool parameters
         assert "tools_^_test-sink-node-id_~_max_keyword_difficulty" in outputs
         assert "conversations" in outputs
+
+
+@pytest.mark.asyncio
+async def test_smart_decision_maker_uses_customized_name_for_blocks():
+    """Test that SmartDecisionMakerBlock uses customized_name from node metadata for tool names."""
+    from unittest.mock import MagicMock
+
+    from backend.blocks.basic import StoreValueBlock
+    from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
+    from backend.data.graph import Link, Node
+
+    # Create a mock node with customized_name in metadata
+    mock_node = MagicMock(spec=Node)
+    mock_node.id = "test-node-id"
+    mock_node.block_id = StoreValueBlock().id
+    mock_node.metadata = {"customized_name": "My Custom Tool Name"}
+    mock_node.block = StoreValueBlock()
+
+    # Create a mock link
+    mock_link = MagicMock(spec=Link)
+    mock_link.sink_name = "input"
+
+    # Call the function directly
+    result = await SmartDecisionMakerBlock._create_block_function_signature(
+        mock_node, [mock_link]
+    )
+
+    # Verify the tool name uses the customized name (cleaned up)
+    assert result["type"] == "function"
+    assert result["function"]["name"] == "my_custom_tool_name"  # Cleaned version
+    assert result["function"]["_sink_node_id"] == "test-node-id"
+
+
+@pytest.mark.asyncio
+async def test_smart_decision_maker_falls_back_to_block_name():
+    """Test that SmartDecisionMakerBlock falls back to block.name when no customized_name."""
+    from unittest.mock import MagicMock
+
+    from backend.blocks.basic import StoreValueBlock
+    from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
+    from backend.data.graph import Link, Node
+
+    # Create a mock node without customized_name
+    mock_node = MagicMock(spec=Node)
+    mock_node.id = "test-node-id"
+    mock_node.block_id = StoreValueBlock().id
+    mock_node.metadata = {}  # No customized_name
+    mock_node.block = StoreValueBlock()
+
+    # Create a mock link
+    mock_link = MagicMock(spec=Link)
+    mock_link.sink_name = "input"
+
+    # Call the function directly
+    result = await SmartDecisionMakerBlock._create_block_function_signature(
+        mock_node, [mock_link]
+    )
+
+    # Verify the tool name uses the block's default name
+    assert result["type"] == "function"
+    assert result["function"]["name"] == "storevalueblock"  # Default block name cleaned
+    assert result["function"]["_sink_node_id"] == "test-node-id"
+
+
+@pytest.mark.asyncio
+async def test_smart_decision_maker_uses_customized_name_for_agents():
+    """Test that SmartDecisionMakerBlock uses customized_name from metadata for agent nodes."""
+    from unittest.mock import AsyncMock, MagicMock, patch
+
+    from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
+    from backend.data.graph import Link, Node
+
+    # Create a mock node with customized_name in metadata
+    mock_node = MagicMock(spec=Node)
+    mock_node.id = "test-agent-node-id"
+    mock_node.metadata = {"customized_name": "My Custom Agent"}
+    mock_node.input_default = {
+        "graph_id": "test-graph-id",
+        "graph_version": 1,
+        "input_schema": {"properties": {"test_input": {"description": "Test input"}}},
+    }
+
+    # Create a mock link
+    mock_link = MagicMock(spec=Link)
+    mock_link.sink_name = "test_input"
+
+    # Mock the database client
+    mock_graph_meta = MagicMock()
+    mock_graph_meta.name = "Original Agent Name"
+    mock_graph_meta.description = "Agent description"
+
+    mock_db_client = AsyncMock()
+    mock_db_client.get_graph_metadata.return_value = mock_graph_meta
+
+    with patch(
+        "backend.blocks.smart_decision_maker.get_database_manager_async_client",
+        return_value=mock_db_client,
+    ):
+        result = await SmartDecisionMakerBlock._create_agent_function_signature(
+            mock_node, [mock_link]
+        )
+
+    # Verify the tool name uses the customized name (cleaned up)
+    assert result["type"] == "function"
+    assert result["function"]["name"] == "my_custom_agent"  # Cleaned version
+    assert result["function"]["_sink_node_id"] == "test-agent-node-id"
+
+
+@pytest.mark.asyncio
+async def test_smart_decision_maker_agent_falls_back_to_graph_name():
+    """Test that agent node falls back to graph name when no customized_name."""
+    from unittest.mock import AsyncMock, MagicMock, patch
+
+    from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
+    from backend.data.graph import Link, Node
+
+    # Create a mock node without customized_name
+    mock_node = MagicMock(spec=Node)
+    mock_node.id = "test-agent-node-id"
+    mock_node.metadata = {}  # No customized_name
+    mock_node.input_default = {
+        "graph_id": "test-graph-id",
+        "graph_version": 1,
+        "input_schema": {"properties": {"test_input": {"description": "Test input"}}},
+    }
+
+    # Create a mock link
+    mock_link = MagicMock(spec=Link)
+    mock_link.sink_name = "test_input"
+
+    # Mock the database client
+    mock_graph_meta = MagicMock()
+    mock_graph_meta.name = "Original Agent Name"
+    mock_graph_meta.description = "Agent description"
+
+    mock_db_client = AsyncMock()
+    mock_db_client.get_graph_metadata.return_value = mock_graph_meta
+
+    with patch(
+        "backend.blocks.smart_decision_maker.get_database_manager_async_client",
+        return_value=mock_db_client,
+    ):
+        result = await SmartDecisionMakerBlock._create_agent_function_signature(
+            mock_node, [mock_link]
+        )
+
+    # Verify the tool name uses the graph's default name
+    assert result["type"] == "function"
+    assert result["function"]["name"] == "original_agent_name"  # Graph name cleaned
+    assert result["function"]["_sink_node_id"] == "test-agent-node-id"

autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker_dict.py

@@ -15,6 +15,7 @@ async def test_smart_decision_maker_handles_dynamic_dict_fields():
     mock_node.block = CreateDictionaryBlock()
     mock_node.block_id = CreateDictionaryBlock().id
     mock_node.input_default = {}
+    mock_node.metadata = {}
 
     # Create mock links with dynamic dictionary fields
     mock_links = [
@@ -77,6 +78,7 @@ async def test_smart_decision_maker_handles_dynamic_list_fields():
     mock_node.block = AddToListBlock()
     mock_node.block_id = AddToListBlock().id
     mock_node.input_default = {}
+    mock_node.metadata = {}
 
     # Create mock links with dynamic list fields
     mock_links = [

autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker_dynamic_fields.py

@@ -44,6 +44,7 @@ async def test_create_block_function_signature_with_dict_fields():
     mock_node.block = CreateDictionaryBlock()
     mock_node.block_id = CreateDictionaryBlock().id
     mock_node.input_default = {}
+    mock_node.metadata = {}
 
     # Create mock links with dynamic dictionary fields (source sanitized, sink original)
     mock_links = [
@@ -106,6 +107,7 @@ async def test_create_block_function_signature_with_list_fields():
     mock_node.block = AddToListBlock()
     mock_node.block_id = AddToListBlock().id
     mock_node.input_default = {}
+    mock_node.metadata = {}
 
     # Create mock links with dynamic list fields
     mock_links = [
@@ -159,6 +161,7 @@ async def test_create_block_function_signature_with_object_fields():
     mock_node.block = MatchTextPatternBlock()
     mock_node.block_id = MatchTextPatternBlock().id
     mock_node.input_default = {}
+    mock_node.metadata = {}
 
     # Create mock links with dynamic object fields
     mock_links = [
@@ -208,11 +211,13 @@ async def test_create_tool_node_signatures():
         mock_dict_node.block = CreateDictionaryBlock()
         mock_dict_node.block_id = CreateDictionaryBlock().id
         mock_dict_node.input_default = {}
+        mock_dict_node.metadata = {}
 
         mock_list_node = Mock()
         mock_list_node.block = AddToListBlock()
         mock_list_node.block_id = AddToListBlock().id
         mock_list_node.input_default = {}
+        mock_list_node.metadata = {}
 
         # Mock links with dynamic fields
         dict_link1 = Mock(
@@ -423,6 +428,7 @@ async def test_mixed_regular_and_dynamic_fields():
     mock_node.block.name = "TestBlock"
     mock_node.block.description = "A test block"
     mock_node.block.input_schema = Mock()
+    mock_node.metadata = {}
 
     # Mock the get_field_schema to return a proper schema for regular fields
     def get_field_schema(field_name):

autogpt_platform/backend/backend/blocks/wordpress/__init__.py

@@ -1,3 +1,3 @@
-from .blog import WordPressCreatePostBlock
+from .blog import WordPressCreatePostBlock, WordPressGetAllPostsBlock
 
-__all__ = ["WordPressCreatePostBlock"]
+__all__ = ["WordPressCreatePostBlock", "WordPressGetAllPostsBlock"]

autogpt_platform/backend/backend/blocks/wordpress/_api.py

@@ -161,7 +161,7 @@ async def oauth_exchange_code_for_tokens(
         grant_type="authorization_code",
     ).model_dump(exclude_none=True)
 
-    response = await Requests().post(
+    response = await Requests(raise_for_status=False).post(
         f"{WORDPRESS_BASE_URL}oauth2/token",
         headers=headers,
         data=data,
@@ -205,7 +205,7 @@ async def oauth_refresh_tokens(
         grant_type="refresh_token",
     ).model_dump(exclude_none=True)
 
-    response = await Requests().post(
+    response = await Requests(raise_for_status=False).post(
         f"{WORDPRESS_BASE_URL}oauth2/token",
         headers=headers,
         data=data,
@@ -252,7 +252,7 @@ async def validate_token(
         "token": token,
     }
 
-    response = await Requests().get(
+    response = await Requests(raise_for_status=False).get(
         f"{WORDPRESS_BASE_URL}oauth2/token-info",
         params=params,
     )
@@ -296,7 +296,7 @@ async def make_api_request(
 
     url = f"{WORDPRESS_BASE_URL.rstrip('/')}{endpoint}"
 
-    request_method = getattr(Requests(), method.lower())
+    request_method = getattr(Requests(raise_for_status=False), method.lower())
     response = await request_method(
         url,
         headers=headers,
@@ -476,6 +476,7 @@ async def create_post(
         data["tags"] = ",".join(str(t) for t in data["tags"])
 
     # Make the API request
+    site = normalize_site(site)
     endpoint = f"/rest/v1.1/sites/{site}/posts/new"
 
     headers = {
@@ -483,7 +484,7 @@ async def create_post(
         "Content-Type": "application/x-www-form-urlencoded",
     }
 
-    response = await Requests().post(
+    response = await Requests(raise_for_status=False).post(
         f"{WORDPRESS_BASE_URL.rstrip('/')}{endpoint}",
         headers=headers,
         data=data,
@@ -499,3 +500,132 @@ async def create_post(
     )
     error_message = error_data.get("message", response.text)
     raise ValueError(f"Failed to create post: {response.status} - {error_message}")
+
+
+class Post(BaseModel):
+    """Response model for individual posts in a posts list response.
+
+    This is a simplified version compared to PostResponse, as the list endpoint
+    returns less detailed information than the create/get single post endpoints.
+    """
+
+    ID: int
+    site_ID: int
+    author: PostAuthor
+    date: datetime
+    modified: datetime
+    title: str
+    URL: str
+    short_URL: str
+    content: str | None = None
+    excerpt: str | None = None
+    slug: str
+    guid: str
+    status: str
+    sticky: bool
+    password: str | None = ""
+    parent: Union[Dict[str, Any], bool, None] = None
+    type: str
+    discussion: Dict[str, Union[str, bool, int]] | None = None
+    likes_enabled: bool | None = None
+    sharing_enabled: bool | None = None
+    like_count: int | None = None
+    i_like: bool | None = None
+    is_reblogged: bool | None = None
+    is_following: bool | None = None
+    global_ID: str | None = None
+    featured_image: str | None = None
+    post_thumbnail: Dict[str, Any] | None = None
+    format: str | None = None
+    geo: Union[Dict[str, Any], bool, None] = None
+    menu_order: int | None = None
+    page_template: str | None = None
+    publicize_URLs: List[str] | None = None
+    terms: Dict[str, Dict[str, Any]] | None = None
+    tags: Dict[str, Dict[str, Any]] | None = None
+    categories: Dict[str, Dict[str, Any]] | None = None
+    attachments: Dict[str, Dict[str, Any]] | None = None
+    attachment_count: int | None = None
+    metadata: List[Dict[str, Any]] | None = None
+    meta: Dict[str, Any] | None = None
+    capabilities: Dict[str, bool] | None = None
+    revisions: List[int] | None = None
+    other_URLs: Dict[str, Any] | None = None
+
+
+class PostsResponse(BaseModel):
+    """Response model for WordPress posts list."""
+
+    found: int
+    posts: List[Post]
+    meta: Dict[str, Any]
+
+
+def normalize_site(site: str) -> str:
+    """
+    Normalize a site identifier by stripping protocol and trailing slashes.
+
+    Args:
+        site: Site URL, domain, or ID (e.g., "https://myblog.wordpress.com/", "myblog.wordpress.com", "123456789")
+
+    Returns:
+        Normalized site identifier (domain or ID only)
+    """
+    site = site.strip()
+    if site.startswith("https://"):
+        site = site[8:]
+    elif site.startswith("http://"):
+        site = site[7:]
+    return site.rstrip("/")
+
+
+async def get_posts(
+    credentials: Credentials,
+    site: str,
+    status: PostStatus | None = None,
+    number: int = 100,
+    offset: int = 0,
+) -> PostsResponse:
+    """
+    Get posts from a WordPress site.
+
+    Args:
+        credentials: OAuth credentials
+        site: Site ID or domain (e.g., "myblog.wordpress.com" or "123456789")
+        status: Filter by post status using PostStatus enum, or None for all
+        number: Number of posts to retrieve (max 100)
+        offset: Number of posts to skip (for pagination)
+
+    Returns:
+        PostsResponse with the list of posts
+    """
+    site = normalize_site(site)
+    endpoint = f"/rest/v1.1/sites/{site}/posts"
+
+    headers = {
+        "Authorization": credentials.auth_header(),
+    }
+
+    params: Dict[str, Any] = {
+        "number": max(1, min(number, 100)),  # 1–100 posts per request
+        "offset": offset,
+    }
+
+    if status:
+        params["status"] = status.value
+    response = await Requests(raise_for_status=False).get(
+        f"{WORDPRESS_BASE_URL.rstrip('/')}{endpoint}",
+        headers=headers,
+        params=params,
+    )
+
+    if response.ok:
+        return PostsResponse.model_validate(response.json())
+
+    error_data = (
+        response.json()
+        if response.headers.get("content-type", "").startswith("application/json")
+        else {}
+    )
+    error_message = error_data.get("message", response.text)
+    raise ValueError(f"Failed to get posts: {response.status} - {error_message}")

autogpt_platform/backend/backend/blocks/wordpress/blog.py

@@ -9,7 +9,15 @@ from backend.sdk import (
     SchemaField,
 )
 
-from ._api import CreatePostRequest, PostResponse, PostStatus, create_post
+from ._api import (
+    CreatePostRequest,
+    Post,
+    PostResponse,
+    PostsResponse,
+    PostStatus,
+    create_post,
+    get_posts,
+)
 from ._config import wordpress
 
 
@@ -49,8 +57,15 @@ class WordPressCreatePostBlock(Block):
         media_urls: list[str] = SchemaField(
             description="URLs of images to sideload and attach to the post", default=[]
         )
+        publish_as_draft: bool = SchemaField(
+            description="If True, publishes the post as a draft. If False, publishes it publicly.",
+            default=False,
+        )
 
     class Output(BlockSchemaOutput):
+        site: str = SchemaField(
+            description="The site ID or domain (pass-through for chaining with other blocks)"
+        )
         post_id: int = SchemaField(description="The ID of the created post")
         post_url: str = SchemaField(description="The full URL of the created post")
         short_url: str = SchemaField(description="The shortened wp.me URL")
@@ -78,7 +93,9 @@ class WordPressCreatePostBlock(Block):
             tags=input_data.tags,
             featured_image=input_data.featured_image,
             media_urls=input_data.media_urls,
-            status=PostStatus.PUBLISH,
+            status=(
+                PostStatus.DRAFT if input_data.publish_as_draft else PostStatus.PUBLISH
+            ),
         )
 
         post_response: PostResponse = await create_post(
@@ -87,7 +104,69 @@ class WordPressCreatePostBlock(Block):
             post_data=post_request,
         )
 
+        yield "site", input_data.site
         yield "post_id", post_response.ID
         yield "post_url", post_response.URL
         yield "short_url", post_response.short_URL
         yield "post_data", post_response.model_dump()
+
+
+class WordPressGetAllPostsBlock(Block):
+    """
+    Fetches all posts from a WordPress.com site or Jetpack-enabled site.
+    Supports filtering by status and pagination.
+    """
+
+    class Input(BlockSchemaInput):
+        credentials: CredentialsMetaInput = wordpress.credentials_field()
+        site: str = SchemaField(
+            description="Site ID or domain (e.g., 'myblog.wordpress.com' or '123456789')"
+        )
+        status: PostStatus | None = SchemaField(
+            description="Filter by post status, or None for all",
+            default=None,
+        )
+        number: int = SchemaField(
+            description="Number of posts to retrieve (max 100 per request)", default=20
+        )
+        offset: int = SchemaField(
+            description="Number of posts to skip (for pagination)", default=0
+        )
+
+    class Output(BlockSchemaOutput):
+        site: str = SchemaField(
+            description="The site ID or domain (pass-through for chaining with other blocks)"
+        )
+        found: int = SchemaField(description="Total number of posts found")
+        posts: list[Post] = SchemaField(
+            description="List of post objects with their details"
+        )
+        post: Post = SchemaField(
+            description="Individual post object (yielded for each post)"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="97728fa7-7f6f-4789-ba0c-f2c114119536",
+            description="Fetch all posts from WordPress.com or Jetpack sites",
+            categories={BlockCategory.SOCIAL},
+            input_schema=self.Input,
+            output_schema=self.Output,
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: Credentials, **kwargs
+    ) -> BlockOutput:
+        posts_response: PostsResponse = await get_posts(
+            credentials=credentials,
+            site=input_data.site,
+            status=input_data.status,
+            number=input_data.number,
+            offset=input_data.offset,
+        )
+
+        yield "site", input_data.site
+        yield "found", posts_response.found
+        yield "posts", posts_response.posts
+        for post in posts_response.posts:
+            yield "post", post

autogpt_platform/backend/backend/blocks/xml_parser.py

@@ -1,5 +1,5 @@
 from gravitasml.parser import Parser
-from gravitasml.token import tokenize
+from gravitasml.token import Token, tokenize
 
 from backend.data.block import Block, BlockOutput, BlockSchemaInput, BlockSchemaOutput
 from backend.data.model import SchemaField
@@ -25,6 +25,38 @@ class XMLParserBlock(Block):
             ],
         )
 
+    @staticmethod
+    def _validate_tokens(tokens: list[Token]) -> None:
+        """Ensure the XML has a single root element and no stray text."""
+        if not tokens:
+            raise ValueError("XML input is empty.")
+
+        depth = 0
+        root_seen = False
+
+        for token in tokens:
+            if token.type == "TAG_OPEN":
+                if depth == 0 and root_seen:
+                    raise ValueError("XML must have a single root element.")
+                depth += 1
+                if depth == 1:
+                    root_seen = True
+            elif token.type == "TAG_CLOSE":
+                depth -= 1
+                if depth < 0:
+                    raise SyntaxError("Unexpected closing tag in XML input.")
+            elif token.type in {"TEXT", "ESCAPE"}:
+                if depth == 0 and token.value:
+                    raise ValueError(
+                        "XML contains text outside the root element; "
+                        "wrap content in a single root tag."
+                    )
+
+        if depth != 0:
+            raise SyntaxError("Unclosed tag detected in XML input.")
+        if not root_seen:
+            raise ValueError("XML must include a root element.")
+
     async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         # Security fix: Add size limits to prevent XML bomb attacks
         MAX_XML_SIZE = 10 * 1024 * 1024  # 10MB limit for XML input
@@ -35,7 +67,9 @@ class XMLParserBlock(Block):
             )
 
         try:
-            tokens = tokenize(input_data.input_xml)
+            tokens = list(tokenize(input_data.input_xml))
+            self._validate_tokens(tokens)
+
             parser = Parser(tokens)
             parsed_result = parser.parse()
             yield "parsed_xml", parsed_result

autogpt_platform/backend/backend/data/block.py

@@ -25,7 +25,6 @@ from prisma.models import AgentBlock
 from prisma.types import AgentBlockCreateInput
 from pydantic import BaseModel
 
-from backend.data.llm_registry import update_schema_with_llm_registry
 from backend.data.model import NodeExecutionStats
 from backend.integrations.providers import ProviderName
 from backend.util import json
@@ -51,6 +50,8 @@ from .model import (
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
+    from backend.data.execution import ExecutionContext
+
     from .graph import Link
 
 app_config = Config()
@@ -142,59 +143,35 @@ class BlockInfo(BaseModel):
 
 
 class BlockSchema(BaseModel):
-    cached_jsonschema: ClassVar[dict[str, Any] | None] = None
-
-    @classmethod
-    def clear_schema_cache(cls) -> None:
-        """Clear the cached JSON schema for this class."""
-        # Use None instead of {} because {} is truthy and would prevent regeneration
-        cls.cached_jsonschema = None  # type: ignore
-
-    @staticmethod
-    def clear_all_schema_caches() -> None:
-        """Clear cached JSON schemas for all BlockSchema subclasses."""
-
-        def clear_recursive(cls: type) -> None:
-            """Recursively clear cache for class and all subclasses."""
-            if hasattr(cls, "clear_schema_cache"):
-                cls.clear_schema_cache()
-            for subclass in cls.__subclasses__():
-                clear_recursive(subclass)
-
-        clear_recursive(BlockSchema)
+    cached_jsonschema: ClassVar[dict[str, Any]]
 
     @classmethod
     def jsonschema(cls) -> dict[str, Any]:
-        # Generate schema if not cached
-        if not cls.cached_jsonschema:
-            model = jsonref.replace_refs(cls.model_json_schema(), merge_props=True)
+        if cls.cached_jsonschema:
+            return cls.cached_jsonschema
 
-            def ref_to_dict(obj):
-                if isinstance(obj, dict):
-                    # OpenAPI <3.1 does not support sibling fields that has a $ref key
-                    # So sometimes, the schema has an "allOf"/"anyOf"/"oneOf" with 1 item.
-                    keys = {"allOf", "anyOf", "oneOf"}
-                    one_key = next(
-                        (k for k in keys if k in obj and len(obj[k]) == 1), None
-                    )
-                    if one_key:
-                        obj.update(obj[one_key][0])
+        model = jsonref.replace_refs(cls.model_json_schema(), merge_props=True)
 
-                    return {
-                        key: ref_to_dict(value)
-                        for key, value in obj.items()
-                        if not key.startswith("$") and key != one_key
-                    }
-                elif isinstance(obj, list):
-                    return [ref_to_dict(item) for item in obj]
+        def ref_to_dict(obj):
+            if isinstance(obj, dict):
+                # OpenAPI <3.1 does not support sibling fields that has a $ref key
+                # So sometimes, the schema has an "allOf"/"anyOf"/"oneOf" with 1 item.
+                keys = {"allOf", "anyOf", "oneOf"}
+                one_key = next((k for k in keys if k in obj and len(obj[k]) == 1), None)
+                if one_key:
+                    obj.update(obj[one_key][0])
 
-                return obj
+                return {
+                    key: ref_to_dict(value)
+                    for key, value in obj.items()
+                    if not key.startswith("$") and key != one_key
+                }
+            elif isinstance(obj, list):
+                return [ref_to_dict(item) for item in obj]
 
-            cls.cached_jsonschema = cast(dict[str, Any], ref_to_dict(model))
+            return obj
 
-        # Always post-process to ensure LLM registry data is up-to-date
-        # This refreshes model options and discriminator mappings even if schema was cached
-        update_schema_with_llm_registry(cls.cached_jsonschema, cls)
+        cls.cached_jsonschema = cast(dict[str, Any], ref_to_dict(model))
 
         return cls.cached_jsonschema
 
@@ -497,6 +474,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
         self.block_type = block_type
         self.webhook_config = webhook_config
         self.execution_stats: NodeExecutionStats = NodeExecutionStats()
+        self.requires_human_review: bool = False
 
         if self.webhook_config:
             if isinstance(self.webhook_config, BlockWebhookConfig):
@@ -639,7 +617,77 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
                     block_id=self.id,
                 ) from ex
 
+    async def is_block_exec_need_review(
+        self,
+        input_data: BlockInput,
+        *,
+        user_id: str,
+        node_exec_id: str,
+        graph_exec_id: str,
+        graph_id: str,
+        graph_version: int,
+        execution_context: "ExecutionContext",
+        **kwargs,
+    ) -> tuple[bool, BlockInput]:
+        """
+        Check if this block execution needs human review and handle the review process.
+
+        Returns:
+            Tuple of (should_pause, input_data_to_use)
+            - should_pause: True if execution should be paused for review
+            - input_data_to_use: The input data to use (may be modified by reviewer)
+        """
+        # Skip review if not required or safe mode is disabled
+        if not self.requires_human_review or not execution_context.safe_mode:
+            return False, input_data
+
+        from backend.blocks.helpers.review import HITLReviewHelper
+
+        # Handle the review request and get decision
+        decision = await HITLReviewHelper.handle_review_decision(
+            input_data=input_data,
+            user_id=user_id,
+            node_exec_id=node_exec_id,
+            graph_exec_id=graph_exec_id,
+            graph_id=graph_id,
+            graph_version=graph_version,
+            execution_context=execution_context,
+            block_name=self.name,
+            editable=True,
+        )
+
+        if decision is None:
+            # We're awaiting review - pause execution
+            return True, input_data
+
+        if not decision.should_proceed:
+            # Review was rejected, raise an error to stop execution
+            raise BlockExecutionError(
+                message=f"Block execution rejected by reviewer: {decision.message}",
+                block_name=self.name,
+                block_id=self.id,
+            )
+
+        # Review was approved - use the potentially modified data
+        # ReviewResult.data must be a dict for block inputs
+        reviewed_data = decision.review_result.data
+        if not isinstance(reviewed_data, dict):
+            raise BlockExecutionError(
+                message=f"Review data must be a dict for block input, got {type(reviewed_data).__name__}",
+                block_name=self.name,
+                block_id=self.id,
+            )
+        return False, reviewed_data
+
     async def _execute(self, input_data: BlockInput, **kwargs) -> BlockOutput:
+        # Check for review requirement and get potentially modified input data
+        should_pause, input_data = await self.is_block_exec_need_review(
+            input_data, **kwargs
+        )
+        if should_pause:
+            return
+
+        # Validate the input data (original or reviewer-modified) once
         if error := self.input_schema.validate_data(input_data):
             raise BlockInputError(
                 message=f"Unable to execute block with invalid input data: {error}",
@@ -647,6 +695,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
                 block_id=self.id,
             )
 
+        # Use the validated input data
         async for output_name, output_data in self.run(
             self.input_schema(**{k: v for k, v in input_data.items() if v is not None}),
             **kwargs,
@@ -810,28 +859,6 @@ def is_block_auth_configured(
 
 
 async def initialize_blocks() -> None:
-    # Refresh LLM registry before initializing blocks so blocks can use registry data
-    # This ensures the registry cache is populated even in executor context
-    try:
-        from backend.data import llm_registry
-        from backend.data.block_cost_config import refresh_llm_costs
-
-        # Only refresh if we have DB access (check if Prisma is connected)
-        from backend.data.db import is_connected
-
-        if is_connected():
-            await llm_registry.refresh_llm_registry()
-            refresh_llm_costs()
-            logger.info("LLM registry refreshed during block initialization")
-        else:
-            logger.warning(
-                "Prisma not connected, skipping LLM registry refresh during block initialization"
-            )
-    except Exception as exc:
-        logger.warning(
-            "Failed to refresh LLM registry during block initialization: %s", exc
-        )
-
     # First, sync all provider costs to blocks
     # Imported here to avoid circular import
     from backend.sdk.cost_integration import sync_all_provider_costs

autogpt_platform/backend/backend/data/block_cost_config.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Type
 
 from backend.blocks.ai_image_customizer import AIImageCustomizerBlock, GeminiImageModel
@@ -24,18 +23,19 @@ from backend.blocks.ideogram import IdeogramModelBlock
 from backend.blocks.jina.embeddings import JinaEmbeddingBlock
 from backend.blocks.jina.search import ExtractWebsiteContentBlock, SearchTheWebBlock
 from backend.blocks.llm import (
+    MODEL_METADATA,
     AIConversationBlock,
     AIListGeneratorBlock,
     AIStructuredResponseGeneratorBlock,
     AITextGeneratorBlock,
     AITextSummarizerBlock,
+    LlmModel,
 )
 from backend.blocks.replicate.flux_advanced import ReplicateFluxAdvancedModelBlock
 from backend.blocks.replicate.replicate_block import ReplicateModelBlock
 from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
 from backend.blocks.talking_head import CreateTalkingAvatarVideoBlock
 from backend.blocks.text_to_speech_block import UnrealTextToSpeechBlock
-from backend.data import llm_registry
 from backend.data.block import Block, BlockCost, BlockCostType
 from backend.integrations.credentials_store import (
     aiml_api_credentials,
@@ -55,63 +55,210 @@ from backend.integrations.credentials_store import (
     v0_credentials,
 )
 
-logger = logging.getLogger(__name__)
+# =============== Configure the cost for each LLM Model call =============== #
 
-PROVIDER_CREDENTIALS = {
-    "openai": openai_credentials,
-    "anthropic": anthropic_credentials,
-    "groq": groq_credentials,
-    "open_router": open_router_credentials,
-    "llama_api": llama_api_credentials,
-    "aiml_api": aiml_api_credentials,
-    "v0": v0_credentials,
+MODEL_COST: dict[LlmModel, int] = {
+    LlmModel.O3: 4,
+    LlmModel.O3_MINI: 2,
+    LlmModel.O1: 16,
+    LlmModel.O1_MINI: 4,
+    # GPT-5 models
+    LlmModel.GPT5_2: 6,
+    LlmModel.GPT5_1: 5,
+    LlmModel.GPT5: 2,
+    LlmModel.GPT5_MINI: 1,
+    LlmModel.GPT5_NANO: 1,
+    LlmModel.GPT5_CHAT: 5,
+    LlmModel.GPT41: 2,
+    LlmModel.GPT41_MINI: 1,
+    LlmModel.GPT4O_MINI: 1,
+    LlmModel.GPT4O: 3,
+    LlmModel.GPT4_TURBO: 10,
+    LlmModel.GPT3_5_TURBO: 1,
+    LlmModel.CLAUDE_4_1_OPUS: 21,
+    LlmModel.CLAUDE_4_OPUS: 21,
+    LlmModel.CLAUDE_4_SONNET: 5,
+    LlmModel.CLAUDE_4_5_HAIKU: 4,
+    LlmModel.CLAUDE_4_5_OPUS: 14,
+    LlmModel.CLAUDE_4_5_SONNET: 9,
+    LlmModel.CLAUDE_3_7_SONNET: 5,
+    LlmModel.CLAUDE_3_HAIKU: 1,
+    LlmModel.AIML_API_QWEN2_5_72B: 1,
+    LlmModel.AIML_API_LLAMA3_1_70B: 1,
+    LlmModel.AIML_API_LLAMA3_3_70B: 1,
+    LlmModel.AIML_API_META_LLAMA_3_1_70B: 1,
+    LlmModel.AIML_API_LLAMA_3_2_3B: 1,
+    LlmModel.LLAMA3_3_70B: 1,
+    LlmModel.LLAMA3_1_8B: 1,
+    LlmModel.OLLAMA_LLAMA3_3: 1,
+    LlmModel.OLLAMA_LLAMA3_2: 1,
+    LlmModel.OLLAMA_LLAMA3_8B: 1,
+    LlmModel.OLLAMA_LLAMA3_405B: 1,
+    LlmModel.OLLAMA_DOLPHIN: 1,
+    LlmModel.OPENAI_GPT_OSS_120B: 1,
+    LlmModel.OPENAI_GPT_OSS_20B: 1,
+    LlmModel.GEMINI_2_5_PRO: 4,
+    LlmModel.GEMINI_3_PRO_PREVIEW: 5,
+    LlmModel.MISTRAL_NEMO: 1,
+    LlmModel.COHERE_COMMAND_R_08_2024: 1,
+    LlmModel.COHERE_COMMAND_R_PLUS_08_2024: 3,
+    LlmModel.DEEPSEEK_CHAT: 2,
+    LlmModel.PERPLEXITY_SONAR: 1,
+    LlmModel.PERPLEXITY_SONAR_PRO: 5,
+    LlmModel.PERPLEXITY_SONAR_DEEP_RESEARCH: 10,
+    LlmModel.NOUSRESEARCH_HERMES_3_LLAMA_3_1_405B: 1,
+    LlmModel.NOUSRESEARCH_HERMES_3_LLAMA_3_1_70B: 1,
+    LlmModel.AMAZON_NOVA_LITE_V1: 1,
+    LlmModel.AMAZON_NOVA_MICRO_V1: 1,
+    LlmModel.AMAZON_NOVA_PRO_V1: 1,
+    LlmModel.MICROSOFT_WIZARDLM_2_8X22B: 1,
+    LlmModel.GRYPHE_MYTHOMAX_L2_13B: 1,
+    LlmModel.META_LLAMA_4_SCOUT: 1,
+    LlmModel.META_LLAMA_4_MAVERICK: 1,
+    LlmModel.LLAMA_API_LLAMA_4_SCOUT: 1,
+    LlmModel.LLAMA_API_LLAMA4_MAVERICK: 1,
+    LlmModel.LLAMA_API_LLAMA3_3_8B: 1,
+    LlmModel.LLAMA_API_LLAMA3_3_70B: 1,
+    LlmModel.GROK_4: 9,
+    LlmModel.GROK_4_FAST: 1,
+    LlmModel.GROK_4_1_FAST: 1,
+    LlmModel.GROK_CODE_FAST_1: 1,
+    LlmModel.KIMI_K2: 1,
+    LlmModel.QWEN3_235B_A22B_THINKING: 1,
+    LlmModel.QWEN3_CODER: 9,
+    LlmModel.GEMINI_2_5_FLASH: 1,
+    LlmModel.GEMINI_2_0_FLASH: 1,
+    LlmModel.GEMINI_2_5_FLASH_LITE_PREVIEW: 1,
+    LlmModel.GEMINI_2_0_FLASH_LITE: 1,
+    LlmModel.DEEPSEEK_R1_0528: 1,
+    # v0 by Vercel models
+    LlmModel.V0_1_5_MD: 1,
+    LlmModel.V0_1_5_LG: 2,
+    LlmModel.V0_1_0_MD: 1,
 }
 
-# =============== Configure the cost for each LLM Model call =============== #
-# All LLM costs now come from the database via llm_registry
-
-LLM_COST: list[BlockCost] = []
+for model in LlmModel:
+    if model not in MODEL_COST:
+        raise ValueError(f"Missing MODEL_COST for model: {model}")
 
 
-def _build_llm_costs_from_registry() -> list[BlockCost]:
-    """Build BlockCost list from all models in the LLM registry."""
-    costs: list[BlockCost] = []
-    for model in llm_registry.iter_dynamic_models():
-        for cost in model.costs:
-            credentials = PROVIDER_CREDENTIALS.get(cost.credential_provider)
-            if not credentials:
-                logger.warning(
-                    "Skipping cost entry for %s due to unknown credentials provider %s",
-                    model.slug,
-                    cost.credential_provider,
-                )
-                continue
-            cost_filter = {
-                "model": model.slug,
+LLM_COST = (
+    # Anthropic Models
+    [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
                 "credentials": {
-                    "id": credentials.id,
-                    "provider": credentials.provider,
-                    "type": credentials.type,
+                    "id": anthropic_credentials.id,
+                    "provider": anthropic_credentials.provider,
+                    "type": anthropic_credentials.type,
                 },
-            }
-            costs.append(
-                BlockCost(
-                    cost_type=BlockCostType.RUN,
-                    cost_filter=cost_filter,
-                    cost_amount=cost.credit_cost,
-                )
-            )
-    return costs
-
-
-def refresh_llm_costs() -> None:
-    """Refresh LLM costs from the registry. All costs now come from the database."""
-    LLM_COST.clear()
-    LLM_COST.extend(_build_llm_costs_from_registry())
-
-
-# Initial load will happen after registry is refreshed at startup
-# Don't call refresh_llm_costs() here - it will be called after registry refresh
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "anthropic"
+    ]
+    # OpenAI Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {
+                    "id": openai_credentials.id,
+                    "provider": openai_credentials.provider,
+                    "type": openai_credentials.type,
+                },
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "openai"
+    ]
+    # Groq Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {"id": groq_credentials.id},
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "groq"
+    ]
+    # Open Router Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {
+                    "id": open_router_credentials.id,
+                    "provider": open_router_credentials.provider,
+                    "type": open_router_credentials.type,
+                },
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "open_router"
+    ]
+    # Llama API Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {
+                    "id": llama_api_credentials.id,
+                    "provider": llama_api_credentials.provider,
+                    "type": llama_api_credentials.type,
+                },
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "llama_api"
+    ]
+    # v0 by Vercel Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {
+                    "id": v0_credentials.id,
+                    "provider": v0_credentials.provider,
+                    "type": v0_credentials.type,
+                },
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "v0"
+    ]
+    # AI/ML Api Models
+    + [
+        BlockCost(
+            cost_type=BlockCostType.RUN,
+            cost_filter={
+                "model": model,
+                "credentials": {
+                    "id": aiml_api_credentials.id,
+                    "provider": aiml_api_credentials.provider,
+                    "type": aiml_api_credentials.type,
+                },
+            },
+            cost_amount=cost,
+        )
+        for model, cost in MODEL_COST.items()
+        if MODEL_METADATA[model].provider == "aiml_api"
+    ]
+)
 
 # =============== This is the exhaustive list of cost for each Block =============== #
 

autogpt_platform/backend/backend/data/db.py

@@ -38,6 +38,20 @@ POOL_TIMEOUT = os.getenv("DB_POOL_TIMEOUT")
 if POOL_TIMEOUT:
     DATABASE_URL = add_param(DATABASE_URL, "pool_timeout", POOL_TIMEOUT)
 
+# Add public schema to search_path for pgvector type access
+# The vector extension is in public schema, but search_path is determined by schema parameter
+# Extract the schema from DATABASE_URL or default to 'platform'
+parsed_url = urlparse(DATABASE_URL)
+url_params = dict(parse_qsl(parsed_url.query))
+db_schema = url_params.get("schema", "platform")
+# Build search_path, avoiding duplicates if db_schema is already 'public'
+search_path_schemas = list(
+    dict.fromkeys([db_schema, "public"])
+)  # Preserves order, removes duplicates
+search_path = ",".join(search_path_schemas)
+# This allows using ::vector without schema qualification
+DATABASE_URL = add_param(DATABASE_URL, "options", f"-c search_path={search_path}")
+
 HTTP_TIMEOUT = int(POOL_TIMEOUT) if POOL_TIMEOUT else None
 
 prisma = Prisma(
@@ -108,21 +122,102 @@ def get_database_schema() -> str:
     return query_params.get("schema", "public")
 
 
-async def query_raw_with_schema(query_template: str, *args) -> list[dict]:
-    """Execute raw SQL query with proper schema handling."""
+async def _raw_with_schema(
+    query_template: str,
+    *args,
+    execute: bool = False,
+    client: Prisma | None = None,
+    set_public_search_path: bool = False,
+) -> list[dict] | int:
+    """Internal: Execute raw SQL with proper schema handling.
+
+    Use query_raw_with_schema() or execute_raw_with_schema() instead.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+        execute: If False, executes SELECT query. If True, executes INSERT/UPDATE/DELETE.
+        client: Optional Prisma client for transactions (only used when execute=True).
+        set_public_search_path: If True, sets search_path to include public schema.
+                                Needed for pgvector types and other public schema objects.
+
+    Returns:
+        - list[dict] if execute=False (query results)
+        - int if execute=True (number of affected rows)
+    """
     schema = get_database_schema()
     schema_prefix = f'"{schema}".' if schema != "public" else ""
     formatted_query = query_template.format(schema_prefix=schema_prefix)
 
     import prisma as prisma_module
 
-    result = await prisma_module.get_client().query_raw(
-        formatted_query, *args  # type: ignore
-    )
+    db_client = client if client else prisma_module.get_client()
+
+    # Set search_path to include public schema if requested
+    # Prisma doesn't support the 'options' connection parameter, so we set it per-session
+    # This is idempotent and safe to call multiple times
+    if set_public_search_path:
+        await db_client.execute_raw(f"SET search_path = {schema}, public")  # type: ignore
+
+    if execute:
+        result = await db_client.execute_raw(formatted_query, *args)  # type: ignore
+    else:
+        result = await db_client.query_raw(formatted_query, *args)  # type: ignore
 
     return result
 
 
+async def query_raw_with_schema(
+    query_template: str, *args, set_public_search_path: bool = False
+) -> list[dict]:
+    """Execute raw SQL SELECT query with proper schema handling.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+        set_public_search_path: If True, sets search_path to include public schema.
+                                Needed for pgvector types and other public schema objects.
+
+    Returns:
+        List of result rows as dictionaries
+
+    Example:
+        results = await query_raw_with_schema(
+            'SELECT * FROM {schema_prefix}"User" WHERE id = $1',
+            user_id
+        )
+    """
+    return await _raw_with_schema(query_template, *args, execute=False, set_public_search_path=set_public_search_path)  # type: ignore
+
+
+async def execute_raw_with_schema(
+    query_template: str,
+    *args,
+    client: Prisma | None = None,
+    set_public_search_path: bool = False,
+) -> int:
+    """Execute raw SQL command (INSERT/UPDATE/DELETE) with proper schema handling.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+        client: Optional Prisma client for transactions
+        set_public_search_path: If True, sets search_path to include public schema.
+                                Needed for pgvector types and other public schema objects.
+
+    Returns:
+        Number of affected rows
+
+    Example:
+        await execute_raw_with_schema(
+            'INSERT INTO {schema_prefix}"User" (id, name) VALUES ($1, $2)',
+            user_id, name,
+            client=tx  # Optional transaction client
+        )
+    """
+    return await _raw_with_schema(query_template, *args, execute=True, client=client, set_public_search_path=set_public_search_path)  # type: ignore
+
+
 class BaseDbModel(BaseModel):
     id: str = Field(default_factory=lambda: str(uuid4()))
 

autogpt_platform/backend/backend/data/execution.py

@@ -383,6 +383,7 @@ class GraphExecutionWithNodes(GraphExecution):
         self,
         execution_context: ExecutionContext,
         compiled_nodes_input_masks: Optional[NodesInputMasks] = None,
+        nodes_to_skip: Optional[set[str]] = None,
     ):
         return GraphExecutionEntry(
             user_id=self.user_id,
@@ -390,6 +391,7 @@ class GraphExecutionWithNodes(GraphExecution):
             graph_version=self.graph_version or 0,
             graph_exec_id=self.id,
             nodes_input_masks=compiled_nodes_input_masks,
+            nodes_to_skip=nodes_to_skip or set(),
             execution_context=execution_context,
         )
 
@@ -1145,6 +1147,8 @@ class GraphExecutionEntry(BaseModel):
     graph_id: str
     graph_version: int
     nodes_input_masks: Optional[NodesInputMasks] = None
+    nodes_to_skip: set[str] = Field(default_factory=set)
+    """Node IDs that should be skipped due to optional credentials not being configured."""
     execution_context: ExecutionContext = Field(default_factory=ExecutionContext)
 
 

autogpt_platform/backend/backend/data/graph.py

@@ -94,6 +94,15 @@ class Node(BaseDbModel):
     input_links: list[Link] = []
     output_links: list[Link] = []
 
+    @property
+    def credentials_optional(self) -> bool:
+        """
+        Whether credentials are optional for this node.
+        When True and credentials are not configured, the node will be skipped
+        during execution rather than causing a validation error.
+        """
+        return self.metadata.get("credentials_optional", False)
+
     @property
     def block(self) -> AnyBlockSchema | "_UnknownBlockBase":
         """Get the block for this node. Returns UnknownBlock if block is deleted/missing."""
@@ -235,7 +244,10 @@ class BaseGraph(BaseDbModel):
         return any(
             node.block_id
             for node in self.nodes
-            if node.block.block_type == BlockType.HUMAN_IN_THE_LOOP
+            if (
+                node.block.block_type == BlockType.HUMAN_IN_THE_LOOP
+                or node.block.requires_human_review
+            )
         )
 
     @property
@@ -326,7 +338,35 @@ class Graph(BaseGraph):
     @computed_field
     @property
     def credentials_input_schema(self) -> dict[str, Any]:
-        return self._credentials_input_schema.jsonschema()
+        schema = self._credentials_input_schema.jsonschema()
+
+        # Determine which credential fields are required based on credentials_optional metadata
+        graph_credentials_inputs = self.aggregate_credentials_inputs()
+        required_fields = []
+
+        # Build a map of node_id -> node for quick lookup
+        all_nodes = {node.id: node for node in self.nodes}
+        for sub_graph in self.sub_graphs:
+            for node in sub_graph.nodes:
+                all_nodes[node.id] = node
+
+        for field_key, (
+            _field_info,
+            node_field_pairs,
+        ) in graph_credentials_inputs.items():
+            # A field is required if ANY node using it has credentials_optional=False
+            is_required = False
+            for node_id, _field_name in node_field_pairs:
+                node = all_nodes.get(node_id)
+                if node and not node.credentials_optional:
+                    is_required = True
+                    break
+
+            if is_required:
+                required_fields.append(field_key)
+
+        schema["required"] = required_fields
+        return schema
 
     @property
     def _credentials_input_schema(self) -> type[BlockSchema]:
@@ -1443,10 +1483,8 @@ async def migrate_llm_models(migrate_to: LlmModel):
             if field.annotation == LlmModel:
                 llm_model_fields[block.id] = field_name
 
-    # Get all model slugs from the registry (dynamic, not hardcoded enum)
-    from backend.data import llm_registry
-
-    enum_values = list(llm_registry.get_all_model_slugs_for_validation())
+    # Convert enum values to a list of strings for the SQL query
+    enum_values = [v.value for v in LlmModel]
     escaped_enum_values = repr(tuple(enum_values))  # hack but works
 
     # Update each block

autogpt_platform/backend/backend/data/graph_test.py

@@ -1,5 +1,6 @@
 import json
 from typing import Any
+from unittest.mock import AsyncMock, patch
 from uuid import UUID
 
 import fastapi.exceptions
@@ -18,6 +19,17 @@ from backend.usecases.sample import create_test_user
 from backend.util.test import SpinTestServer
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 @pytest.mark.asyncio(loop_scope="session")
 async def test_graph_creation(server: SpinTestServer, snapshot: Snapshot):
     """
@@ -396,3 +408,58 @@ async def test_access_store_listing_graph(server: SpinTestServer):
         created_graph.id, created_graph.version, "3e53486c-cf57-477e-ba2a-cb02dc828e1b"
     )
     assert got_graph is not None
+
+
+# ============================================================================
+# Tests for Optional Credentials Feature
+# ============================================================================
+
+
+def test_node_credentials_optional_default():
+    """Test that credentials_optional defaults to False when not set in metadata."""
+    node = Node(
+        id="test_node",
+        block_id=StoreValueBlock().id,
+        input_default={},
+        metadata={},
+    )
+    assert node.credentials_optional is False
+
+
+def test_node_credentials_optional_true():
+    """Test that credentials_optional returns True when explicitly set."""
+    node = Node(
+        id="test_node",
+        block_id=StoreValueBlock().id,
+        input_default={},
+        metadata={"credentials_optional": True},
+    )
+    assert node.credentials_optional is True
+
+
+def test_node_credentials_optional_false():
+    """Test that credentials_optional returns False when explicitly set to False."""
+    node = Node(
+        id="test_node",
+        block_id=StoreValueBlock().id,
+        input_default={},
+        metadata={"credentials_optional": False},
+    )
+    assert node.credentials_optional is False
+
+
+def test_node_credentials_optional_with_other_metadata():
+    """Test that credentials_optional works correctly with other metadata present."""
+    node = Node(
+        id="test_node",
+        block_id=StoreValueBlock().id,
+        input_default={},
+        metadata={
+            "position": {"x": 100, "y": 200},
+            "customized_name": "My Custom Node",
+            "credentials_optional": True,
+        },
+    )
+    assert node.credentials_optional is True
+    assert node.metadata["position"] == {"x": 100, "y": 200}
+    assert node.metadata["customized_name"] == "My Custom Node"

autogpt_platform/backend/backend/data/llm_registry/__init__.py

@@ -1,72 +0,0 @@
-"""
-LLM Registry module for managing LLM models, providers, and costs dynamically.
-
-This module provides a database-driven registry system for LLM models,
-replacing hardcoded model configurations with a flexible admin-managed system.
-"""
-
-from backend.data.llm_registry.model_types import ModelMetadata
-
-# Re-export for backwards compatibility
-from backend.data.llm_registry.notifications import (
-    REGISTRY_REFRESH_CHANNEL,
-    publish_registry_refresh_notification,
-    subscribe_to_registry_refresh,
-)
-from backend.data.llm_registry.registry import (
-    RegistryModel,
-    RegistryModelCost,
-    RegistryModelCreator,
-    get_all_model_slugs_for_validation,
-    get_default_model_slug,
-    get_dynamic_model_slugs,
-    get_fallback_model_for_disabled,
-    get_llm_discriminator_mapping,
-    get_llm_model_cost,
-    get_llm_model_metadata,
-    get_llm_model_schema_options,
-    get_model_info,
-    is_model_enabled,
-    iter_dynamic_models,
-    refresh_llm_registry,
-    register_static_costs,
-    register_static_metadata,
-)
-from backend.data.llm_registry.schema_utils import (
-    is_llm_model_field,
-    refresh_llm_discriminator_mapping,
-    refresh_llm_model_options,
-    update_schema_with_llm_registry,
-)
-
-__all__ = [
-    # Types
-    "ModelMetadata",
-    "RegistryModel",
-    "RegistryModelCost",
-    "RegistryModelCreator",
-    # Registry functions
-    "get_all_model_slugs_for_validation",
-    "get_default_model_slug",
-    "get_dynamic_model_slugs",
-    "get_fallback_model_for_disabled",
-    "get_llm_discriminator_mapping",
-    "get_llm_model_cost",
-    "get_llm_model_metadata",
-    "get_llm_model_schema_options",
-    "get_model_info",
-    "is_model_enabled",
-    "iter_dynamic_models",
-    "refresh_llm_registry",
-    "register_static_costs",
-    "register_static_metadata",
-    # Notifications
-    "REGISTRY_REFRESH_CHANNEL",
-    "publish_registry_refresh_notification",
-    "subscribe_to_registry_refresh",
-    # Schema utilities
-    "is_llm_model_field",
-    "refresh_llm_discriminator_mapping",
-    "refresh_llm_model_options",
-    "update_schema_with_llm_registry",
-]

autogpt_platform/backend/backend/data/llm_registry/model_types.py

@@ -1,11 +0,0 @@
-"""Type definitions for LLM model metadata."""
-
-from typing import NamedTuple
-
-
-class ModelMetadata(NamedTuple):
-    """Metadata for an LLM model."""
-
-    provider: str
-    context_window: int
-    max_output_tokens: int | None

autogpt_platform/backend/backend/data/llm_registry/notifications.py

@@ -1,89 +0,0 @@
-"""
-Redis pub/sub notifications for LLM registry updates.
-
-When models are added/updated/removed via the admin UI, this module
-publishes notifications to Redis that all executor services subscribe to,
-ensuring they refresh their registry cache in real-time.
-"""
-
-import asyncio
-import logging
-from typing import Any
-
-from backend.data.redis_client import connect_async
-
-logger = logging.getLogger(__name__)
-
-# Redis channel name for LLM registry refresh notifications
-REGISTRY_REFRESH_CHANNEL = "llm_registry:refresh"
-
-
-async def publish_registry_refresh_notification() -> None:
-    """
-    Publish a notification to Redis that the LLM registry has been updated.
-    All executor services subscribed to this channel will refresh their registry.
-    """
-    try:
-        redis = await connect_async()
-        await redis.publish(REGISTRY_REFRESH_CHANNEL, "refresh")
-        logger.info("Published LLM registry refresh notification to Redis")
-    except Exception as exc:
-        logger.warning(
-            "Failed to publish LLM registry refresh notification: %s",
-            exc,
-            exc_info=True,
-        )
-
-
-async def subscribe_to_registry_refresh(
-    on_refresh: Any,  # Async callable that takes no args
-) -> None:
-    """
-    Subscribe to Redis notifications for LLM registry updates.
-    This runs in a loop and processes messages as they arrive.
-
-    Args:
-        on_refresh: Async callable to execute when a refresh notification is received
-    """
-    try:
-        redis = await connect_async()
-        pubsub = redis.pubsub()
-        await pubsub.subscribe(REGISTRY_REFRESH_CHANNEL)
-        logger.info(
-            "Subscribed to LLM registry refresh notifications on channel: %s",
-            REGISTRY_REFRESH_CHANNEL,
-        )
-
-        # Process messages in a loop
-        while True:
-            try:
-                message = await pubsub.get_message(
-                    ignore_subscribe_messages=True, timeout=1.0
-                )
-                if (
-                    message
-                    and message["type"] == "message"
-                    and message["channel"] == REGISTRY_REFRESH_CHANNEL
-                ):
-                    logger.info("Received LLM registry refresh notification")
-                    try:
-                        await on_refresh()
-                    except Exception as exc:
-                        logger.error(
-                            "Error refreshing LLM registry from notification: %s",
-                            exc,
-                            exc_info=True,
-                        )
-            except Exception as exc:
-                logger.warning(
-                    "Error processing registry refresh message: %s", exc, exc_info=True
-                )
-                # Continue listening even if one message fails
-                await asyncio.sleep(1)
-    except Exception as exc:
-        logger.error(
-            "Failed to subscribe to LLM registry refresh notifications: %s",
-            exc,
-            exc_info=True,
-        )
-        raise

autogpt_platform/backend/backend/data/llm_registry/registry.py

@@ -1,370 +0,0 @@
-"""Core LLM registry implementation for managing models dynamically."""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from dataclasses import dataclass, field
-from typing import Any, Iterable
-
-import prisma.models
-
-from backend.data.llm_registry.model_types import ModelMetadata
-
-logger = logging.getLogger(__name__)
-
-
-def _json_to_dict(value: Any) -> dict[str, Any]:
-    """Convert Prisma Json type to dict, with fallback to empty dict."""
-    if value is None:
-        return {}
-    if isinstance(value, dict):
-        return value
-    # Prisma Json type should always be a dict at runtime
-    return dict(value) if value else {}
-
-
-@dataclass(frozen=True)
-class RegistryModelCost:
-    """Cost configuration for an LLM model."""
-
-    credit_cost: int
-    credential_provider: str
-    credential_id: str | None
-    credential_type: str | None
-    currency: str | None
-    metadata: dict[str, Any]
-
-
-@dataclass(frozen=True)
-class RegistryModelCreator:
-    """Creator information for an LLM model."""
-
-    id: str
-    name: str
-    display_name: str
-    description: str | None
-    website_url: str | None
-    logo_url: str | None
-
-
-@dataclass(frozen=True)
-class RegistryModel:
-    """Represents a model in the LLM registry."""
-
-    slug: str
-    display_name: str
-    description: str | None
-    metadata: ModelMetadata
-    capabilities: dict[str, Any]
-    extra_metadata: dict[str, Any]
-    provider_display_name: str
-    is_enabled: bool
-    is_recommended: bool = False
-    costs: tuple[RegistryModelCost, ...] = field(default_factory=tuple)
-    creator: RegistryModelCreator | None = None
-
-
-_static_metadata: dict[str, ModelMetadata] = {}
-_static_costs: dict[str, int] = {}
-_dynamic_models: dict[str, RegistryModel] = {}
-_schema_options: list[dict[str, str]] = []
-_discriminator_mapping: dict[str, str] = {}
-_lock = asyncio.Lock()
-
-
-def register_static_metadata(metadata: dict[Any, ModelMetadata]) -> None:
-    """Register static metadata for legacy models (deprecated)."""
-    _static_metadata.update({str(key): value for key, value in metadata.items()})
-    _refresh_cached_schema()
-
-
-def register_static_costs(costs: dict[Any, int]) -> None:
-    """Register static costs for legacy models (deprecated)."""
-    _static_costs.update({str(key): value for key, value in costs.items()})
-
-
-def _build_schema_options() -> list[dict[str, str]]:
-    """Build schema options for model selection dropdown. Only includes enabled models."""
-    options: list[dict[str, str]] = []
-    # Only include enabled models in the dropdown options
-    for model in sorted(_dynamic_models.values(), key=lambda m: m.display_name.lower()):
-        if model.is_enabled:
-            options.append(
-                {
-                    "label": model.display_name,
-                    "value": model.slug,
-                    "group": model.metadata.provider,
-                    "description": model.description or "",
-                }
-            )
-
-    for slug, metadata in _static_metadata.items():
-        if slug in _dynamic_models:
-            continue
-        options.append(
-            {
-                "label": slug,
-                "value": slug,
-                "group": metadata.provider,
-                "description": "",
-            }
-        )
-    return options
-
-
-async def refresh_llm_registry() -> None:
-    """Refresh the LLM registry from the database. Loads all models (enabled and disabled)."""
-    async with _lock:
-        try:
-            records = await prisma.models.LlmModel.prisma().find_many(
-                include={
-                    "Provider": True,
-                    "Costs": True,
-                    "Creator": True,
-                }
-            )
-            logger.debug("Found %d LLM model records in database", len(records))
-        except Exception as exc:
-            logger.error(
-                "Failed to refresh LLM registry from DB: %s", exc, exc_info=True
-            )
-            return
-
-        dynamic: dict[str, RegistryModel] = {}
-        for record in records:
-            provider_name = (
-                record.Provider.name if record.Provider else record.providerId
-            )
-            metadata = ModelMetadata(
-                provider=provider_name,
-                context_window=record.contextWindow,
-                max_output_tokens=record.maxOutputTokens,
-            )
-            costs = tuple(
-                RegistryModelCost(
-                    credit_cost=cost.creditCost,
-                    credential_provider=cost.credentialProvider,
-                    credential_id=cost.credentialId,
-                    credential_type=cost.credentialType,
-                    currency=cost.currency,
-                    metadata=_json_to_dict(cost.metadata),
-                )
-                for cost in (record.Costs or [])
-            )
-
-            # Map creator if present
-            creator = None
-            if record.Creator:
-                creator = RegistryModelCreator(
-                    id=record.Creator.id,
-                    name=record.Creator.name,
-                    display_name=record.Creator.displayName,
-                    description=record.Creator.description,
-                    website_url=record.Creator.websiteUrl,
-                    logo_url=record.Creator.logoUrl,
-                )
-
-            dynamic[record.slug] = RegistryModel(
-                slug=record.slug,
-                display_name=record.displayName,
-                description=record.description,
-                metadata=metadata,
-                capabilities=_json_to_dict(record.capabilities),
-                extra_metadata=_json_to_dict(record.metadata),
-                provider_display_name=(
-                    record.Provider.displayName
-                    if record.Provider
-                    else record.providerId
-                ),
-                is_enabled=record.isEnabled,
-                is_recommended=record.isRecommended,
-                costs=costs,
-                creator=creator,
-            )
-
-        # Atomic swap - build new structures then replace references
-        # This ensures readers never see partially updated state
-        global _dynamic_models
-        _dynamic_models = dynamic
-        _refresh_cached_schema()
-        logger.info(
-            "LLM registry refreshed with %s dynamic models (enabled: %s, disabled: %s)",
-            len(dynamic),
-            sum(1 for m in dynamic.values() if m.is_enabled),
-            sum(1 for m in dynamic.values() if not m.is_enabled),
-        )
-
-
-def _refresh_cached_schema() -> None:
-    """Refresh cached schema options and discriminator mapping."""
-    global _schema_options, _discriminator_mapping
-
-    # Build new structures
-    new_options = _build_schema_options()
-    new_mapping = {slug: entry.metadata.provider for slug, entry in _dynamic_models.items()}
-    for slug, metadata in _static_metadata.items():
-        new_mapping.setdefault(slug, metadata.provider)
-
-    # Atomic swap - replace references to ensure readers see consistent state
-    _schema_options = new_options
-    _discriminator_mapping = new_mapping
-
-
-def get_llm_model_metadata(slug: str) -> ModelMetadata | None:
-    """Get model metadata by slug. Checks dynamic models first, then static metadata."""
-    if slug in _dynamic_models:
-        return _dynamic_models[slug].metadata
-    return _static_metadata.get(slug)
-
-
-def get_llm_model_cost(slug: str) -> tuple[RegistryModelCost, ...]:
-    """Get model cost configuration by slug."""
-    if slug in _dynamic_models:
-        return _dynamic_models[slug].costs
-    cost_value = _static_costs.get(slug)
-    if cost_value is None:
-        return tuple()
-    return (
-        RegistryModelCost(
-            credit_cost=cost_value,
-            credential_provider="static",
-            credential_id=None,
-            credential_type=None,
-            currency=None,
-            metadata={},
-        ),
-    )
-
-
-def get_llm_model_schema_options() -> list[dict[str, str]]:
-    """
-    Get schema options for LLM model selection dropdown.
-
-    Returns a copy of cached schema options that are refreshed when the registry is
-    updated via refresh_llm_registry() (called on startup and via Redis pub/sub).
-    """
-    # Return a copy to prevent external mutation
-    return list(_schema_options)
-
-
-def get_llm_discriminator_mapping() -> dict[str, str]:
-    """
-    Get discriminator mapping for LLM models.
-
-    Returns a copy of cached discriminator mapping that is refreshed when the registry
-    is updated via refresh_llm_registry() (called on startup and via Redis pub/sub).
-    """
-    # Return a copy to prevent external mutation
-    return dict(_discriminator_mapping)
-
-
-def get_dynamic_model_slugs() -> set[str]:
-    """Get all dynamic model slugs from the registry."""
-    return set(_dynamic_models.keys())
-
-
-def get_all_model_slugs_for_validation() -> set[str]:
-    """
-    Get ALL model slugs (both enabled and disabled) for validation purposes.
-
-    This is used for JSON schema enum validation - we need to accept any known
-    model value (even disabled ones) so that existing graphs don't fail validation.
-    The actual fallback/enforcement happens at runtime in llm_call().
-    """
-    all_slugs = set(_dynamic_models.keys())
-    all_slugs.update(_static_metadata.keys())
-    return all_slugs
-
-
-def iter_dynamic_models() -> Iterable[RegistryModel]:
-    """Iterate over all dynamic models in the registry."""
-    return tuple(_dynamic_models.values())
-
-
-def get_fallback_model_for_disabled(disabled_model_slug: str) -> RegistryModel | None:
-    """
-    Find a fallback model when the requested model is disabled.
-
-    Looks for an enabled model from the same provider. Prefers models with
-    similar names or capabilities if possible.
-
-    Args:
-        disabled_model_slug: The slug of the disabled model
-
-    Returns:
-        An enabled RegistryModel from the same provider, or None if no fallback found
-    """
-    disabled_model = _dynamic_models.get(disabled_model_slug)
-    if not disabled_model:
-        return None
-
-    provider = disabled_model.metadata.provider
-
-    # Find all enabled models from the same provider
-    candidates = [
-        model
-        for model in _dynamic_models.values()
-        if model.is_enabled and model.metadata.provider == provider
-    ]
-
-    if not candidates:
-        return None
-
-    # Sort by: prefer models with similar context window, then by name
-    candidates.sort(
-        key=lambda m: (
-            abs(m.metadata.context_window - disabled_model.metadata.context_window),
-            m.display_name.lower(),
-        )
-    )
-
-    return candidates[0]
-
-
-def is_model_enabled(model_slug: str) -> bool:
-    """Check if a model is enabled in the registry."""
-    model = _dynamic_models.get(model_slug)
-    if not model:
-        # Model not in registry - assume it's a static/legacy model and allow it
-        return True
-    return model.is_enabled
-
-
-def get_model_info(model_slug: str) -> RegistryModel | None:
-    """Get model info from the registry."""
-    return _dynamic_models.get(model_slug)
-
-
-def get_default_model_slug() -> str | None:
-    """
-    Get the default model slug to use for block defaults.
-
-    Returns the recommended model if set (configured via admin UI),
-    otherwise returns the first enabled model alphabetically.
-    Returns None if no models are available or enabled.
-    """
-    # Return the recommended model if one is set and enabled
-    for model in _dynamic_models.values():
-        if model.is_recommended and model.is_enabled:
-            return model.slug
-
-    # No recommended model set - find first enabled model alphabetically
-    for model in sorted(_dynamic_models.values(), key=lambda m: m.display_name.lower()):
-        if model.is_enabled:
-            logger.warning(
-                "No recommended model set, using '%s' as default",
-                model.slug,
-            )
-            return model.slug
-
-    # No enabled models available
-    if _dynamic_models:
-        logger.error(
-            "No enabled models found in registry (%d models registered but all disabled)",
-            len(_dynamic_models),
-        )
-    else:
-        logger.error("No models registered in LLM registry")
-
-    return None

autogpt_platform/backend/backend/data/llm_registry/schema_utils.py

@@ -1,130 +0,0 @@
-"""
-Helper utilities for LLM registry integration with block schemas.
-
-This module handles the dynamic injection of discriminator mappings
-and model options from the LLM registry into block schemas.
-"""
-
-import logging
-from typing import Any
-
-from backend.data.llm_registry.registry import (
-    get_all_model_slugs_for_validation,
-    get_default_model_slug,
-    get_llm_discriminator_mapping,
-    get_llm_model_schema_options,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def is_llm_model_field(field_name: str, field_info: Any) -> bool:
-    """
-    Check if a field is an LLM model selection field.
-
-    Returns True if the field has 'options' in json_schema_extra
-    (set by llm_model_schema_extra() in blocks/llm.py).
-    """
-    if not hasattr(field_info, "json_schema_extra"):
-        return False
-
-    extra = field_info.json_schema_extra
-    if isinstance(extra, dict):
-        return "options" in extra
-
-    return False
-
-
-def refresh_llm_model_options(field_schema: dict[str, Any]) -> None:
-    """
-    Refresh LLM model options from the registry.
-
-    Updates 'options' (for frontend dropdown) to show only enabled models,
-    but keeps the 'enum' (for validation) inclusive of ALL known models.
-
-    This is important because:
-    - Options: What users see in the dropdown (enabled models only)
-    - Enum: What values pass validation (all known models, including disabled)
-
-    Existing graphs may have disabled models selected - they should pass validation
-    and the fallback logic in llm_call() will handle using an alternative model.
-    """
-    fresh_options = get_llm_model_schema_options()
-    if not fresh_options:
-        return
-
-    # Update options array (UI dropdown) - only enabled models
-    if "options" in field_schema:
-        field_schema["options"] = fresh_options
-
-    all_known_slugs = get_all_model_slugs_for_validation()
-    if all_known_slugs and "enum" in field_schema:
-        existing_enum = set(field_schema.get("enum", []))
-        combined_enum = existing_enum | all_known_slugs
-        field_schema["enum"] = sorted(combined_enum)
-
-    # Set the default value from the registry (gpt-4o if available, else first enabled)
-    # This ensures new blocks have a sensible default pre-selected
-    default_slug = get_default_model_slug()
-    if default_slug:
-        field_schema["default"] = default_slug
-
-
-def refresh_llm_discriminator_mapping(field_schema: dict[str, Any]) -> None:
-    """
-    Refresh discriminator_mapping for fields that use model-based discrimination.
-
-    The discriminator is already set when AICredentialsField() creates the field.
-    We only need to refresh the mapping when models are added/removed.
-    """
-    if field_schema.get("discriminator") != "model":
-        return
-
-    # Always refresh the mapping to get latest models
-    fresh_mapping = get_llm_discriminator_mapping()
-    if fresh_mapping:
-        field_schema["discriminator_mapping"] = fresh_mapping
-
-
-def update_schema_with_llm_registry(
-    schema: dict[str, Any], model_class: type | None = None
-) -> None:
-    """
-    Update a JSON schema with current LLM registry data.
-
-    Refreshes:
-    1. Model options for LLM model selection fields (dropdown choices)
-    2. Discriminator mappings for credentials fields (model → provider)
-
-    Args:
-        schema: The JSON schema to update (mutated in-place)
-        model_class: The Pydantic model class (optional, for field introspection)
-    """
-    properties = schema.get("properties", {})
-
-    for field_name, field_schema in properties.items():
-        if not isinstance(field_schema, dict):
-            continue
-
-        # Refresh model options for LLM model fields
-        if model_class and hasattr(model_class, "model_fields"):
-            field_info = model_class.model_fields.get(field_name)
-            if field_info and is_llm_model_field(field_name, field_info):
-                try:
-                    refresh_llm_model_options(field_schema)
-                except Exception as exc:
-                    logger.warning(
-                        "Failed to refresh LLM options for field %s: %s",
-                        field_name,
-                        exc,
-                    )
-
-        # Refresh discriminator mapping for fields that use model discrimination
-        try:
-            refresh_llm_discriminator_mapping(field_schema)
-        except Exception as exc:
-            logger.warning(
-                "Failed to refresh discriminator mapping for field %s: %s",
-                field_name,
-                exc,
-            )

autogpt_platform/backend/backend/data/model.py

@@ -40,7 +40,6 @@ from pydantic_core import (
 )
 from typing_extensions import TypedDict
 
-from backend.data.llm_registry import update_schema_with_llm_registry
 from backend.integrations.providers import ProviderName
 from backend.util.json import loads as json_loads
 from backend.util.settings import Secrets
@@ -545,9 +544,7 @@ class CredentialsMetaInput(BaseModel, Generic[CP, CT]):
             else:
                 schema["credentials_provider"] = allowed_providers
             schema["credentials_types"] = model_class.allowed_cred_types()
-
-        # Ensure LLM discriminators are populated (delegates to shared helper)
-        update_schema_with_llm_registry(schema, model_class)
+        # Do not return anything, just mutate schema in place
 
     model_config = ConfigDict(
         json_schema_extra=_add_json_schema_extra,  # type: ignore
@@ -696,20 +693,16 @@ def CredentialsField(
     This is enforced by the `BlockSchema` base class.
     """
 
-    # Build field_schema_extra - always include discriminator and mapping if discriminator is set
-    field_schema_extra: dict[str, Any] = {}
-
-    # Always include discriminator if provided
-    if discriminator is not None:
-        field_schema_extra["discriminator"] = discriminator
-        # Always include discriminator_mapping when discriminator is set (even if empty initially)
-        field_schema_extra["discriminator_mapping"] = discriminator_mapping or {}
-
-    # Include other optional fields (only if not None)
-    if required_scopes:
-        field_schema_extra["credentials_scopes"] = list(required_scopes)
-    if discriminator_values:
-        field_schema_extra["discriminator_values"] = discriminator_values
+    field_schema_extra = {
+        k: v
+        for k, v in {
+            "credentials_scopes": list(required_scopes) or None,
+            "discriminator": discriminator,
+            "discriminator_mapping": discriminator_mapping,
+            "discriminator_values": discriminator_values,
+        }.items()
+        if v is not None
+    }
 
     # Merge any json_schema_extra passed in kwargs
     if "json_schema_extra" in kwargs:

autogpt_platform/backend/backend/executor/database.py

@@ -7,6 +7,10 @@ from backend.api.features.library.db import (
     list_library_agents,
 )
 from backend.api.features.store.db import get_store_agent_details, get_store_agents
+from backend.api.features.store.embeddings import (
+    backfill_missing_embeddings,
+    get_embedding_stats,
+)
 from backend.data import db
 from backend.data.analytics import (
     get_accuracy_trends_and_alerts,
@@ -208,6 +212,10 @@ class DatabaseManager(AppService):
     get_store_agents = _(get_store_agents)
     get_store_agent_details = _(get_store_agent_details)
 
+    # Store Embeddings
+    get_embedding_stats = _(get_embedding_stats)
+    backfill_missing_embeddings = _(backfill_missing_embeddings)
+
     # Summary data - async
     get_user_execution_summary_data = _(get_user_execution_summary_data)
 
@@ -259,6 +267,10 @@ class DatabaseManagerClient(AppServiceClient):
     get_store_agents = _(d.get_store_agents)
     get_store_agent_details = _(d.get_store_agent_details)
 
+    # Store Embeddings
+    get_embedding_stats = _(d.get_embedding_stats)
+    backfill_missing_embeddings = _(d.backfill_missing_embeddings)
+
 
 class DatabaseManagerAsyncClient(AppServiceClient):
     d = DatabaseManager

autogpt_platform/backend/backend/executor/llm_registry_init.py

@@ -1,66 +0,0 @@
-"""
-Helper functions for LLM registry initialization in executor context.
-
-These functions handle refreshing the LLM registry when the executor starts
-and subscribing to real-time updates via Redis pub/sub.
-"""
-
-import logging
-
-from backend.data import db, llm_registry
-from backend.data.block import BlockSchema, initialize_blocks
-from backend.data.block_cost_config import refresh_llm_costs
-from backend.data.llm_registry import subscribe_to_registry_refresh
-
-logger = logging.getLogger(__name__)
-
-
-async def initialize_registry_for_executor() -> None:
-    """
-    Initialize blocks and refresh LLM registry in the executor context.
-
-    This must run in the executor's event loop to have access to the database.
-    """
-    try:
-        # Connect to database if not already connected
-        if not db.is_connected():
-            await db.connect()
-            logger.info("[GraphExecutor] Connected to database for registry refresh")
-
-        # Initialize blocks (internally refreshes LLM registry and costs)
-        await initialize_blocks()
-        logger.info("[GraphExecutor] Blocks initialized")
-    except Exception as exc:
-        logger.warning(
-            "[GraphExecutor] Failed to refresh LLM registry on startup: %s",
-            exc,
-            exc_info=True,
-        )
-
-
-async def refresh_registry_on_notification() -> None:
-    """Refresh LLM registry when notified via Redis pub/sub."""
-    try:
-        # Ensure DB is connected
-        if not db.is_connected():
-            await db.connect()
-
-        # Refresh registry and costs
-        await llm_registry.refresh_llm_registry()
-        refresh_llm_costs()
-
-        # Clear block schema caches so they regenerate with new model options
-        BlockSchema.clear_all_schema_caches()
-
-        logger.info("[GraphExecutor] LLM registry refreshed from notification")
-    except Exception as exc:
-        logger.error(
-            "[GraphExecutor] Failed to refresh LLM registry from notification: %s",
-            exc,
-            exc_info=True,
-        )
-
-
-async def subscribe_to_registry_updates() -> None:
-    """Subscribe to Redis pub/sub for LLM registry refresh notifications."""
-    await subscribe_to_registry_refresh(refresh_registry_on_notification)

autogpt_platform/backend/backend/executor/manager.py

@@ -178,6 +178,7 @@ async def execute_node(
     execution_processor: "ExecutionProcessor",
     execution_stats: NodeExecutionStats | None = None,
     nodes_input_masks: Optional[NodesInputMasks] = None,
+    nodes_to_skip: Optional[set[str]] = None,
 ) -> BlockOutput:
     """
     Execute a node in the graph. This will trigger a block execution on a node,
@@ -245,6 +246,7 @@ async def execute_node(
         "user_id": user_id,
         "execution_context": execution_context,
         "execution_processor": execution_processor,
+        "nodes_to_skip": nodes_to_skip or set(),
     }
 
     # Last-minute fetch credentials + acquire a system-wide read-write lock to prevent
@@ -542,6 +544,7 @@ class ExecutionProcessor:
         node_exec_progress: NodeExecutionProgress,
         nodes_input_masks: Optional[NodesInputMasks],
         graph_stats_pair: tuple[GraphExecutionStats, threading.Lock],
+        nodes_to_skip: Optional[set[str]] = None,
     ) -> NodeExecutionStats:
         log_metadata = LogMetadata(
             logger=_logger,
@@ -564,6 +567,7 @@ class ExecutionProcessor:
             db_client=db_client,
             log_metadata=log_metadata,
             nodes_input_masks=nodes_input_masks,
+            nodes_to_skip=nodes_to_skip,
         )
         if isinstance(status, BaseException):
             raise status
@@ -609,6 +613,7 @@ class ExecutionProcessor:
         db_client: "DatabaseManagerAsyncClient",
         log_metadata: LogMetadata,
         nodes_input_masks: Optional[NodesInputMasks] = None,
+        nodes_to_skip: Optional[set[str]] = None,
     ) -> ExecutionStatus:
         status = ExecutionStatus.RUNNING
 
@@ -645,6 +650,7 @@ class ExecutionProcessor:
                 execution_processor=self,
                 execution_stats=stats,
                 nodes_input_masks=nodes_input_masks,
+                nodes_to_skip=nodes_to_skip,
             ):
                 await persist_output(output_name, output_data)
 
@@ -696,20 +702,6 @@ class ExecutionProcessor:
         )
         self.node_execution_thread.start()
         self.node_evaluation_thread.start()
-
-        # Initialize LLM registry and subscribe to updates
-        from backend.executor.llm_registry_init import (
-            initialize_registry_for_executor,
-            subscribe_to_registry_updates,
-        )
-
-        asyncio.run_coroutine_threadsafe(
-            initialize_registry_for_executor(), self.node_execution_loop
-        )
-        asyncio.run_coroutine_threadsafe(
-            subscribe_to_registry_updates(), self.node_execution_loop
-        )
-
         logger.info(f"[GraphExecutor] {self.tid} started")
 
     @error_logged(swallow=False)
@@ -970,6 +962,21 @@ class ExecutionProcessor:
 
                 queued_node_exec = execution_queue.get()
 
+                # Check if this node should be skipped due to optional credentials
+                if queued_node_exec.node_id in graph_exec.nodes_to_skip:
+                    log_metadata.info(
+                        f"Skipping node execution {queued_node_exec.node_exec_id} "
+                        f"for node {queued_node_exec.node_id} - optional credentials not configured"
+                    )
+                    # Mark the node as completed without executing
+                    # No outputs will be produced, so downstream nodes won't trigger
+                    update_node_execution_status(
+                        db_client=db_client,
+                        exec_id=queued_node_exec.node_exec_id,
+                        status=ExecutionStatus.COMPLETED,
+                    )
+                    continue
+
                 log_metadata.debug(
                     f"Dispatching node execution {queued_node_exec.node_exec_id} "
                     f"for node {queued_node_exec.node_id}",
@@ -1030,6 +1037,7 @@ class ExecutionProcessor:
                             execution_stats,
                             execution_stats_lock,
                         ),
+                        nodes_to_skip=graph_exec.nodes_to_skip,
                     ),
                     self.node_execution_loop,
                 )

autogpt_platform/backend/backend/executor/manager_test.py

@@ -1,4 +1,5 @@
 import logging
+from unittest.mock import AsyncMock, patch
 
 import fastapi.responses
 import pytest
@@ -19,6 +20,17 @@ from backend.util.test import SpinTestServer, wait_execution
 logger = logging.getLogger(__name__)
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 async def create_graph(s: SpinTestServer, g: graph.Graph, u: User) -> graph.Graph:
     logger.info(f"Creating graph for user {u.id}")
     return await s.agent_server.test_create_graph(CreateGraph(graph=g), u.id)

autogpt_platform/backend/backend/executor/scheduler.py

@@ -2,6 +2,7 @@ import asyncio
 import logging
 import os
 import threading
+import time
 import uuid
 from enum import Enum
 from typing import Optional
@@ -37,7 +38,7 @@ from backend.monitoring import (
     report_execution_accuracy_alerts,
     report_late_executions,
 )
-from backend.util.clients import get_scheduler_client
+from backend.util.clients import get_database_manager_client, get_scheduler_client
 from backend.util.cloud_storage import cleanup_expired_files_async
 from backend.util.exceptions import (
     GraphNotFoundError,
@@ -254,6 +255,88 @@ def execution_accuracy_alerts():
     return report_execution_accuracy_alerts()
 
 
+def ensure_embeddings_coverage():
+    """
+    Ensure all content types (store agents, blocks, docs) have embeddings for search.
+
+    Processes ALL missing embeddings in batches of 10 per content type until 100% coverage.
+    Missing embeddings = content invisible in hybrid search.
+
+    Schedule: Runs every 6 hours (balanced between coverage and API costs).
+    - Catches new content added between scheduled runs
+    - Batch size 10 per content type: gradual processing to avoid rate limits
+    - Manual trigger available via execute_ensure_embeddings_coverage endpoint
+    """
+    db_client = get_database_manager_client()
+    stats = db_client.get_embedding_stats()
+
+    # Check for error from get_embedding_stats() first
+    if "error" in stats:
+        logger.error(
+            f"Failed to get embedding stats: {stats['error']} - skipping backfill"
+        )
+        return {"processed": 0, "success": 0, "failed": 0, "error": stats["error"]}
+
+    # Extract totals from new stats structure
+    totals = stats.get("totals", {})
+    without_embeddings = totals.get("without_embeddings", 0)
+    coverage_percent = totals.get("coverage_percent", 0)
+
+    if without_embeddings == 0:
+        logger.info("All content has embeddings, skipping backfill")
+        return {"processed": 0, "success": 0, "failed": 0}
+
+    # Log per-content-type stats for visibility
+    by_type = stats.get("by_type", {})
+    for content_type, type_stats in by_type.items():
+        if type_stats.get("without_embeddings", 0) > 0:
+            logger.info(
+                f"{content_type}: {type_stats['without_embeddings']} items without embeddings "
+                f"({type_stats['coverage_percent']}% coverage)"
+            )
+
+    logger.info(
+        f"Total: {without_embeddings} items without embeddings "
+        f"({coverage_percent}% coverage) - processing all"
+    )
+
+    total_processed = 0
+    total_success = 0
+    total_failed = 0
+
+    # Process in batches until no more missing embeddings
+    while True:
+        result = db_client.backfill_missing_embeddings(batch_size=10)
+
+        total_processed += result["processed"]
+        total_success += result["success"]
+        total_failed += result["failed"]
+
+        if result["processed"] == 0:
+            # No more missing embeddings
+            break
+
+        if result["success"] == 0 and result["processed"] > 0:
+            # All attempts in this batch failed - stop to avoid infinite loop
+            logger.error(
+                f"All {result['processed']} embedding attempts failed - stopping backfill"
+            )
+            break
+
+        # Small delay between batches to avoid rate limits
+        time.sleep(1)
+
+    logger.info(
+        f"Embedding backfill completed: {total_success}/{total_processed} succeeded, "
+        f"{total_failed} failed"
+    )
+    return {
+        "processed": total_processed,
+        "success": total_success,
+        "failed": total_failed,
+    }
+
+
 # Monitoring functions are now imported from monitoring module
 
 
@@ -475,6 +558,19 @@ class Scheduler(AppService):
                 jobstore=Jobstores.EXECUTION.value,
             )
 
+            # Embedding Coverage - Every 6 hours
+            # Ensures all approved agents have embeddings for hybrid search
+            # Critical: missing embeddings = agents invisible in search
+            self.scheduler.add_job(
+                ensure_embeddings_coverage,
+                id="ensure_embeddings_coverage",
+                trigger="interval",
+                hours=6,
+                replace_existing=True,
+                max_instances=1,  # Prevent overlapping runs
+                jobstore=Jobstores.EXECUTION.value,
+            )
+
         self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
         self.scheduler.add_listener(job_missed_listener, EVENT_JOB_MISSED)
         self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)
@@ -632,6 +728,11 @@ class Scheduler(AppService):
         """Manually trigger execution accuracy alert checking."""
         return execution_accuracy_alerts()
 
+    @expose
+    def execute_ensure_embeddings_coverage(self):
+        """Manually trigger embedding backfill for approved store agents."""
+        return ensure_embeddings_coverage()
+
 
 class SchedulerClient(AppServiceClient):
     @classmethod

autogpt_platform/backend/backend/executor/utils.py

@@ -239,14 +239,19 @@ async def _validate_node_input_credentials(
     graph: GraphModel,
     user_id: str,
     nodes_input_masks: Optional[NodesInputMasks] = None,
-) -> dict[str, dict[str, str]]:
+) -> tuple[dict[str, dict[str, str]], set[str]]:
     """
-    Checks all credentials for all nodes of the graph and returns structured errors.
+    Checks all credentials for all nodes of the graph and returns structured errors
+    and a set of nodes that should be skipped due to optional missing credentials.
 
     Returns:
-        dict[node_id, dict[field_name, error_message]]: Credential validation errors per node
+        tuple[
+            dict[node_id, dict[field_name, error_message]]: Credential validation errors per node,
+            set[node_id]: Nodes that should be skipped (optional credentials not configured)
+        ]
     """
     credential_errors: dict[str, dict[str, str]] = defaultdict(dict)
+    nodes_to_skip: set[str] = set()
 
     for node in graph.nodes:
         block = node.block
@@ -256,27 +261,46 @@ async def _validate_node_input_credentials(
         if not credentials_fields:
             continue
 
+        # Track if any credential field is missing for this node
+        has_missing_credentials = False
+
         for field_name, credentials_meta_type in credentials_fields.items():
             try:
+                # Check nodes_input_masks first, then input_default
+                field_value = None
                 if (
                     nodes_input_masks
                     and (node_input_mask := nodes_input_masks.get(node.id))
                     and field_name in node_input_mask
                 ):
-                    credentials_meta = credentials_meta_type.model_validate(
-                        node_input_mask[field_name]
-                    )
+                    field_value = node_input_mask[field_name]
                 elif field_name in node.input_default:
-                    credentials_meta = credentials_meta_type.model_validate(
-                        node.input_default[field_name]
-                    )
-                else:
-                    # Missing credentials
-                    credential_errors[node.id][
-                        field_name
-                    ] = "These credentials are required"
-                    continue
+                    # For optional credentials, don't use input_default - treat as missing
+                    # This prevents stale credential IDs from failing validation
+                    if node.credentials_optional:
+                        field_value = None
+                    else:
+                        field_value = node.input_default[field_name]
+
+                # Check if credentials are missing (None, empty, or not present)
+                if field_value is None or (
+                    isinstance(field_value, dict) and not field_value.get("id")
+                ):
+                    has_missing_credentials = True
+                    # If node has credentials_optional flag, mark for skipping instead of error
+                    if node.credentials_optional:
+                        continue  # Don't add error, will be marked for skip after loop
+                    else:
+                        credential_errors[node.id][
+                            field_name
+                        ] = "These credentials are required"
+                        continue
+
+                credentials_meta = credentials_meta_type.model_validate(field_value)
+
             except ValidationError as e:
+                # Validation error means credentials were provided but invalid
+                # This should always be an error, even if optional
                 credential_errors[node.id][field_name] = f"Invalid credentials: {e}"
                 continue
 
@@ -287,6 +311,7 @@ async def _validate_node_input_credentials(
                 )
             except Exception as e:
                 # Handle any errors fetching credentials
+                # If credentials were explicitly configured but unavailable, it's an error
                 credential_errors[node.id][
                     field_name
                 ] = f"Credentials not available: {e}"
@@ -313,7 +338,19 @@ async def _validate_node_input_credentials(
                 ] = "Invalid credentials: type/provider mismatch"
                 continue
 
-    return credential_errors
+        # If node has optional credentials and any are missing, mark for skipping
+        # But only if there are no other errors for this node
+        if (
+            has_missing_credentials
+            and node.credentials_optional
+            and node.id not in credential_errors
+        ):
+            nodes_to_skip.add(node.id)
+            logger.info(
+                f"Node #{node.id} will be skipped: optional credentials not configured"
+            )
+
+    return credential_errors, nodes_to_skip
 
 
 def make_node_credentials_input_map(
@@ -355,21 +392,25 @@ async def validate_graph_with_credentials(
     graph: GraphModel,
     user_id: str,
     nodes_input_masks: Optional[NodesInputMasks] = None,
-) -> Mapping[str, Mapping[str, str]]:
+) -> tuple[Mapping[str, Mapping[str, str]], set[str]]:
     """
-    Validate graph including credentials and return structured errors per node.
+    Validate graph including credentials and return structured errors per node,
+    along with a set of nodes that should be skipped due to optional missing credentials.
 
     Returns:
-        dict[node_id, dict[field_name, error_message]]: Validation errors per node
+        tuple[
+            dict[node_id, dict[field_name, error_message]]: Validation errors per node,
+            set[node_id]: Nodes that should be skipped (optional credentials not configured)
+        ]
     """
     # Get input validation errors
     node_input_errors = GraphModel.validate_graph_get_errors(
         graph, for_run=True, nodes_input_masks=nodes_input_masks
     )
 
-    # Get credential input/availability/validation errors
-    node_credential_input_errors = await _validate_node_input_credentials(
-        graph, user_id, nodes_input_masks
+    # Get credential input/availability/validation errors and nodes to skip
+    node_credential_input_errors, nodes_to_skip = (
+        await _validate_node_input_credentials(graph, user_id, nodes_input_masks)
     )
 
     # Merge credential errors with structural errors
@@ -378,7 +419,7 @@ async def validate_graph_with_credentials(
             node_input_errors[node_id] = {}
         node_input_errors[node_id].update(field_errors)
 
-    return node_input_errors
+    return node_input_errors, nodes_to_skip
 
 
 async def _construct_starting_node_execution_input(
@@ -386,7 +427,7 @@ async def _construct_starting_node_execution_input(
     user_id: str,
     graph_inputs: BlockInput,
     nodes_input_masks: Optional[NodesInputMasks] = None,
-) -> list[tuple[str, BlockInput]]:
+) -> tuple[list[tuple[str, BlockInput]], set[str]]:
     """
     Validates and prepares the input data for executing a graph.
     This function checks the graph for starting nodes, validates the input data
@@ -400,11 +441,14 @@ async def _construct_starting_node_execution_input(
         node_credentials_map: `dict[node_id, dict[input_name, CredentialsMetaInput]]`
 
     Returns:
-        list[tuple[str, BlockInput]]: A list of tuples, each containing the node ID and
-            the corresponding input data for that node.
+        tuple[
+            list[tuple[str, BlockInput]]: A list of tuples, each containing the node ID
+                and the corresponding input data for that node.
+            set[str]: Node IDs that should be skipped (optional credentials not configured)
+        ]
     """
     # Use new validation function that includes credentials
-    validation_errors = await validate_graph_with_credentials(
+    validation_errors, nodes_to_skip = await validate_graph_with_credentials(
         graph, user_id, nodes_input_masks
     )
     n_error_nodes = len(validation_errors)
@@ -445,7 +489,7 @@ async def _construct_starting_node_execution_input(
             "No starting nodes found for the graph, make sure an AgentInput or blocks with no inbound links are present as starting nodes."
         )
 
-    return nodes_input
+    return nodes_input, nodes_to_skip
 
 
 async def validate_and_construct_node_execution_input(
@@ -456,7 +500,7 @@ async def validate_and_construct_node_execution_input(
     graph_credentials_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,
     nodes_input_masks: Optional[NodesInputMasks] = None,
     is_sub_graph: bool = False,
-) -> tuple[GraphModel, list[tuple[str, BlockInput]], NodesInputMasks]:
+) -> tuple[GraphModel, list[tuple[str, BlockInput]], NodesInputMasks, set[str]]:
     """
     Public wrapper that handles graph fetching, credential mapping, and validation+construction.
     This centralizes the logic used by both scheduler validation and actual execution.
@@ -473,6 +517,7 @@ async def validate_and_construct_node_execution_input(
         GraphModel: Full graph object for the given `graph_id`.
         list[tuple[node_id, BlockInput]]: Starting node IDs with corresponding inputs.
         dict[str, BlockInput]: Node input masks including all passed-in credentials.
+        set[str]: Node IDs that should be skipped (optional credentials not configured).
 
     Raises:
         NotFoundError: If the graph is not found.
@@ -514,14 +559,16 @@ async def validate_and_construct_node_execution_input(
         nodes_input_masks or {},
     )
 
-    starting_nodes_input = await _construct_starting_node_execution_input(
-        graph=graph,
-        user_id=user_id,
-        graph_inputs=graph_inputs,
-        nodes_input_masks=nodes_input_masks,
+    starting_nodes_input, nodes_to_skip = (
+        await _construct_starting_node_execution_input(
+            graph=graph,
+            user_id=user_id,
+            graph_inputs=graph_inputs,
+            nodes_input_masks=nodes_input_masks,
+        )
     )
 
-    return graph, starting_nodes_input, nodes_input_masks
+    return graph, starting_nodes_input, nodes_input_masks, nodes_to_skip
 
 
 def _merge_nodes_input_masks(
@@ -779,6 +826,9 @@ async def add_graph_execution(
 
         # Use existing execution's compiled input masks
         compiled_nodes_input_masks = graph_exec.nodes_input_masks or {}
+        # For resumed executions, nodes_to_skip was already determined at creation time
+        # TODO: Consider storing nodes_to_skip in DB if we need to preserve it across resumes
+        nodes_to_skip: set[str] = set()
 
         logger.info(f"Resuming graph execution #{graph_exec.id} for graph #{graph_id}")
     else:
@@ -787,7 +837,7 @@ async def add_graph_execution(
         )
 
         # Create new execution
-        graph, starting_nodes_input, compiled_nodes_input_masks = (
+        graph, starting_nodes_input, compiled_nodes_input_masks, nodes_to_skip = (
             await validate_and_construct_node_execution_input(
                 graph_id=graph_id,
                 user_id=user_id,
@@ -836,6 +886,7 @@ async def add_graph_execution(
     try:
         graph_exec_entry = graph_exec.to_graph_execution_entry(
             compiled_nodes_input_masks=compiled_nodes_input_masks,
+            nodes_to_skip=nodes_to_skip,
             execution_context=execution_context,
         )
         logger.info(f"Publishing execution {graph_exec.id} to execution queue")

autogpt_platform/backend/backend/executor/utils_test.py

@@ -367,10 +367,13 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):
     )
 
     # Setup mock returns
+    # The function returns (graph, starting_nodes_input, compiled_nodes_input_masks, nodes_to_skip)
+    nodes_to_skip: set[str] = set()
     mock_validate.return_value = (
         mock_graph,
         starting_nodes_input,
         compiled_nodes_input_masks,
+        nodes_to_skip,
     )
     mock_prisma.is_connected.return_value = True
     mock_edb.create_graph_execution = mocker.AsyncMock(return_value=mock_graph_exec)
@@ -456,3 +459,212 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):
     # Both executions should succeed (though they create different objects)
     assert result1 == mock_graph_exec
     assert result2 == mock_graph_exec_2
+
+
+# ============================================================================
+# Tests for Optional Credentials Feature
+# ============================================================================
+
+
+@pytest.mark.asyncio
+async def test_validate_node_input_credentials_returns_nodes_to_skip(
+    mocker: MockerFixture,
+):
+    """
+    Test that _validate_node_input_credentials returns nodes_to_skip set
+    for nodes with credentials_optional=True and missing credentials.
+    """
+    from backend.executor.utils import _validate_node_input_credentials
+
+    # Create a mock node with credentials_optional=True
+    mock_node = mocker.MagicMock()
+    mock_node.id = "node-with-optional-creds"
+    mock_node.credentials_optional = True
+    mock_node.input_default = {}  # No credentials configured
+
+    # Create a mock block with credentials field
+    mock_block = mocker.MagicMock()
+    mock_credentials_field_type = mocker.MagicMock()
+    mock_block.input_schema.get_credentials_fields.return_value = {
+        "credentials": mock_credentials_field_type
+    }
+    mock_node.block = mock_block
+
+    # Create mock graph
+    mock_graph = mocker.MagicMock()
+    mock_graph.nodes = [mock_node]
+
+    # Call the function
+    errors, nodes_to_skip = await _validate_node_input_credentials(
+        graph=mock_graph,
+        user_id="test-user-id",
+        nodes_input_masks=None,
+    )
+
+    # Node should be in nodes_to_skip, not in errors
+    assert mock_node.id in nodes_to_skip
+    assert mock_node.id not in errors
+
+
+@pytest.mark.asyncio
+async def test_validate_node_input_credentials_required_missing_creds_error(
+    mocker: MockerFixture,
+):
+    """
+    Test that _validate_node_input_credentials returns errors
+    for nodes with credentials_optional=False and missing credentials.
+    """
+    from backend.executor.utils import _validate_node_input_credentials
+
+    # Create a mock node with credentials_optional=False (required)
+    mock_node = mocker.MagicMock()
+    mock_node.id = "node-with-required-creds"
+    mock_node.credentials_optional = False
+    mock_node.input_default = {}  # No credentials configured
+
+    # Create a mock block with credentials field
+    mock_block = mocker.MagicMock()
+    mock_credentials_field_type = mocker.MagicMock()
+    mock_block.input_schema.get_credentials_fields.return_value = {
+        "credentials": mock_credentials_field_type
+    }
+    mock_node.block = mock_block
+
+    # Create mock graph
+    mock_graph = mocker.MagicMock()
+    mock_graph.nodes = [mock_node]
+
+    # Call the function
+    errors, nodes_to_skip = await _validate_node_input_credentials(
+        graph=mock_graph,
+        user_id="test-user-id",
+        nodes_input_masks=None,
+    )
+
+    # Node should be in errors, not in nodes_to_skip
+    assert mock_node.id in errors
+    assert "credentials" in errors[mock_node.id]
+    assert "required" in errors[mock_node.id]["credentials"].lower()
+    assert mock_node.id not in nodes_to_skip
+
+
+@pytest.mark.asyncio
+async def test_validate_graph_with_credentials_returns_nodes_to_skip(
+    mocker: MockerFixture,
+):
+    """
+    Test that validate_graph_with_credentials returns nodes_to_skip set
+    from _validate_node_input_credentials.
+    """
+    from backend.executor.utils import validate_graph_with_credentials
+
+    # Mock _validate_node_input_credentials to return specific values
+    mock_validate = mocker.patch(
+        "backend.executor.utils._validate_node_input_credentials"
+    )
+    expected_errors = {"node1": {"field": "error"}}
+    expected_nodes_to_skip = {"node2", "node3"}
+    mock_validate.return_value = (expected_errors, expected_nodes_to_skip)
+
+    # Mock GraphModel with validate_graph_get_errors method
+    mock_graph = mocker.MagicMock()
+    mock_graph.validate_graph_get_errors.return_value = {}
+
+    # Call the function
+    errors, nodes_to_skip = await validate_graph_with_credentials(
+        graph=mock_graph,
+        user_id="test-user-id",
+        nodes_input_masks=None,
+    )
+
+    # Verify nodes_to_skip is passed through
+    assert nodes_to_skip == expected_nodes_to_skip
+    assert "node1" in errors
+
+
+@pytest.mark.asyncio
+async def test_add_graph_execution_with_nodes_to_skip(mocker: MockerFixture):
+    """
+    Test that add_graph_execution properly passes nodes_to_skip
+    to the graph execution entry.
+    """
+    from backend.data.execution import GraphExecutionWithNodes
+    from backend.executor.utils import add_graph_execution
+
+    # Mock data
+    graph_id = "test-graph-id"
+    user_id = "test-user-id"
+    inputs = {"test_input": "test_value"}
+    graph_version = 1
+
+    # Mock the graph object
+    mock_graph = mocker.MagicMock()
+    mock_graph.version = graph_version
+
+    # Starting nodes and masks
+    starting_nodes_input = [("node1", {"input1": "value1"})]
+    compiled_nodes_input_masks = {}
+    nodes_to_skip = {"skipped-node-1", "skipped-node-2"}
+
+    # Mock the graph execution object
+    mock_graph_exec = mocker.MagicMock(spec=GraphExecutionWithNodes)
+    mock_graph_exec.id = "execution-id-123"
+    mock_graph_exec.node_executions = []
+
+    # Track what's passed to to_graph_execution_entry
+    captured_kwargs = {}
+
+    def capture_to_entry(**kwargs):
+        captured_kwargs.update(kwargs)
+        return mocker.MagicMock()
+
+    mock_graph_exec.to_graph_execution_entry.side_effect = capture_to_entry
+
+    # Setup mocks
+    mock_validate = mocker.patch(
+        "backend.executor.utils.validate_and_construct_node_execution_input"
+    )
+    mock_edb = mocker.patch("backend.executor.utils.execution_db")
+    mock_prisma = mocker.patch("backend.executor.utils.prisma")
+    mock_udb = mocker.patch("backend.executor.utils.user_db")
+    mock_gdb = mocker.patch("backend.executor.utils.graph_db")
+    mock_get_queue = mocker.patch("backend.executor.utils.get_async_execution_queue")
+    mock_get_event_bus = mocker.patch(
+        "backend.executor.utils.get_async_execution_event_bus"
+    )
+
+    # Setup returns - include nodes_to_skip in the tuple
+    mock_validate.return_value = (
+        mock_graph,
+        starting_nodes_input,
+        compiled_nodes_input_masks,
+        nodes_to_skip,  # This should be passed through
+    )
+    mock_prisma.is_connected.return_value = True
+    mock_edb.create_graph_execution = mocker.AsyncMock(return_value=mock_graph_exec)
+    mock_edb.update_graph_execution_stats = mocker.AsyncMock(
+        return_value=mock_graph_exec
+    )
+    mock_edb.update_node_execution_status_batch = mocker.AsyncMock()
+
+    mock_user = mocker.MagicMock()
+    mock_user.timezone = "UTC"
+    mock_settings = mocker.MagicMock()
+    mock_settings.human_in_the_loop_safe_mode = True
+
+    mock_udb.get_user_by_id = mocker.AsyncMock(return_value=mock_user)
+    mock_gdb.get_graph_settings = mocker.AsyncMock(return_value=mock_settings)
+    mock_get_queue.return_value = mocker.AsyncMock()
+    mock_get_event_bus.return_value = mocker.MagicMock(publish=mocker.AsyncMock())
+
+    # Call the function
+    await add_graph_execution(
+        graph_id=graph_id,
+        user_id=user_id,
+        inputs=inputs,
+        graph_version=graph_version,
+    )
+
+    # Verify nodes_to_skip was passed to to_graph_execution_entry
+    assert "nodes_to_skip" in captured_kwargs
+    assert captured_kwargs["nodes_to_skip"] == nodes_to_skip

autogpt_platform/backend/backend/integrations/oauth/__init__.py

@@ -8,6 +8,7 @@ from .discord import DiscordOAuthHandler
 from .github import GitHubOAuthHandler
 from .google import GoogleOAuthHandler
 from .notion import NotionOAuthHandler
+from .reddit import RedditOAuthHandler
 from .twitter import TwitterOAuthHandler
 
 if TYPE_CHECKING:
@@ -20,6 +21,7 @@ _ORIGINAL_HANDLERS = [
     GitHubOAuthHandler,
     GoogleOAuthHandler,
     NotionOAuthHandler,
+    RedditOAuthHandler,
     TwitterOAuthHandler,
     TodoistOAuthHandler,
 ]

autogpt_platform/backend/backend/integrations/oauth/reddit.py

@@ -0,0 +1,208 @@
+import time
+import urllib.parse
+from typing import ClassVar, Optional
+
+from pydantic import SecretStr
+
+from backend.data.model import OAuth2Credentials
+from backend.integrations.oauth.base import BaseOAuthHandler
+from backend.integrations.providers import ProviderName
+from backend.util.request import Requests
+from backend.util.settings import Settings
+
+settings = Settings()
+
+
+class RedditOAuthHandler(BaseOAuthHandler):
+    """
+    Reddit OAuth 2.0 handler.
+
+    Based on the documentation at:
+    - https://github.com/reddit-archive/reddit/wiki/OAuth2
+
+    Notes:
+    - Reddit requires `duration=permanent` to get refresh tokens
+    - Access tokens expire after 1 hour (3600 seconds)
+    - Reddit requires HTTP Basic Auth for token requests
+    - Reddit requires a unique User-Agent header
+    """
+
+    PROVIDER_NAME = ProviderName.REDDIT
+    DEFAULT_SCOPES: ClassVar[list[str]] = [
+        "identity",  # Get username, verify auth
+        "read",  # Access posts and comments
+        "submit",  # Submit new posts and comments
+        "edit",  # Edit own posts and comments
+        "history",  # Access user's post history
+        "privatemessages",  # Access inbox and send private messages
+        "flair",  # Access and set flair on posts/subreddits
+    ]
+
+    AUTHORIZE_URL = "https://www.reddit.com/api/v1/authorize"
+    TOKEN_URL = "https://www.reddit.com/api/v1/access_token"
+    USERNAME_URL = "https://oauth.reddit.com/api/v1/me"
+    REVOKE_URL = "https://www.reddit.com/api/v1/revoke_token"
+
+    def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.redirect_uri = redirect_uri
+
+    def get_login_url(
+        self, scopes: list[str], state: str, code_challenge: Optional[str]
+    ) -> str:
+        """Generate Reddit OAuth 2.0 authorization URL"""
+        scopes = self.handle_default_scopes(scopes)
+
+        params = {
+            "response_type": "code",
+            "client_id": self.client_id,
+            "redirect_uri": self.redirect_uri,
+            "scope": " ".join(scopes),
+            "state": state,
+            "duration": "permanent",  # Required for refresh tokens
+        }
+
+        return f"{self.AUTHORIZE_URL}?{urllib.parse.urlencode(params)}"
+
+    async def exchange_code_for_tokens(
+        self, code: str, scopes: list[str], code_verifier: Optional[str]
+    ) -> OAuth2Credentials:
+        """Exchange authorization code for access tokens"""
+        scopes = self.handle_default_scopes(scopes)
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": settings.config.reddit_user_agent,
+        }
+
+        data = {
+            "grant_type": "authorization_code",
+            "code": code,
+            "redirect_uri": self.redirect_uri,
+        }
+
+        # Reddit requires HTTP Basic Auth for token requests
+        auth = (self.client_id, self.client_secret)
+
+        response = await Requests().post(
+            self.TOKEN_URL, headers=headers, data=data, auth=auth
+        )
+
+        if not response.ok:
+            error_text = response.text()
+            raise ValueError(
+                f"Reddit token exchange failed: {response.status} - {error_text}"
+            )
+
+        tokens = response.json()
+
+        if "error" in tokens:
+            raise ValueError(f"Reddit OAuth error: {tokens.get('error')}")
+
+        username = await self._get_username(tokens["access_token"])
+
+        return OAuth2Credentials(
+            provider=self.PROVIDER_NAME,
+            title=None,
+            username=username,
+            access_token=tokens["access_token"],
+            refresh_token=tokens.get("refresh_token"),
+            access_token_expires_at=int(time.time()) + tokens.get("expires_in", 3600),
+            refresh_token_expires_at=None,  # Reddit refresh tokens don't expire
+            scopes=scopes,
+        )
+
+    async def _get_username(self, access_token: str) -> str:
+        """Get the username from the access token"""
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "User-Agent": settings.config.reddit_user_agent,
+        }
+
+        response = await Requests().get(self.USERNAME_URL, headers=headers)
+
+        if not response.ok:
+            raise ValueError(f"Failed to get Reddit username: {response.status}")
+
+        data = response.json()
+        return data.get("name", "unknown")
+
+    async def _refresh_tokens(
+        self, credentials: OAuth2Credentials
+    ) -> OAuth2Credentials:
+        """Refresh access tokens using refresh token"""
+        if not credentials.refresh_token:
+            raise ValueError("No refresh token available")
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": settings.config.reddit_user_agent,
+        }
+
+        data = {
+            "grant_type": "refresh_token",
+            "refresh_token": credentials.refresh_token.get_secret_value(),
+        }
+
+        auth = (self.client_id, self.client_secret)
+
+        response = await Requests().post(
+            self.TOKEN_URL, headers=headers, data=data, auth=auth
+        )
+
+        if not response.ok:
+            error_text = response.text()
+            raise ValueError(
+                f"Reddit token refresh failed: {response.status} - {error_text}"
+            )
+
+        tokens = response.json()
+
+        if "error" in tokens:
+            raise ValueError(f"Reddit OAuth error: {tokens.get('error')}")
+
+        username = await self._get_username(tokens["access_token"])
+
+        # Reddit may or may not return a new refresh token
+        new_refresh_token = tokens.get("refresh_token")
+        if new_refresh_token:
+            refresh_token: SecretStr | None = SecretStr(new_refresh_token)
+        elif credentials.refresh_token:
+            # Keep the existing refresh token
+            refresh_token = credentials.refresh_token
+        else:
+            refresh_token = None
+
+        return OAuth2Credentials(
+            id=credentials.id,
+            provider=self.PROVIDER_NAME,
+            title=credentials.title,
+            username=username,
+            access_token=tokens["access_token"],
+            refresh_token=refresh_token,
+            access_token_expires_at=int(time.time()) + tokens.get("expires_in", 3600),
+            refresh_token_expires_at=None,
+            scopes=credentials.scopes,
+        )
+
+    async def revoke_tokens(self, credentials: OAuth2Credentials) -> bool:
+        """Revoke the access token"""
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": settings.config.reddit_user_agent,
+        }
+
+        data = {
+            "token": credentials.access_token.get_secret_value(),
+            "token_type_hint": "access_token",
+        }
+
+        auth = (self.client_id, self.client_secret)
+
+        response = await Requests().post(
+            self.REVOKE_URL, headers=headers, data=data, auth=auth
+        )
+
+        # Reddit returns 204 No Content on successful revocation
+        return response.ok

autogpt_platform/backend/backend/server/v2/llm/db.py

@@ -1,849 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Iterable, Sequence, cast
-
-import prisma
-import prisma.models
-
-from backend.data.db import transaction
-from backend.server.v2.llm import model as llm_model
-
-
-def _json_dict(value: Any | None) -> dict[str, Any]:
-    if not value:
-        return {}
-    if isinstance(value, dict):
-        return value
-    return {}
-
-
-def _map_cost(record: prisma.models.LlmModelCost) -> llm_model.LlmModelCost:
-    return llm_model.LlmModelCost(
-        id=record.id,
-        unit=record.unit,
-        credit_cost=record.creditCost,
-        credential_provider=record.credentialProvider,
-        credential_id=record.credentialId,
-        credential_type=record.credentialType,
-        currency=record.currency,
-        metadata=_json_dict(record.metadata),
-    )
-
-
-def _map_creator(
-    record: prisma.models.LlmModelCreator,
-) -> llm_model.LlmModelCreator:
-    return llm_model.LlmModelCreator(
-        id=record.id,
-        name=record.name,
-        display_name=record.displayName,
-        description=record.description,
-        website_url=record.websiteUrl,
-        logo_url=record.logoUrl,
-        metadata=_json_dict(record.metadata),
-    )
-
-
-def _map_model(record: prisma.models.LlmModel) -> llm_model.LlmModel:
-    costs = []
-    if record.Costs:
-        costs = [_map_cost(cost) for cost in record.Costs]
-
-    creator = None
-    if hasattr(record, "Creator") and record.Creator:
-        creator = _map_creator(record.Creator)
-
-    return llm_model.LlmModel(
-        id=record.id,
-        slug=record.slug,
-        display_name=record.displayName,
-        description=record.description,
-        provider_id=record.providerId,
-        creator_id=record.creatorId,
-        creator=creator,
-        context_window=record.contextWindow,
-        max_output_tokens=record.maxOutputTokens,
-        is_enabled=record.isEnabled,
-        is_recommended=record.isRecommended,
-        capabilities=_json_dict(record.capabilities),
-        metadata=_json_dict(record.metadata),
-        costs=costs,
-    )
-
-
-def _map_provider(record: prisma.models.LlmProvider) -> llm_model.LlmProvider:
-    models: list[llm_model.LlmModel] = []
-    if record.Models:
-        models = [_map_model(model) for model in record.Models]
-
-    return llm_model.LlmProvider(
-        id=record.id,
-        name=record.name,
-        display_name=record.displayName,
-        description=record.description,
-        default_credential_provider=record.defaultCredentialProvider,
-        default_credential_id=record.defaultCredentialId,
-        default_credential_type=record.defaultCredentialType,
-        supports_tools=record.supportsTools,
-        supports_json_output=record.supportsJsonOutput,
-        supports_reasoning=record.supportsReasoning,
-        supports_parallel_tool=record.supportsParallelTool,
-        metadata=_json_dict(record.metadata),
-        models=models,
-    )
-
-
-async def list_providers(
-    include_models: bool = True, enabled_only: bool = False
-) -> list[llm_model.LlmProvider]:
-    """
-    List all LLM providers.
-
-    Args:
-        include_models: Whether to include models for each provider
-        enabled_only: If True, only include enabled models (for public routes)
-    """
-    include: Any = None
-    if include_models:
-        model_where = {"isEnabled": True} if enabled_only else None
-        include = {
-            "Models": {
-                "include": {"Costs": True, "Creator": True},
-                "where": model_where,
-            }
-        }
-    records = await prisma.models.LlmProvider.prisma().find_many(include=include)
-    return [_map_provider(record) for record in records]
-
-
-async def upsert_provider(
-    request: llm_model.UpsertLlmProviderRequest,
-    provider_id: str | None = None,
-) -> llm_model.LlmProvider:
-    data: Any = {
-        "name": request.name,
-        "displayName": request.display_name,
-        "description": request.description,
-        "defaultCredentialProvider": request.default_credential_provider,
-        "defaultCredentialId": request.default_credential_id,
-        "defaultCredentialType": request.default_credential_type,
-        "supportsTools": request.supports_tools,
-        "supportsJsonOutput": request.supports_json_output,
-        "supportsReasoning": request.supports_reasoning,
-        "supportsParallelTool": request.supports_parallel_tool,
-        "metadata": request.metadata,
-    }
-    include: Any = {"Models": {"include": {"Costs": True, "Creator": True}}}
-    if provider_id:
-        record = await prisma.models.LlmProvider.prisma().update(
-            where={"id": provider_id},
-            data=data,
-            include=include,
-        )
-    else:
-        record = await prisma.models.LlmProvider.prisma().create(
-            data=data,
-            include=include,
-        )
-    if record is None:
-        raise ValueError("Failed to create/update provider")
-    return _map_provider(record)
-
-
-async def list_models(
-    provider_id: str | None = None, enabled_only: bool = False
-) -> list[llm_model.LlmModel]:
-    """
-    List LLM models.
-
-    Args:
-        provider_id: Optional filter by provider ID
-        enabled_only: If True, only return enabled models (for public routes)
-    """
-    where: Any = {}
-    if provider_id:
-        where["providerId"] = provider_id
-    if enabled_only:
-        where["isEnabled"] = True
-
-    records = await prisma.models.LlmModel.prisma().find_many(
-        where=where if where else None,
-        include={"Costs": True, "Creator": True},
-    )
-    return [_map_model(record) for record in records]
-
-
-def _cost_create_payload(
-    costs: Sequence[llm_model.LlmModelCostInput],
-) -> dict[str, Iterable[dict[str, Any]]]:
-
-    create_items = []
-    for cost in costs:
-        item: dict[str, Any] = {
-            "unit": cost.unit,
-            "creditCost": cost.credit_cost,
-            "credentialProvider": cost.credential_provider,
-        }
-        # Only include optional fields if they have values
-        if cost.credential_id:
-            item["credentialId"] = cost.credential_id
-        if cost.credential_type:
-            item["credentialType"] = cost.credential_type
-        if cost.currency:
-            item["currency"] = cost.currency
-        # Handle metadata - use Prisma Json type
-        if cost.metadata is not None and cost.metadata != {}:
-            item["metadata"] = prisma.Json(cost.metadata)
-        create_items.append(item)
-    return {"create": create_items}
-
-
-async def create_model(
-    request: llm_model.CreateLlmModelRequest,
-) -> llm_model.LlmModel:
-    data: Any = {
-        "slug": request.slug,
-        "displayName": request.display_name,
-        "description": request.description,
-        "providerId": request.provider_id,
-        "contextWindow": request.context_window,
-        "maxOutputTokens": request.max_output_tokens,
-        "isEnabled": request.is_enabled,
-        "capabilities": request.capabilities,
-        "metadata": request.metadata,
-        "Costs": _cost_create_payload(request.costs),
-    }
-    if request.creator_id:
-        data["creatorId"] = request.creator_id
-
-    record = await prisma.models.LlmModel.prisma().create(
-        data=data,
-        include={"Costs": True, "Creator": True},
-    )
-    return _map_model(record)
-
-
-async def update_model(
-    model_id: str,
-    request: llm_model.UpdateLlmModelRequest,
-) -> llm_model.LlmModel:
-    # Build scalar field updates (non-relation fields)
-    scalar_data: Any = {}
-    if request.display_name is not None:
-        scalar_data["displayName"] = request.display_name
-    if request.description is not None:
-        scalar_data["description"] = request.description
-    if request.context_window is not None:
-        scalar_data["contextWindow"] = request.context_window
-    if request.max_output_tokens is not None:
-        scalar_data["maxOutputTokens"] = request.max_output_tokens
-    if request.is_enabled is not None:
-        scalar_data["isEnabled"] = request.is_enabled
-    if request.capabilities is not None:
-        scalar_data["capabilities"] = request.capabilities
-    if request.metadata is not None:
-        scalar_data["metadata"] = request.metadata
-    # Foreign keys can be updated directly as scalar fields
-    if request.provider_id is not None:
-        scalar_data["providerId"] = request.provider_id
-    if request.creator_id is not None:
-        # Empty string means remove the creator
-        scalar_data["creatorId"] = request.creator_id if request.creator_id else None
-
-    # If we have costs to update, we need to handle them separately
-    # because nested writes have different constraints
-    if request.costs is not None:
-        # Wrap cost replacement in a transaction for atomicity
-        async with transaction() as tx:
-            # First update scalar fields
-            if scalar_data:
-                await tx.llmmodel.update(
-                    where={"id": model_id},
-                    data=scalar_data,
-                )
-            # Then handle costs: delete existing and create new
-            await tx.llmmodelcost.delete_many(where={"llmModelId": model_id})
-            if request.costs:
-                cost_payload = _cost_create_payload(request.costs)
-                for cost_item in cost_payload["create"]:
-                    cost_item["llmModelId"] = model_id
-                    await tx.llmmodelcost.create(data=cast(Any, cost_item))
-        # Fetch the updated record (outside transaction)
-        record = await prisma.models.LlmModel.prisma().find_unique(
-            where={"id": model_id},
-            include={"Costs": True, "Creator": True},
-        )
-    else:
-        # No costs update - simple update
-        record = await prisma.models.LlmModel.prisma().update(
-            where={"id": model_id},
-            data=scalar_data,
-            include={"Costs": True, "Creator": True},
-        )
-
-    if not record:
-        raise ValueError(f"Model with id '{model_id}' not found")
-    return _map_model(record)
-
-
-async def toggle_model(
-    model_id: str,
-    is_enabled: bool,
-    migrate_to_slug: str | None = None,
-    migration_reason: str | None = None,
-    custom_credit_cost: int | None = None,
-) -> llm_model.ToggleLlmModelResponse:
-    """
-    Toggle a model's enabled status, optionally migrating workflows when disabling.
-
-    Args:
-        model_id: UUID of the model to toggle
-        is_enabled: New enabled status
-        migrate_to_slug: If disabling and this is provided, migrate all workflows
-                         using this model to the specified replacement model
-        migration_reason: Optional reason for the migration (e.g., "Provider outage")
-        custom_credit_cost: Optional custom pricing override for migrated workflows.
-                           When set, the billing system should use this cost instead
-                           of the target model's cost for affected nodes.
-
-    Returns:
-        ToggleLlmModelResponse with the updated model and optional migration stats
-    """
-    import json
-
-    # Get the model being toggled
-    model = await prisma.models.LlmModel.prisma().find_unique(
-        where={"id": model_id}, include={"Costs": True}
-    )
-    if not model:
-        raise ValueError(f"Model with id '{model_id}' not found")
-
-    nodes_migrated = 0
-    migration_id: str | None = None
-
-    # If disabling with migration, perform migration first
-    if not is_enabled and migrate_to_slug:
-        # Validate replacement model exists and is enabled
-        replacement = await prisma.models.LlmModel.prisma().find_unique(
-            where={"slug": migrate_to_slug}
-        )
-        if not replacement:
-            raise ValueError(f"Replacement model '{migrate_to_slug}' not found")
-        if not replacement.isEnabled:
-            raise ValueError(
-                f"Replacement model '{migrate_to_slug}' is disabled. "
-                f"Please enable it before using it as a replacement."
-            )
-
-        # Perform all operations atomically within a single transaction
-        # This ensures no nodes are missed between query and update
-        async with transaction() as tx:
-            # Get the IDs of nodes that will be migrated (inside transaction for consistency)
-            node_ids_result = await tx.query_raw(
-                """
-                SELECT id
-                FROM "AgentNode"
-                WHERE "constantInput"::jsonb->>'model' = $1
-                FOR UPDATE
-                """,
-                model.slug,
-            )
-            migrated_node_ids = (
-                [row["id"] for row in node_ids_result] if node_ids_result else []
-            )
-            nodes_migrated = len(migrated_node_ids)
-
-            if nodes_migrated > 0:
-                # Update by IDs to ensure we only update the exact nodes we queried
-                node_ids_pg_array = "{" + ",".join(migrated_node_ids) + "}"
-                await tx.execute_raw(
-                    """
-                    UPDATE "AgentNode"
-                    SET "constantInput" = JSONB_SET(
-                        "constantInput"::jsonb,
-                        '{model}',
-                        to_jsonb($1::text)
-                    )
-                    WHERE id::text = ANY($2::text[])
-                    """,
-                    migrate_to_slug,
-                    node_ids_pg_array,
-                )
-
-            record = await tx.llmmodel.update(
-                where={"id": model_id},
-                data={"isEnabled": is_enabled},
-                include={"Costs": True},
-            )
-
-            # Create migration record for revert capability
-            if nodes_migrated > 0:
-                migration_data: Any = {
-                    "sourceModelSlug": model.slug,
-                    "targetModelSlug": migrate_to_slug,
-                    "reason": migration_reason,
-                    "migratedNodeIds": json.dumps(migrated_node_ids),
-                    "nodeCount": nodes_migrated,
-                    "customCreditCost": custom_credit_cost,
-                }
-                migration_record = await tx.llmmodelmigration.create(
-                    data=migration_data
-                )
-                migration_id = migration_record.id
-    else:
-        # Simple toggle without migration
-        record = await prisma.models.LlmModel.prisma().update(
-            where={"id": model_id},
-            data={"isEnabled": is_enabled},
-            include={"Costs": True},
-        )
-
-    if record is None:
-        raise ValueError(f"Model with id '{model_id}' not found")
-    return llm_model.ToggleLlmModelResponse(
-        model=_map_model(record),
-        nodes_migrated=nodes_migrated,
-        migrated_to_slug=migrate_to_slug if nodes_migrated > 0 else None,
-        migration_id=migration_id,
-    )
-
-
-async def get_model_usage(model_id: str) -> llm_model.LlmModelUsageResponse:
-    """Get usage count for a model."""
-    import prisma as prisma_module
-
-    model = await prisma.models.LlmModel.prisma().find_unique(where={"id": model_id})
-    if not model:
-        raise ValueError(f"Model with id '{model_id}' not found")
-
-    count_result = await prisma_module.get_client().query_raw(
-        """
-        SELECT COUNT(*) as count
-        FROM "AgentNode"
-        WHERE "constantInput"::jsonb->>'model' = $1
-        """,
-        model.slug,
-    )
-    node_count = int(count_result[0]["count"]) if count_result else 0
-
-    return llm_model.LlmModelUsageResponse(model_slug=model.slug, node_count=node_count)
-
-
-async def delete_model(
-    model_id: str, replacement_model_slug: str
-) -> llm_model.DeleteLlmModelResponse:
-    """
-    Delete a model and migrate all AgentNodes using it to a replacement model.
-
-    This performs an atomic operation within a database transaction:
-    1. Validates the model exists
-    2. Validates the replacement model exists and is enabled
-    3. Counts affected nodes
-    4. Migrates all AgentNode.constantInput->model to replacement (in transaction)
-    5. Deletes the LlmModel record (CASCADE deletes costs) (in transaction)
-
-    Args:
-        model_id: UUID of the model to delete
-        replacement_model_slug: Slug of the model to migrate to
-
-    Returns:
-        DeleteLlmModelResponse with migration stats
-
-    Raises:
-        ValueError: If model not found, replacement not found, or replacement is disabled
-    """
-    # 1. Get the model being deleted (validation - outside transaction)
-    model = await prisma.models.LlmModel.prisma().find_unique(
-        where={"id": model_id}, include={"Costs": True}
-    )
-    if not model:
-        raise ValueError(f"Model with id '{model_id}' not found")
-
-    deleted_slug = model.slug
-    deleted_display_name = model.displayName
-
-    # 2. Validate replacement model exists and is enabled (validation - outside transaction)
-    replacement = await prisma.models.LlmModel.prisma().find_unique(
-        where={"slug": replacement_model_slug}
-    )
-    if not replacement:
-        raise ValueError(f"Replacement model '{replacement_model_slug}' not found")
-    if not replacement.isEnabled:
-        raise ValueError(
-            f"Replacement model '{replacement_model_slug}' is disabled. "
-            f"Please enable it before using it as a replacement."
-        )
-
-    # 3 & 4. Perform count, migration and deletion atomically within a transaction
-    nodes_affected = 0
-    async with transaction() as tx:
-        # Count affected nodes (inside transaction for consistency)
-        count_result = await tx.query_raw(
-            """
-            SELECT COUNT(*) as count
-            FROM "AgentNode"
-            WHERE "constantInput"::jsonb->>'model' = $1
-            """,
-            deleted_slug,
-        )
-        nodes_affected = int(count_result[0]["count"]) if count_result else 0
-
-        # Migrate all AgentNode.constantInput->model to replacement
-        if nodes_affected > 0:
-            await tx.execute_raw(
-                """
-                UPDATE "AgentNode"
-                SET "constantInput" = JSONB_SET(
-                    "constantInput"::jsonb,
-                    '{model}',
-                    to_jsonb($1::text)
-                )
-                WHERE "constantInput"::jsonb->>'model' = $2
-                """,
-                replacement_model_slug,
-                deleted_slug,
-            )
-
-        # Delete the model (CASCADE will delete costs automatically)
-        await tx.llmmodel.delete(where={"id": model_id})
-
-    return llm_model.DeleteLlmModelResponse(
-        deleted_model_slug=deleted_slug,
-        deleted_model_display_name=deleted_display_name,
-        replacement_model_slug=replacement_model_slug,
-        nodes_migrated=nodes_affected,
-        message=(
-            f"Successfully deleted model '{deleted_display_name}' ({deleted_slug}) "
-            f"and migrated {nodes_affected} workflow node(s) to '{replacement_model_slug}'."
-        ),
-    )
-
-
-def _map_migration(
-    record: prisma.models.LlmModelMigration,
-) -> llm_model.LlmModelMigration:
-    return llm_model.LlmModelMigration(
-        id=record.id,
-        source_model_slug=record.sourceModelSlug,
-        target_model_slug=record.targetModelSlug,
-        reason=record.reason,
-        node_count=record.nodeCount,
-        custom_credit_cost=record.customCreditCost,
-        is_reverted=record.isReverted,
-        created_at=record.createdAt.isoformat(),
-        reverted_at=record.revertedAt.isoformat() if record.revertedAt else None,
-    )
-
-
-async def list_migrations(
-    include_reverted: bool = False,
-) -> list[llm_model.LlmModelMigration]:
-    """
-    List model migrations, optionally including reverted ones.
-
-    Args:
-        include_reverted: If True, include reverted migrations. Default is False.
-
-    Returns:
-        List of LlmModelMigration records
-    """
-    where: Any = None if include_reverted else {"isReverted": False}
-    records = await prisma.models.LlmModelMigration.prisma().find_many(
-        where=where,
-        order={"createdAt": "desc"},
-    )
-    return [_map_migration(record) for record in records]
-
-
-async def get_migration(migration_id: str) -> llm_model.LlmModelMigration | None:
-    """Get a specific migration by ID."""
-    record = await prisma.models.LlmModelMigration.prisma().find_unique(
-        where={"id": migration_id}
-    )
-    return _map_migration(record) if record else None
-
-
-async def revert_migration(
-    migration_id: str,
-    re_enable_source_model: bool = True,
-) -> llm_model.RevertMigrationResponse:
-    """
-    Revert a model migration, restoring affected nodes to their original model.
-
-    This only reverts the specific nodes that were migrated, not all nodes
-    currently using the target model.
-
-    Args:
-        migration_id: UUID of the migration to revert
-        re_enable_source_model: Whether to re-enable the source model if it's disabled
-
-    Returns:
-        RevertMigrationResponse with revert stats
-
-    Raises:
-        ValueError: If migration not found, already reverted, or source model not available
-    """
-    import json
-    from datetime import datetime, timezone
-
-    # Get the migration record
-    migration = await prisma.models.LlmModelMigration.prisma().find_unique(
-        where={"id": migration_id}
-    )
-    if not migration:
-        raise ValueError(f"Migration with id '{migration_id}' not found")
-
-    if migration.isReverted:
-        raise ValueError(
-            f"Migration '{migration_id}' has already been reverted "
-            f"on {migration.revertedAt.isoformat() if migration.revertedAt else 'unknown date'}"
-        )
-
-    # Check if source model exists
-    source_model = await prisma.models.LlmModel.prisma().find_unique(
-        where={"slug": migration.sourceModelSlug}
-    )
-    if not source_model:
-        raise ValueError(
-            f"Source model '{migration.sourceModelSlug}' no longer exists. "
-            f"Cannot revert migration."
-        )
-
-    # Get the migrated node IDs (Prisma auto-parses JSONB to list)
-    migrated_node_ids: list[str] = (
-        migration.migratedNodeIds
-        if isinstance(migration.migratedNodeIds, list)
-        else json.loads(migration.migratedNodeIds)  # type: ignore
-    )
-    if not migrated_node_ids:
-        raise ValueError("No nodes to revert in this migration")
-
-    # Track if we need to re-enable the source model
-    source_model_was_disabled = not source_model.isEnabled
-    should_re_enable = source_model_was_disabled and re_enable_source_model
-    source_model_re_enabled = False
-
-    # Perform revert atomically
-    async with transaction() as tx:
-        # Re-enable the source model if requested and it was disabled
-        if should_re_enable:
-            await tx.llmmodel.update(
-                where={"id": source_model.id},
-                data={"isEnabled": True},
-            )
-            source_model_re_enabled = True
-
-        # Update only the specific nodes that were migrated
-        # We need to check that they still have the target model (haven't been changed since)
-        # Use a single batch update for efficiency
-        # Format node IDs as PostgreSQL text array literal for comparison
-        node_ids_pg_array = "{" + ",".join(migrated_node_ids) + "}"
-        result = await tx.execute_raw(
-            """
-            UPDATE "AgentNode"
-            SET "constantInput" = JSONB_SET(
-                "constantInput"::jsonb,
-                '{model}',
-                to_jsonb($1::text)
-            )
-            WHERE id::text = ANY($2::text[])
-            AND "constantInput"::jsonb->>'model' = $3
-            """,
-            migration.sourceModelSlug,
-            node_ids_pg_array,
-            migration.targetModelSlug,
-        )
-        nodes_reverted = result if result else 0
-
-        # Mark migration as reverted
-        await tx.llmmodelmigration.update(
-            where={"id": migration_id},
-            data={
-                "isReverted": True,
-                "revertedAt": datetime.now(timezone.utc),
-            },
-        )
-
-    # Calculate nodes that were already changed since migration
-    nodes_already_changed = len(migrated_node_ids) - nodes_reverted
-
-    # Build appropriate message
-    message_parts = [
-        f"Successfully reverted migration: {nodes_reverted} node(s) restored "
-        f"from '{migration.targetModelSlug}' to '{migration.sourceModelSlug}'."
-    ]
-    if nodes_already_changed > 0:
-        message_parts.append(
-            f" {nodes_already_changed} node(s) were already changed and not reverted."
-        )
-    if source_model_re_enabled:
-        message_parts.append(
-            f" Model '{migration.sourceModelSlug}' has been re-enabled."
-        )
-
-    return llm_model.RevertMigrationResponse(
-        migration_id=migration_id,
-        source_model_slug=migration.sourceModelSlug,
-        target_model_slug=migration.targetModelSlug,
-        nodes_reverted=nodes_reverted,
-        nodes_already_changed=nodes_already_changed,
-        source_model_re_enabled=source_model_re_enabled,
-        message="".join(message_parts),
-    )
-
-
-# ============================================================================
-# Creator CRUD operations
-# ============================================================================
-
-
-async def list_creators() -> list[llm_model.LlmModelCreator]:
-    """List all LLM model creators."""
-    records = await prisma.models.LlmModelCreator.prisma().find_many(
-        order={"displayName": "asc"}
-    )
-    return [_map_creator(record) for record in records]
-
-
-async def get_creator(creator_id: str) -> llm_model.LlmModelCreator | None:
-    """Get a specific creator by ID."""
-    record = await prisma.models.LlmModelCreator.prisma().find_unique(
-        where={"id": creator_id}
-    )
-    return _map_creator(record) if record else None
-
-
-async def upsert_creator(
-    request: llm_model.UpsertLlmCreatorRequest,
-    creator_id: str | None = None,
-) -> llm_model.LlmModelCreator:
-    """Create or update a model creator."""
-    data: Any = {
-        "name": request.name,
-        "displayName": request.display_name,
-        "description": request.description,
-        "websiteUrl": request.website_url,
-        "logoUrl": request.logo_url,
-        "metadata": request.metadata,
-    }
-    if creator_id:
-        record = await prisma.models.LlmModelCreator.prisma().update(
-            where={"id": creator_id},
-            data=data,
-        )
-    else:
-        record = await prisma.models.LlmModelCreator.prisma().create(data=data)
-    if record is None:
-        raise ValueError("Failed to create/update creator")
-    return _map_creator(record)
-
-
-async def delete_creator(creator_id: str) -> bool:
-    """
-    Delete a model creator.
-
-    This will set creatorId to NULL on all associated models (due to onDelete: SetNull).
-
-    Args:
-        creator_id: UUID of the creator to delete
-
-    Returns:
-        True if deleted successfully
-
-    Raises:
-        ValueError: If creator not found
-    """
-    creator = await prisma.models.LlmModelCreator.prisma().find_unique(
-        where={"id": creator_id}
-    )
-    if not creator:
-        raise ValueError(f"Creator with id '{creator_id}' not found")
-
-    await prisma.models.LlmModelCreator.prisma().delete(where={"id": creator_id})
-    return True
-
-
-async def get_recommended_model() -> llm_model.LlmModel | None:
-    """
-    Get the currently recommended LLM model.
-
-    Returns:
-        The recommended model, or None if no model is marked as recommended.
-    """
-    record = await prisma.models.LlmModel.prisma().find_first(
-        where={"isRecommended": True, "isEnabled": True},
-        include={"Costs": True, "Creator": True},
-    )
-    return _map_model(record) if record else None
-
-
-async def set_recommended_model(
-    model_id: str,
-) -> tuple[llm_model.LlmModel, str | None]:
-    """
-    Set a model as the recommended model.
-
-    This will clear the isRecommended flag from any other model and set it
-    on the specified model. The model must be enabled.
-
-    Args:
-        model_id: UUID of the model to set as recommended
-
-    Returns:
-        Tuple of (the updated model, previous recommended model slug or None)
-
-    Raises:
-        ValueError: If model not found or not enabled
-    """
-    # First, verify the model exists and is enabled
-    target_model = await prisma.models.LlmModel.prisma().find_unique(
-        where={"id": model_id}
-    )
-    if not target_model:
-        raise ValueError(f"Model with id '{model_id}' not found")
-    if not target_model.isEnabled:
-        raise ValueError(
-            f"Cannot set disabled model '{target_model.slug}' as recommended"
-        )
-
-    # Get the current recommended model (if any)
-    current_recommended = await prisma.models.LlmModel.prisma().find_first(
-        where={"isRecommended": True}
-    )
-    previous_slug = current_recommended.slug if current_recommended else None
-
-    # Use a transaction to ensure atomicity
-    async with transaction() as tx:
-        # Clear isRecommended from all models
-        await tx.llmmodel.update_many(
-            where={"isRecommended": True},
-            data={"isRecommended": False},
-        )
-        # Set the new recommended model
-        await tx.llmmodel.update(
-            where={"id": model_id},
-            data={"isRecommended": True},
-        )
-
-    # Fetch and return the updated model
-    updated_record = await prisma.models.LlmModel.prisma().find_unique(
-        where={"id": model_id},
-        include={"Costs": True, "Creator": True},
-    )
-    if not updated_record:
-        raise ValueError("Failed to fetch updated model")
-
-    return _map_model(updated_record), previous_slug
-
-
-async def get_recommended_model_slug() -> str | None:
-    """
-    Get the slug of the currently recommended LLM model.
-
-    Returns:
-        The slug of the recommended model, or None if no model is marked as recommended.
-    """
-    record = await prisma.models.LlmModel.prisma().find_first(
-        where={"isRecommended": True, "isEnabled": True},
-    )
-    return record.slug if record else None

autogpt_platform/backend/backend/server/v2/llm/model.py

@@ -1,231 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any, Optional
-
-import prisma.enums
-import pydantic
-
-# Pattern for valid model slugs: alphanumeric start, then alphanumeric, dots, underscores, slashes, hyphens
-SLUG_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._/-]*$")
-
-
-class LlmModelCost(pydantic.BaseModel):
-    id: str
-    unit: prisma.enums.LlmCostUnit = prisma.enums.LlmCostUnit.RUN
-    credit_cost: int
-    credential_provider: str
-    credential_id: Optional[str] = None
-    credential_type: Optional[str] = None
-    currency: Optional[str] = None
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-
-
-class LlmModelCreator(pydantic.BaseModel):
-    """Represents the organization that created/trained the model (e.g., OpenAI, Meta)."""
-
-    id: str
-    name: str
-    display_name: str
-    description: Optional[str] = None
-    website_url: Optional[str] = None
-    logo_url: Optional[str] = None
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-
-
-class LlmModel(pydantic.BaseModel):
-    id: str
-    slug: str
-    display_name: str
-    description: Optional[str] = None
-    provider_id: str
-    creator_id: Optional[str] = None
-    creator: Optional[LlmModelCreator] = None
-    context_window: int
-    max_output_tokens: Optional[int] = None
-    is_enabled: bool = True
-    is_recommended: bool = False
-    capabilities: dict[str, Any] = pydantic.Field(default_factory=dict)
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-    costs: list[LlmModelCost] = pydantic.Field(default_factory=list)
-
-
-class LlmProvider(pydantic.BaseModel):
-    id: str
-    name: str
-    display_name: str
-    description: Optional[str] = None
-    default_credential_provider: Optional[str] = None
-    default_credential_id: Optional[str] = None
-    default_credential_type: Optional[str] = None
-    supports_tools: bool = True
-    supports_json_output: bool = True
-    supports_reasoning: bool = False
-    supports_parallel_tool: bool = False
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-    models: list[LlmModel] = pydantic.Field(default_factory=list)
-
-
-class LlmProvidersResponse(pydantic.BaseModel):
-    providers: list[LlmProvider]
-
-
-class LlmModelsResponse(pydantic.BaseModel):
-    models: list[LlmModel]
-
-
-class LlmCreatorsResponse(pydantic.BaseModel):
-    creators: list[LlmModelCreator]
-
-
-class UpsertLlmProviderRequest(pydantic.BaseModel):
-    name: str
-    display_name: str
-    description: Optional[str] = None
-    default_credential_provider: Optional[str] = None
-    default_credential_id: Optional[str] = None
-    default_credential_type: Optional[str] = "api_key"
-    supports_tools: bool = True
-    supports_json_output: bool = True
-    supports_reasoning: bool = False
-    supports_parallel_tool: bool = False
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-
-
-class UpsertLlmCreatorRequest(pydantic.BaseModel):
-    name: str
-    display_name: str
-    description: Optional[str] = None
-    website_url: Optional[str] = None
-    logo_url: Optional[str] = None
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-
-
-class LlmModelCostInput(pydantic.BaseModel):
-    unit: prisma.enums.LlmCostUnit = prisma.enums.LlmCostUnit.RUN
-    credit_cost: int
-    credential_provider: str
-    credential_id: Optional[str] = None
-    credential_type: Optional[str] = "api_key"
-    currency: Optional[str] = None
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-
-
-class CreateLlmModelRequest(pydantic.BaseModel):
-    slug: str
-    display_name: str
-    description: Optional[str] = None
-    provider_id: str
-    creator_id: Optional[str] = None
-    context_window: int
-    max_output_tokens: Optional[int] = None
-    is_enabled: bool = True
-    capabilities: dict[str, Any] = pydantic.Field(default_factory=dict)
-    metadata: dict[str, Any] = pydantic.Field(default_factory=dict)
-    costs: list[LlmModelCostInput]
-
-    @pydantic.field_validator("slug")
-    @classmethod
-    def validate_slug(cls, v: str) -> str:
-        if not v or len(v) > 100:
-            raise ValueError("Slug must be 1-100 characters")
-        if not SLUG_PATTERN.match(v):
-            raise ValueError(
-                "Slug must start with alphanumeric and contain only "
-                "alphanumeric characters, dots, underscores, slashes, or hyphens"
-            )
-        return v
-
-
-class UpdateLlmModelRequest(pydantic.BaseModel):
-    display_name: Optional[str] = None
-    description: Optional[str] = None
-    context_window: Optional[int] = None
-    max_output_tokens: Optional[int] = None
-    is_enabled: Optional[bool] = None
-    capabilities: Optional[dict[str, Any]] = None
-    metadata: Optional[dict[str, Any]] = None
-    provider_id: Optional[str] = None
-    creator_id: Optional[str] = None
-    costs: Optional[list[LlmModelCostInput]] = None
-
-
-class ToggleLlmModelRequest(pydantic.BaseModel):
-    is_enabled: bool
-    migrate_to_slug: Optional[str] = None
-    migration_reason: Optional[str] = None  # e.g., "Provider outage"
-    # Custom pricing override for migrated workflows. When set, billing should use
-    # this cost instead of the target model's cost for affected nodes.
-    # See LlmModelMigration in schema.prisma for full documentation.
-    custom_credit_cost: Optional[int] = None
-
-
-class ToggleLlmModelResponse(pydantic.BaseModel):
-    model: LlmModel
-    nodes_migrated: int = 0
-    migrated_to_slug: Optional[str] = None
-    migration_id: Optional[str] = None  # ID of the migration record for revert
-
-
-class DeleteLlmModelResponse(pydantic.BaseModel):
-    deleted_model_slug: str
-    deleted_model_display_name: str
-    replacement_model_slug: str
-    nodes_migrated: int
-    message: str
-
-
-class LlmModelUsageResponse(pydantic.BaseModel):
-    model_slug: str
-    node_count: int
-
-
-# Migration tracking models
-class LlmModelMigration(pydantic.BaseModel):
-    id: str
-    source_model_slug: str
-    target_model_slug: str
-    reason: Optional[str] = None
-    node_count: int
-    # Custom pricing override - billing should use this instead of target model's cost
-    custom_credit_cost: Optional[int] = None
-    is_reverted: bool = False
-    created_at: str  # ISO datetime string
-    reverted_at: Optional[str] = None
-
-
-class LlmMigrationsResponse(pydantic.BaseModel):
-    migrations: list[LlmModelMigration]
-
-
-class RevertMigrationRequest(pydantic.BaseModel):
-    re_enable_source_model: bool = (
-        True  # Whether to re-enable the source model if disabled
-    )
-
-
-class RevertMigrationResponse(pydantic.BaseModel):
-    migration_id: str
-    source_model_slug: str
-    target_model_slug: str
-    nodes_reverted: int
-    nodes_already_changed: int = (
-        0  # Nodes that were modified since migration (not reverted)
-    )
-    source_model_re_enabled: bool = False  # Whether the source model was re-enabled
-    message: str
-
-
-class SetRecommendedModelRequest(pydantic.BaseModel):
-    model_id: str
-
-
-class SetRecommendedModelResponse(pydantic.BaseModel):
-    model: LlmModel
-    previous_recommended_slug: Optional[str] = None
-    message: str
-
-
-class RecommendedModelResponse(pydantic.BaseModel):
-    model: Optional[LlmModel] = None
-    slug: Optional[str] = None

autogpt_platform/backend/backend/server/v2/llm/routes.py

@@ -1,25 +0,0 @@
-import autogpt_libs.auth
-import fastapi
-
-from backend.server.v2.llm import db as llm_db
-from backend.server.v2.llm import model as llm_model
-
-router = fastapi.APIRouter(
-    prefix="/llm",
-    tags=["llm"],
-    dependencies=[fastapi.Security(autogpt_libs.auth.requires_user)],
-)
-
-
-@router.get("/models", response_model=llm_model.LlmModelsResponse)
-async def list_models():
-    """List all enabled LLM models available to users."""
-    models = await llm_db.list_models(enabled_only=True)
-    return llm_model.LlmModelsResponse(models=models)
-
-
-@router.get("/providers", response_model=llm_model.LlmProvidersResponse)
-async def list_providers():
-    """List all LLM providers with their enabled models."""
-    providers = await llm_db.list_providers(include_models=True, enabled_only=True)
-    return llm_model.LlmProvidersResponse(providers=providers)

autogpt_platform/backend/backend/util/clients.py

@@ -10,6 +10,7 @@ from backend.util.settings import Settings
 settings = Settings()
 
 if TYPE_CHECKING:
+    from openai import AsyncOpenAI
     from supabase import AClient, Client
 
     from backend.data.execution import (
@@ -139,6 +140,24 @@ async def get_async_supabase() -> "AClient":
     )
 
 
+# ============ OpenAI Client ============ #
+
+
+@cached(ttl_seconds=3600)
+def get_openai_client() -> "AsyncOpenAI | None":
+    """
+    Get a process-cached async OpenAI client for embeddings.
+
+    Returns None if API key is not configured.
+    """
+    from openai import AsyncOpenAI
+
+    api_key = settings.secrets.openai_internal_api_key
+    if not api_key:
+        return None
+    return AsyncOpenAI(api_key=api_key)
+
+
 # ============ Notification Queue Helpers ============ #
 
 

autogpt_platform/backend/backend/util/settings.py

@@ -264,7 +264,7 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
     )
 
     reddit_user_agent: str = Field(
-        default="AutoGPT:1.0 (by /u/autogpt)",
+        default="web:AutoGPT:v0.6.0 (by /u/autogpt)",
         description="The user agent for the Reddit API",
     )
 

autogpt_platform/backend/gen_prisma_types_stub.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""
+Generate a lightweight stub for prisma/types.py that collapses all exported
+symbols to Any. This prevents Pyright from spending time/budget on Prisma's
+query DSL types while keeping runtime behavior unchanged.
+
+Usage:
+    poetry run gen-prisma-stub
+
+This script automatically finds the prisma package location and generates
+the types.pyi stub file in the same directory as types.py.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import sys
+from pathlib import Path
+from typing import Iterable, Set
+
+
+def _iter_assigned_names(target: ast.expr) -> Iterable[str]:
+    """Extract names from assignment targets (handles tuple unpacking)."""
+    if isinstance(target, ast.Name):
+        yield target.id
+    elif isinstance(target, (ast.Tuple, ast.List)):
+        for elt in target.elts:
+            yield from _iter_assigned_names(elt)
+
+
+def _is_private(name: str) -> bool:
+    """Check if a name is private (starts with _ but not __)."""
+    return name.startswith("_") and not name.startswith("__")
+
+
+def _is_safe_type_alias(node: ast.Assign) -> bool:
+    """Check if an assignment is a safe type alias that shouldn't be stubbed.
+
+    Safe types are:
+    - Literal types (don't cause type budget issues)
+    - Simple type references (SortMode, SortOrder, etc.)
+    - TypeVar definitions
+    """
+    if not node.value:
+        return False
+
+    # Check if it's a Subscript (like Literal[...], Union[...], TypeVar[...])
+    if isinstance(node.value, ast.Subscript):
+        # Get the base type name
+        if isinstance(node.value.value, ast.Name):
+            base_name = node.value.value.id
+            # Literal types are safe
+            if base_name == "Literal":
+                return True
+            # TypeVar is safe
+            if base_name == "TypeVar":
+                return True
+        elif isinstance(node.value.value, ast.Attribute):
+            # Handle typing_extensions.Literal etc.
+            if node.value.value.attr == "Literal":
+                return True
+
+    # Check if it's a simple Name reference (like SortMode = _types.SortMode)
+    if isinstance(node.value, ast.Attribute):
+        return True
+
+    # Check if it's a Call (like TypeVar(...))
+    if isinstance(node.value, ast.Call):
+        if isinstance(node.value.func, ast.Name):
+            if node.value.func.id == "TypeVar":
+                return True
+
+    return False
+
+
+def collect_top_level_symbols(
+    tree: ast.Module, source_lines: list[str]
+) -> tuple[Set[str], Set[str], list[str], Set[str]]:
+    """Collect all top-level symbols from an AST module.
+
+    Returns:
+        Tuple of (class_names, function_names, safe_variable_sources, unsafe_variable_names)
+        safe_variable_sources contains the actual source code lines for safe variables
+    """
+    classes: Set[str] = set()
+    functions: Set[str] = set()
+    safe_variable_sources: list[str] = []
+    unsafe_variables: Set[str] = set()
+
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            if not _is_private(node.name):
+                classes.add(node.name)
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            if not _is_private(node.name):
+                functions.add(node.name)
+        elif isinstance(node, ast.Assign):
+            is_safe = _is_safe_type_alias(node)
+            names = []
+            for t in node.targets:
+                for n in _iter_assigned_names(t):
+                    if not _is_private(n):
+                        names.append(n)
+            if names:
+                if is_safe:
+                    # Extract the source code for this assignment
+                    start_line = node.lineno - 1  # 0-indexed
+                    end_line = node.end_lineno if node.end_lineno else node.lineno
+                    source = "\n".join(source_lines[start_line:end_line])
+                    safe_variable_sources.append(source)
+                else:
+                    unsafe_variables.update(names)
+        elif isinstance(node, ast.AnnAssign) and node.target:
+            # Annotated assignments are always stubbed
+            for n in _iter_assigned_names(node.target):
+                if not _is_private(n):
+                    unsafe_variables.add(n)
+
+    return classes, functions, safe_variable_sources, unsafe_variables
+
+
+def find_prisma_types_path() -> Path:
+    """Find the prisma types.py file in the installed package."""
+    spec = importlib.util.find_spec("prisma")
+    if spec is None or spec.origin is None:
+        raise RuntimeError("Could not find prisma package. Is it installed?")
+
+    prisma_dir = Path(spec.origin).parent
+    types_path = prisma_dir / "types.py"
+
+    if not types_path.exists():
+        raise RuntimeError(f"prisma/types.py not found at {types_path}")
+
+    return types_path
+
+
+def generate_stub(src_path: Path, stub_path: Path) -> int:
+    """Generate the .pyi stub file from the source types.py."""
+    code = src_path.read_text(encoding="utf-8", errors="ignore")
+    source_lines = code.splitlines()
+    tree = ast.parse(code, filename=str(src_path))
+    classes, functions, safe_variable_sources, unsafe_variables = (
+        collect_top_level_symbols(tree, source_lines)
+    )
+
+    header = """\
+# -*- coding: utf-8 -*-
+# Auto-generated stub file - DO NOT EDIT
+# Generated by gen_prisma_types_stub.py
+#
+# This stub intentionally collapses complex Prisma query DSL types to Any.
+# Prisma's generated types can explode Pyright's type inference budgets
+# on large schemas. We collapse them to Any so the rest of the codebase
+# can remain strongly typed while keeping runtime behavior unchanged.
+#
+# Safe types (Literal, TypeVar, simple references) are preserved from the
+# original types.py to maintain proper type checking where possible.
+
+from __future__ import annotations
+from typing import Any
+from typing_extensions import Literal
+
+# Re-export commonly used typing constructs that may be imported from this module
+from typing import TYPE_CHECKING, TypeVar, Generic, Union, Optional, List, Dict
+
+# Base type alias for stubbed Prisma types - allows any dict structure
+_PrismaDict = dict[str, Any]
+
+"""
+
+    lines = [header]
+
+    # Include safe variable definitions (Literal types, TypeVars, etc.)
+    lines.append("# Safe type definitions preserved from original types.py")
+    for source in safe_variable_sources:
+        lines.append(source)
+    lines.append("")
+
+    # Stub all classes and unsafe variables uniformly as dict[str, Any] aliases
+    # This allows:
+    # 1. Use in type annotations: x: SomeType
+    # 2. Constructor calls: SomeType(...)
+    # 3. Dict literal assignments: x: SomeType = {...}
+    lines.append(
+        "# Stubbed types (collapsed to dict[str, Any] to prevent type budget exhaustion)"
+    )
+    all_stubbed = sorted(classes | unsafe_variables)
+    for name in all_stubbed:
+        lines.append(f"{name} = _PrismaDict")
+
+    lines.append("")
+
+    # Stub functions
+    for name in sorted(functions):
+        lines.append(f"def {name}(*args: Any, **kwargs: Any) -> Any: ...")
+
+    lines.append("")
+
+    stub_path.write_text("\n".join(lines), encoding="utf-8")
+    return (
+        len(classes)
+        + len(functions)
+        + len(safe_variable_sources)
+        + len(unsafe_variables)
+    )
+
+
+def main() -> None:
+    """Main entry point."""
+    try:
+        types_path = find_prisma_types_path()
+        stub_path = types_path.with_suffix(".pyi")
+
+        print(f"Found prisma types.py at: {types_path}")
+        print(f"Generating stub at: {stub_path}")
+
+        num_symbols = generate_stub(types_path, stub_path)
+        print(f"Generated {stub_path.name} with {num_symbols} Any-typed symbols")
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

autogpt_platform/backend/linter.py

@@ -25,6 +25,9 @@ def run(*command: str) -> None:
 
 
 def lint():
+    # Generate Prisma types stub before running pyright to prevent type budget exhaustion
+    run("gen-prisma-stub")
+
     lint_step_args: list[list[str]] = [
         ["ruff", "check", *TARGET_DIRS, "--exit-zero"],
         ["ruff", "format", "--diff", "--check", LIBS_DIR],
@@ -49,4 +52,6 @@ def format():
     run("ruff", "format", LIBS_DIR)
     run("isort", "--profile", "black", BACKEND_DIR)
     run("black", BACKEND_DIR)
+    # Generate Prisma types stub before running pyright to prevent type budget exhaustion
+    run("gen-prisma-stub")
     run("pyright", *TARGET_DIRS)

autogpt_platform/backend/migrations/20251126113000_add_llm_registry/migration.sql

@@ -1,78 +0,0 @@
--- CreateEnum
-CREATE TYPE "LlmCostUnit" AS ENUM ('RUN', 'TOKENS');
-
--- CreateTable
-CREATE TABLE "LlmProvider" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "name" TEXT NOT NULL,
-    "displayName" TEXT NOT NULL,
-    "description" TEXT,
-    "defaultCredentialProvider" TEXT,
-    "defaultCredentialId" TEXT,
-    "defaultCredentialType" TEXT,
-    "supportsTools" BOOLEAN NOT NULL DEFAULT TRUE,
-    "supportsJsonOutput" BOOLEAN NOT NULL DEFAULT TRUE,
-    "supportsReasoning" BOOLEAN NOT NULL DEFAULT FALSE,
-    "supportsParallelTool" BOOLEAN NOT NULL DEFAULT FALSE,
-    "metadata" JSONB NOT NULL DEFAULT '{}'::jsonb,
-
-    CONSTRAINT "LlmProvider_pkey" PRIMARY KEY ("id"),
-    CONSTRAINT "LlmProvider_name_key" UNIQUE ("name")
-);
-
--- CreateTable
-CREATE TABLE "LlmModel" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "slug" TEXT NOT NULL,
-    "displayName" TEXT NOT NULL,
-    "description" TEXT,
-    "providerId" TEXT NOT NULL,
-    "contextWindow" INTEGER NOT NULL,
-    "maxOutputTokens" INTEGER,
-    "isEnabled" BOOLEAN NOT NULL DEFAULT TRUE,
-    "capabilities" JSONB NOT NULL DEFAULT '{}'::jsonb,
-    "metadata" JSONB NOT NULL DEFAULT '{}'::jsonb,
-
-    CONSTRAINT "LlmModel_pkey" PRIMARY KEY ("id"),
-    CONSTRAINT "LlmModel_slug_key" UNIQUE ("slug")
-);
-
--- CreateTable
-CREATE TABLE "LlmModelCost" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "unit" "LlmCostUnit" NOT NULL DEFAULT 'RUN',
-    "creditCost" INTEGER NOT NULL,
-    "credentialProvider" TEXT NOT NULL,
-    "credentialId" TEXT,
-    "credentialType" TEXT,
-    "currency" TEXT,
-    "metadata" JSONB NOT NULL DEFAULT '{}'::jsonb,
-    "llmModelId" TEXT NOT NULL,
-
-    CONSTRAINT "LlmModelCost_pkey" PRIMARY KEY ("id")
-);
-
--- CreateIndex
-CREATE INDEX "LlmModel_providerId_isEnabled_idx" ON "LlmModel"("providerId", "isEnabled");
-
--- CreateIndex
-CREATE INDEX "LlmModel_slug_idx" ON "LlmModel"("slug");
-
--- CreateIndex
-CREATE INDEX "LlmModelCost_llmModelId_idx" ON "LlmModelCost"("llmModelId");
-
--- CreateIndex
-CREATE INDEX "LlmModelCost_credentialProvider_idx" ON "LlmModelCost"("credentialProvider");
-
--- AddForeignKey
-ALTER TABLE "LlmModel" ADD CONSTRAINT "LlmModel_providerId_fkey" FOREIGN KEY ("providerId") REFERENCES "LlmProvider"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-
--- AddForeignKey
-ALTER TABLE "LlmModelCost" ADD CONSTRAINT "LlmModelCost_llmModelId_fkey" FOREIGN KEY ("llmModelId") REFERENCES "LlmModel"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-

autogpt_platform/backend/migrations/20251126120000_seed_llm_registry/migration.sql

@@ -1,225 +0,0 @@
--- Seed LLM Registry from existing hard-coded data
--- This migration populates the LlmProvider, LlmModel, and LlmModelCost tables
--- with data from the existing MODEL_METADATA and MODEL_COST dictionaries
-
--- Insert Providers
-INSERT INTO "LlmProvider" ("id", "name", "displayName", "description", "defaultCredentialProvider", "defaultCredentialType", "supportsTools", "supportsJsonOutput", "supportsReasoning", "supportsParallelTool", "metadata")
-VALUES
-    (gen_random_uuid(), 'openai', 'OpenAI', 'OpenAI language models', 'openai', 'api_key', true, true, true, true, '{}'::jsonb),
-    (gen_random_uuid(), 'anthropic', 'Anthropic', 'Anthropic Claude models', 'anthropic', 'api_key', true, true, true, false, '{}'::jsonb),
-    (gen_random_uuid(), 'groq', 'Groq', 'Groq inference API', 'groq', 'api_key', false, true, false, false, '{}'::jsonb),
-    (gen_random_uuid(), 'open_router', 'OpenRouter', 'OpenRouter unified API', 'open_router', 'api_key', true, true, false, false, '{}'::jsonb),
-    (gen_random_uuid(), 'aiml_api', 'AI/ML API', 'AI/ML API models', 'aiml_api', 'api_key', false, true, false, false, '{}'::jsonb),
-    (gen_random_uuid(), 'ollama', 'Ollama', 'Ollama local models', 'ollama', 'api_key', false, true, false, false, '{}'::jsonb),
-    (gen_random_uuid(), 'llama_api', 'Llama API', 'Llama API models', 'llama_api', 'api_key', false, true, false, false, '{}'::jsonb),
-    (gen_random_uuid(), 'v0', 'v0', 'v0 by Vercel models', 'v0', 'api_key', true, true, false, false, '{}'::jsonb)
-ON CONFLICT ("name") DO NOTHING;
-
--- Insert Models (using CTEs to reference provider IDs)
-WITH provider_ids AS (
-    SELECT "id", "name" FROM "LlmProvider"
-)
-INSERT INTO "LlmModel" ("id", "slug", "displayName", "description", "providerId", "contextWindow", "maxOutputTokens", "isEnabled", "capabilities", "metadata")
-SELECT
-    gen_random_uuid(),
-    model_slug,
-    model_display_name,
-    NULL,
-    p."id",
-    context_window,
-    max_output_tokens,
-    true,
-    '{}'::jsonb,
-    '{}'::jsonb
-FROM (VALUES
-    -- OpenAI models
-    ('o3', 'O3', 'openai', 200000, 100000),
-    ('o3-mini', 'O3 Mini', 'openai', 200000, 100000),
-    ('o1', 'O1', 'openai', 200000, 100000),
-    ('o1-mini', 'O1 Mini', 'openai', 128000, 65536),
-    ('gpt-5-2025-08-07', 'GPT 5', 'openai', 400000, 128000),
-    ('gpt-5.1-2025-11-13', 'GPT 5.1', 'openai', 400000, 128000),
-    ('gpt-5-mini-2025-08-07', 'GPT 5 Mini', 'openai', 400000, 128000),
-    ('gpt-5-nano-2025-08-07', 'GPT 5 Nano', 'openai', 400000, 128000),
-    ('gpt-5-chat-latest', 'GPT 5 Chat', 'openai', 400000, 16384),
-    ('gpt-4.1-2025-04-14', 'GPT 4.1', 'openai', 1047576, 32768),
-    ('gpt-4.1-mini-2025-04-14', 'GPT 4.1 Mini', 'openai', 1047576, 32768),
-    ('gpt-4o-mini', 'GPT 4o Mini', 'openai', 128000, 16384),
-    ('gpt-4o', 'GPT 4o', 'openai', 128000, 16384),
-    ('gpt-4-turbo', 'GPT 4 Turbo', 'openai', 128000, 4096),
-    ('gpt-3.5-turbo', 'GPT 3.5 Turbo', 'openai', 16385, 4096),
-    -- Anthropic models
-    ('claude-opus-4-1-20250805', 'Claude 4.1 Opus', 'anthropic', 200000, 32000),
-    ('claude-opus-4-20250514', 'Claude 4 Opus', 'anthropic', 200000, 32000),
-    ('claude-sonnet-4-20250514', 'Claude 4 Sonnet', 'anthropic', 200000, 64000),
-    ('claude-opus-4-5-20251101', 'Claude 4.5 Opus', 'anthropic', 200000, 64000),
-    ('claude-sonnet-4-5-20250929', 'Claude 4.5 Sonnet', 'anthropic', 200000, 64000),
-    ('claude-haiku-4-5-20251001', 'Claude 4.5 Haiku', 'anthropic', 200000, 64000),
-    ('claude-3-7-sonnet-20250219', 'Claude 3.7 Sonnet', 'anthropic', 200000, 64000),
-    ('claude-3-haiku-20240307', 'Claude 3 Haiku', 'anthropic', 200000, 4096),
-    -- AI/ML API models
-    ('Qwen/Qwen2.5-72B-Instruct-Turbo', 'Qwen 2.5 72B', 'aiml_api', 32000, 8000),
-    ('nvidia/llama-3.1-nemotron-70b-instruct', 'Llama 3.1 Nemotron 70B', 'aiml_api', 128000, 40000),
-    ('meta-llama/Llama-3.3-70B-Instruct-Turbo', 'Llama 3.3 70B', 'aiml_api', 128000, NULL),
-    ('meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo', 'Meta Llama 3.1 70B', 'aiml_api', 131000, 2000),
-    ('meta-llama/Llama-3.2-3B-Instruct-Turbo', 'Llama 3.2 3B', 'aiml_api', 128000, NULL),
-    -- Groq models
-    ('llama-3.3-70b-versatile', 'Llama 3.3 70B', 'groq', 128000, 32768),
-    ('llama-3.1-8b-instant', 'Llama 3.1 8B', 'groq', 128000, 8192),
-    -- Ollama models
-    ('llama3.3', 'Llama 3.3', 'ollama', 8192, NULL),
-    ('llama3.2', 'Llama 3.2', 'ollama', 8192, NULL),
-    ('llama3', 'Llama 3', 'ollama', 8192, NULL),
-    ('llama3.1:405b', 'Llama 3.1 405B', 'ollama', 8192, NULL),
-    ('dolphin-mistral:latest', 'Dolphin Mistral', 'ollama', 32768, NULL),
-    -- OpenRouter models
-    ('google/gemini-2.5-pro-preview-03-25', 'Gemini 2.5 Pro', 'open_router', 1050000, 8192),
-    ('google/gemini-3-pro-preview', 'Gemini 3 Pro Preview', 'open_router', 1048576, 65535),
-    ('google/gemini-2.5-flash', 'Gemini 2.5 Flash', 'open_router', 1048576, 65535),
-    ('google/gemini-2.0-flash-001', 'Gemini 2.0 Flash', 'open_router', 1048576, 8192),
-    ('google/gemini-2.5-flash-lite-preview-06-17', 'Gemini 2.5 Flash Lite Preview', 'open_router', 1048576, 65535),
-    ('google/gemini-2.0-flash-lite-001', 'Gemini 2.0 Flash Lite', 'open_router', 1048576, 8192),
-    ('mistralai/mistral-nemo', 'Mistral Nemo', 'open_router', 128000, 4096),
-    ('cohere/command-r-08-2024', 'Command R', 'open_router', 128000, 4096),
-    ('cohere/command-r-plus-08-2024', 'Command R Plus', 'open_router', 128000, 4096),
-    ('deepseek/deepseek-chat', 'DeepSeek Chat', 'open_router', 64000, 2048),
-    ('deepseek/deepseek-r1-0528', 'DeepSeek R1', 'open_router', 163840, 163840),
-    ('perplexity/sonar', 'Perplexity Sonar', 'open_router', 127000, 8000),
-    ('perplexity/sonar-pro', 'Perplexity Sonar Pro', 'open_router', 200000, 8000),
-    ('perplexity/sonar-deep-research', 'Perplexity Sonar Deep Research', 'open_router', 128000, 16000),
-    ('nousresearch/hermes-3-llama-3.1-405b', 'Hermes 3 Llama 3.1 405B', 'open_router', 131000, 4096),
-    ('nousresearch/hermes-3-llama-3.1-70b', 'Hermes 3 Llama 3.1 70B', 'open_router', 12288, 12288),
-    ('openai/gpt-oss-120b', 'GPT OSS 120B', 'open_router', 131072, 131072),
-    ('openai/gpt-oss-20b', 'GPT OSS 20B', 'open_router', 131072, 32768),
-    ('amazon/nova-lite-v1', 'Amazon Nova Lite', 'open_router', 300000, 5120),
-    ('amazon/nova-micro-v1', 'Amazon Nova Micro', 'open_router', 128000, 5120),
-    ('amazon/nova-pro-v1', 'Amazon Nova Pro', 'open_router', 300000, 5120),
-    ('microsoft/wizardlm-2-8x22b', 'WizardLM 2 8x22B', 'open_router', 65536, 4096),
-    ('gryphe/mythomax-l2-13b', 'MythoMax L2 13B', 'open_router', 4096, 4096),
-    ('meta-llama/llama-4-scout', 'Llama 4 Scout', 'open_router', 131072, 131072),
-    ('meta-llama/llama-4-maverick', 'Llama 4 Maverick', 'open_router', 1048576, 1000000),
-    ('x-ai/grok-4', 'Grok 4', 'open_router', 256000, 256000),
-    ('x-ai/grok-4-fast', 'Grok 4 Fast', 'open_router', 2000000, 30000),
-    ('x-ai/grok-4.1-fast', 'Grok 4.1 Fast', 'open_router', 2000000, 30000),
-    ('x-ai/grok-code-fast-1', 'Grok Code Fast 1', 'open_router', 256000, 10000),
-    ('moonshotai/kimi-k2', 'Kimi K2', 'open_router', 131000, 131000),
-    ('qwen/qwen3-235b-a22b-thinking-2507', 'Qwen 3 235B Thinking', 'open_router', 262144, 262144),
-    ('qwen/qwen3-coder', 'Qwen 3 Coder', 'open_router', 262144, 262144),
-    -- Llama API models
-    ('Llama-4-Scout-17B-16E-Instruct-FP8', 'Llama 4 Scout', 'llama_api', 128000, 4028),
-    ('Llama-4-Maverick-17B-128E-Instruct-FP8', 'Llama 4 Maverick', 'llama_api', 128000, 4028),
-    ('Llama-3.3-8B-Instruct', 'Llama 3.3 8B', 'llama_api', 128000, 4028),
-    ('Llama-3.3-70B-Instruct', 'Llama 3.3 70B', 'llama_api', 128000, 4028),
-    -- v0 models
-    ('v0-1.5-md', 'v0 1.5 MD', 'v0', 128000, 64000),
-    ('v0-1.5-lg', 'v0 1.5 LG', 'v0', 512000, 64000),
-    ('v0-1.0-md', 'v0 1.0 MD', 'v0', 128000, 64000)
-) AS models(model_slug, model_display_name, provider_name, context_window, max_output_tokens)
-JOIN provider_ids p ON p."name" = models.provider_name
-ON CONFLICT ("slug") DO NOTHING;
-
--- Insert Costs (using CTEs to reference model IDs)
-WITH model_ids AS (
-    SELECT "id", "slug", "providerId" FROM "LlmModel"
-),
-provider_ids AS (
-    SELECT "id", "name" FROM "LlmProvider"
-)
-INSERT INTO "LlmModelCost" ("id", "unit", "creditCost", "credentialProvider", "credentialId", "credentialType", "currency", "metadata", "llmModelId")
-SELECT
-    gen_random_uuid(),
-    'RUN'::"LlmCostUnit",
-    cost,
-    p."name",
-    NULL,
-    'api_key',
-    NULL,
-    '{}'::jsonb,
-    m."id"
-FROM (VALUES
-    -- OpenAI costs
-    ('o3', 4),
-    ('o3-mini', 2),
-    ('o1', 16),
-    ('o1-mini', 4),
-    ('gpt-5-2025-08-07', 2),
-    ('gpt-5.1-2025-11-13', 5),
-    ('gpt-5-mini-2025-08-07', 1),
-    ('gpt-5-nano-2025-08-07', 1),
-    ('gpt-5-chat-latest', 5),
-    ('gpt-4.1-2025-04-14', 2),
-    ('gpt-4.1-mini-2025-04-14', 1),
-    ('gpt-4o-mini', 1),
-    ('gpt-4o', 3),
-    ('gpt-4-turbo', 10),
-    ('gpt-3.5-turbo', 1),
-    -- Anthropic costs
-    ('claude-opus-4-1-20250805', 21),
-    ('claude-opus-4-20250514', 21),
-    ('claude-sonnet-4-20250514', 5),
-    ('claude-haiku-4-5-20251001', 4),
-    ('claude-opus-4-5-20251101', 14),
-    ('claude-sonnet-4-5-20250929', 9),
-    ('claude-3-7-sonnet-20250219', 5),
-    ('claude-3-haiku-20240307', 1),
-    -- AI/ML API costs
-    ('Qwen/Qwen2.5-72B-Instruct-Turbo', 1),
-    ('nvidia/llama-3.1-nemotron-70b-instruct', 1),
-    ('meta-llama/Llama-3.3-70B-Instruct-Turbo', 1),
-    ('meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo', 1),
-    ('meta-llama/Llama-3.2-3B-Instruct-Turbo', 1),
-    -- Groq costs
-    ('llama-3.3-70b-versatile', 1),
-    ('llama-3.1-8b-instant', 1),
-    -- Ollama costs
-    ('llama3.3', 1),
-    ('llama3.2', 1),
-    ('llama3', 1),
-    ('llama3.1:405b', 1),
-    ('dolphin-mistral:latest', 1),
-    -- OpenRouter costs
-    ('google/gemini-2.5-pro-preview-03-25', 4),
-    ('google/gemini-3-pro-preview', 5),
-    ('mistralai/mistral-nemo', 1),
-    ('cohere/command-r-08-2024', 1),
-    ('cohere/command-r-plus-08-2024', 3),
-    ('deepseek/deepseek-chat', 2),
-    ('perplexity/sonar', 1),
-    ('perplexity/sonar-pro', 5),
-    ('perplexity/sonar-deep-research', 10),
-    ('nousresearch/hermes-3-llama-3.1-405b', 1),
-    ('nousresearch/hermes-3-llama-3.1-70b', 1),
-    ('amazon/nova-lite-v1', 1),
-    ('amazon/nova-micro-v1', 1),
-    ('amazon/nova-pro-v1', 1),
-    ('microsoft/wizardlm-2-8x22b', 1),
-    ('gryphe/mythomax-l2-13b', 1),
-    ('meta-llama/llama-4-scout', 1),
-    ('meta-llama/llama-4-maverick', 1),
-    ('x-ai/grok-4', 9),
-    ('x-ai/grok-4-fast', 1),
-    ('x-ai/grok-4.1-fast', 1),
-    ('x-ai/grok-code-fast-1', 1),
-    ('moonshotai/kimi-k2', 1),
-    ('qwen/qwen3-235b-a22b-thinking-2507', 1),
-    ('qwen/qwen3-coder', 9),
-    ('google/gemini-2.5-flash', 1),
-    ('google/gemini-2.0-flash-001', 1),
-    ('google/gemini-2.5-flash-lite-preview-06-17', 1),
-    ('google/gemini-2.0-flash-lite-001', 1),
-    ('deepseek/deepseek-r1-0528', 1),
-    ('openai/gpt-oss-120b', 1),
-    ('openai/gpt-oss-20b', 1),
-    -- Llama API costs
-    ('Llama-4-Scout-17B-16E-Instruct-FP8', 1),
-    ('Llama-4-Maverick-17B-128E-Instruct-FP8', 1),
-    ('Llama-3.3-8B-Instruct', 1),
-    ('Llama-3.3-70B-Instruct', 1),
-    -- v0 costs
-    ('v0-1.5-md', 1),
-    ('v0-1.5-lg', 2),
-    ('v0-1.0-md', 1)
-) AS costs(model_slug, cost)
-JOIN model_ids m ON m."slug" = costs.model_slug
-JOIN provider_ids p ON p."id" = m."providerId";
-

autogpt_platform/backend/migrations/20251218100000_add_llm_model_migration/migration.sql

@@ -1,25 +0,0 @@
--- CreateTable
-CREATE TABLE "LlmModelMigration" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL,
-    "sourceModelSlug" TEXT NOT NULL,
-    "targetModelSlug" TEXT NOT NULL,
-    "reason" TEXT,
-    "migratedNodeIds" JSONB NOT NULL DEFAULT '[]',
-    "nodeCount" INTEGER NOT NULL,
-    "customCreditCost" INTEGER,
-    "isReverted" BOOLEAN NOT NULL DEFAULT false,
-    "revertedAt" TIMESTAMP(3),
-
-    CONSTRAINT "LlmModelMigration_pkey" PRIMARY KEY ("id")
-);
-
--- CreateIndex
-CREATE INDEX "LlmModelMigration_sourceModelSlug_idx" ON "LlmModelMigration"("sourceModelSlug");
-
--- CreateIndex
-CREATE INDEX "LlmModelMigration_targetModelSlug_idx" ON "LlmModelMigration"("targetModelSlug");
-
--- CreateIndex
-CREATE INDEX "LlmModelMigration_isReverted_idx" ON "LlmModelMigration"("isReverted");

autogpt_platform/backend/migrations/20251224100000_add_llm_model_creator/migration.sql

@@ -1,127 +0,0 @@
--- Add LlmModelCreator table
--- Creator represents who made/trained the model (e.g., OpenAI, Meta)
--- This is distinct from Provider who hosts/serves the model (e.g., OpenRouter)
-
--- Create the LlmModelCreator table
-CREATE TABLE "LlmModelCreator" (
-    "id" TEXT NOT NULL,
-    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "updatedAt" TIMESTAMP(3) NOT NULL,
-    "name" TEXT NOT NULL,
-    "displayName" TEXT NOT NULL,
-    "description" TEXT,
-    "websiteUrl" TEXT,
-    "logoUrl" TEXT,
-    "metadata" JSONB NOT NULL DEFAULT '{}',
-
-    CONSTRAINT "LlmModelCreator_pkey" PRIMARY KEY ("id")
-);
-
--- Create unique index on name
-CREATE UNIQUE INDEX "LlmModelCreator_name_key" ON "LlmModelCreator"("name");
-
--- Add creatorId column to LlmModel
-ALTER TABLE "LlmModel" ADD COLUMN "creatorId" TEXT;
-
--- Add foreign key constraint
-ALTER TABLE "LlmModel" ADD CONSTRAINT "LlmModel_creatorId_fkey"
-    FOREIGN KEY ("creatorId") REFERENCES "LlmModelCreator"("id") ON DELETE SET NULL ON UPDATE CASCADE;
-
--- Create index on creatorId
-CREATE INDEX "LlmModel_creatorId_idx" ON "LlmModel"("creatorId");
-
--- Seed creators based on known model creators
-INSERT INTO "LlmModelCreator" ("id", "updatedAt", "name", "displayName", "description", "websiteUrl", "metadata")
-VALUES
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'openai', 'OpenAI', 'Creator of GPT models', 'https://openai.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'anthropic', 'Anthropic', 'Creator of Claude models', 'https://anthropic.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'meta', 'Meta', 'Creator of Llama models', 'https://ai.meta.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'google', 'Google', 'Creator of Gemini models', 'https://deepmind.google', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'mistral', 'Mistral AI', 'Creator of Mistral models', 'https://mistral.ai', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'cohere', 'Cohere', 'Creator of Command models', 'https://cohere.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'deepseek', 'DeepSeek', 'Creator of DeepSeek models', 'https://deepseek.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'perplexity', 'Perplexity AI', 'Creator of Sonar models', 'https://perplexity.ai', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'qwen', 'Qwen (Alibaba)', 'Creator of Qwen models', 'https://qwenlm.github.io', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'xai', 'xAI', 'Creator of Grok models', 'https://x.ai', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'amazon', 'Amazon', 'Creator of Nova models', 'https://aws.amazon.com/bedrock', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'microsoft', 'Microsoft', 'Creator of WizardLM models', 'https://microsoft.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'moonshot', 'Moonshot AI', 'Creator of Kimi models', 'https://moonshot.cn', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'nvidia', 'NVIDIA', 'Creator of Nemotron models', 'https://nvidia.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'nous_research', 'Nous Research', 'Creator of Hermes models', 'https://nousresearch.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'vercel', 'Vercel', 'Creator of v0 models', 'https://vercel.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'cognitive_computations', 'Cognitive Computations', 'Creator of Dolphin models', 'https://erichartford.com', '{}'),
-    (gen_random_uuid(), CURRENT_TIMESTAMP, 'gryphe', 'Gryphe', 'Creator of MythoMax models', 'https://huggingface.co/Gryphe', '{}')
-ON CONFLICT ("name") DO NOTHING;
-
--- Update existing models with their creators
--- OpenAI models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'openai')
-WHERE "slug" LIKE 'gpt-%' OR "slug" LIKE 'o1%' OR "slug" LIKE 'o3%' OR "slug" LIKE 'openai/%';
-
--- Anthropic models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'anthropic')
-WHERE "slug" LIKE 'claude-%';
-
--- Meta/Llama models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'meta')
-WHERE "slug" LIKE 'llama%' OR "slug" LIKE 'Llama%' OR "slug" LIKE 'meta-llama/%' OR "slug" LIKE '%/llama-%';
-
--- Google models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'google')
-WHERE "slug" LIKE 'google/%' OR "slug" LIKE 'gemini%';
-
--- Mistral models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'mistral')
-WHERE "slug" LIKE 'mistral%' OR "slug" LIKE 'mistralai/%';
-
--- Cohere models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'cohere')
-WHERE "slug" LIKE 'cohere/%' OR "slug" LIKE 'command-%';
-
--- DeepSeek models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'deepseek')
-WHERE "slug" LIKE 'deepseek/%' OR "slug" LIKE 'deepseek-%';
-
--- Perplexity models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'perplexity')
-WHERE "slug" LIKE 'perplexity/%' OR "slug" LIKE 'sonar%';
-
--- Qwen models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'qwen')
-WHERE "slug" LIKE 'Qwen/%' OR "slug" LIKE 'qwen/%';
-
--- xAI/Grok models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'xai')
-WHERE "slug" LIKE 'x-ai/%' OR "slug" LIKE 'grok%';
-
--- Amazon models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'amazon')
-WHERE "slug" LIKE 'amazon/%' OR "slug" LIKE 'nova-%';
-
--- Microsoft models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'microsoft')
-WHERE "slug" LIKE 'microsoft/%' OR "slug" LIKE 'wizardlm%';
-
--- Moonshot models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'moonshot')
-WHERE "slug" LIKE 'moonshotai/%' OR "slug" LIKE 'kimi%';
-
--- NVIDIA models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'nvidia')
-WHERE "slug" LIKE 'nvidia/%' OR "slug" LIKE '%nemotron%';
-
--- Nous Research models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'nous_research')
-WHERE "slug" LIKE 'nousresearch/%' OR "slug" LIKE 'hermes%';
-
--- Vercel/v0 models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'vercel')
-WHERE "slug" LIKE 'v0-%';
-
--- Dolphin models (Cognitive Computations / Eric Hartford)
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'cognitive_computations')
-WHERE "slug" LIKE 'dolphin-%';
-
--- Gryphe models
-UPDATE "LlmModel" SET "creatorId" = (SELECT "id" FROM "LlmModelCreator" WHERE "name" = 'gryphe')
-WHERE "slug" LIKE 'gryphe/%' OR "slug" LIKE 'mythomax%';

autogpt_platform/backend/migrations/20260105120000_add_agent_node_model_index/migration.sql

@@ -1,4 +0,0 @@
--- CreateIndex
--- Index for efficient LLM model lookups on AgentNode.constantInput->>'model'
--- This improves performance of model migration queries in the LLM registry
-CREATE INDEX "AgentNode_constantInput_model_idx" ON "AgentNode" ((("constantInput"->>'model')));

autogpt_platform/backend/migrations/20260106150000_add_gpt52_model/migration.sql

@@ -1,52 +0,0 @@
--- Add GPT-5.2 model and update O3 slug
--- This migration adds the new GPT-5.2 model added in dev branch
-
--- Update O3 slug to match dev branch format
-UPDATE "LlmModel"
-SET "slug" = 'o3-2025-04-16'
-WHERE "slug" = 'o3';
-
--- Update cost reference for O3 if needed
--- (costs are linked by model ID, so no update needed)
-
--- Add GPT-5.2 model
-WITH provider_id AS (
-    SELECT "id" FROM "LlmProvider" WHERE "name" = 'openai'
-)
-INSERT INTO "LlmModel" ("id", "slug", "displayName", "description", "providerId", "contextWindow", "maxOutputTokens", "isEnabled", "capabilities", "metadata")
-SELECT
-    gen_random_uuid(),
-    'gpt-5.2-2025-12-11',
-    'GPT 5.2',
-    'OpenAI GPT-5.2 model',
-    p."id",
-    400000,
-    128000,
-    true,
-    '{}'::jsonb,
-    '{}'::jsonb
-FROM provider_id p
-ON CONFLICT ("slug") DO NOTHING;
-
--- Add cost for GPT-5.2
-WITH model_id AS (
-    SELECT m."id", p."name" as provider_name
-    FROM "LlmModel" m
-    JOIN "LlmProvider" p ON p."id" = m."providerId"
-    WHERE m."slug" = 'gpt-5.2-2025-12-11'
-)
-INSERT INTO "LlmModelCost" ("id", "unit", "creditCost", "credentialProvider", "credentialId", "credentialType", "currency", "metadata", "llmModelId")
-SELECT
-    gen_random_uuid(),
-    'RUN'::"LlmCostUnit",
-    3,  -- Same cost tier as GPT-5.1
-    m.provider_name,
-    NULL,
-    'api_key',
-    NULL,
-    '{}'::jsonb,
-    m."id"
-FROM model_id m
-WHERE NOT EXISTS (
-    SELECT 1 FROM "LlmModelCost" c WHERE c."llmModelId" = m."id"
-);

autogpt_platform/backend/migrations/20260107100000_add_llm_recommended_model/migration.sql

@@ -1,11 +0,0 @@
--- Add isRecommended field to LlmModel table
--- This allows admins to mark a model as the recommended default
-
-ALTER TABLE "LlmModel" ADD COLUMN "isRecommended" BOOLEAN NOT NULL DEFAULT false;
-
--- Set gpt-4o-mini as the default recommended model (if it exists)
-UPDATE "LlmModel" SET "isRecommended" = true WHERE "slug" = 'gpt-4o-mini' AND "isEnabled" = true;
-
--- Create unique partial index to enforce only one recommended model at the database level
--- This prevents multiple rows from having isRecommended = true
-CREATE UNIQUE INDEX "LlmModel_single_recommended_idx" ON "LlmModel" ("isRecommended") WHERE "isRecommended" = true;

autogpt_platform/backend/migrations/20260109181714_add_docs_embedding/migration.sql

@@ -0,0 +1,46 @@
+-- CreateExtension
+-- Supabase: pgvector must be enabled via Dashboard → Database → Extensions first
+-- Create in public schema so vector type is available across all schemas
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "vector" WITH SCHEMA "public";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'vector extension not available or already exists, skipping';
+END $$;
+
+-- CreateEnum
+CREATE TYPE "ContentType" AS ENUM ('STORE_AGENT', 'BLOCK', 'INTEGRATION', 'DOCUMENTATION', 'LIBRARY_AGENT');
+
+-- CreateTable
+CREATE TABLE "UnifiedContentEmbedding" (
+    "id" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "contentType" "ContentType" NOT NULL,
+    "contentId" TEXT NOT NULL,
+    "userId" TEXT,
+    "embedding" public.vector(1536) NOT NULL,
+    "searchableText" TEXT NOT NULL,
+    "metadata" JSONB NOT NULL DEFAULT '{}',
+
+    CONSTRAINT "UnifiedContentEmbedding_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_contentType_idx" ON "UnifiedContentEmbedding"("contentType");
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_userId_idx" ON "UnifiedContentEmbedding"("userId");
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_contentType_userId_idx" ON "UnifiedContentEmbedding"("contentType", "userId");
+
+-- CreateIndex
+-- NULLS NOT DISTINCT ensures only one public (NULL userId) embedding per contentType+contentId
+-- Requires PostgreSQL 15+. Supabase uses PostgreSQL 15+.
+CREATE UNIQUE INDEX "UnifiedContentEmbedding_contentType_contentId_userId_key" ON "UnifiedContentEmbedding"("contentType", "contentId", "userId") NULLS NOT DISTINCT;
+
+-- CreateIndex
+-- HNSW index for fast vector similarity search on embeddings
+-- Uses cosine distance operator (<=>), which matches the query in hybrid_search.py
+CREATE INDEX "UnifiedContentEmbedding_embedding_idx" ON "UnifiedContentEmbedding" USING hnsw ("embedding" public.vector_cosine_ops);

autogpt_platform/backend/migrations/20260112173500_add_supabase_extensions_to_platform_schema/migration.sql

@@ -0,0 +1,71 @@
+-- Acknowledge Supabase-managed extensions to prevent drift warnings
+-- These extensions are pre-installed by Supabase in specific schemas
+-- This migration ensures they exist where available (Supabase) or skips gracefully (CI)
+
+-- Create schemas (safe in both CI and Supabase)
+CREATE SCHEMA IF NOT EXISTS "extensions";
+
+-- Extensions that exist in both CI and Supabase
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pgcrypto" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgcrypto extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'uuid-ossp extension not available, skipping';
+END $$;
+
+-- Supabase-specific extensions (skip gracefully in CI)
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pg_stat_statements" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_stat_statements extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pg_net" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_net extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pgjwt" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgjwt extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "graphql";
+    CREATE EXTENSION IF NOT EXISTS "pg_graphql" WITH SCHEMA "graphql";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_graphql extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "pgsodium";
+    CREATE EXTENSION IF NOT EXISTS "pgsodium" WITH SCHEMA "pgsodium";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgsodium extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "vault";
+    CREATE EXTENSION IF NOT EXISTS "supabase_vault" WITH SCHEMA "vault";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'supabase_vault extension not available, skipping';
+END $$;
+
+
+-- Return to platform
+CREATE SCHEMA IF NOT EXISTS "platform";

autogpt_platform/backend/poetry.lock

@@ -1924,14 +1924,14 @@ google = ["google-api-python-client (>=2.0.0)", "google-auth (>=2.0.0)"]
 
 [[package]]
 name = "gravitasml"
-version = "0.1.3"
+version = "0.1.4"
 description = ""
 optional = false
 python-versions = "<4.0,>=3.10"
 groups = ["main"]
 files = [
-    {file = "gravitasml-0.1.3-py3-none-any.whl", hash = "sha256:51ff98b4564b7a61f7796f18d5f2558b919d30b3722579296089645b7bc18b85"},
-    {file = "gravitasml-0.1.3.tar.gz", hash = "sha256:04d240b9fa35878252d57a36032130b6516487468847fcdced1022c032a20f57"},
+    {file = "gravitasml-0.1.4-py3-none-any.whl", hash = "sha256:671a18b11d3d8a0e270c6a80c72cd058458b18d5ef7560d00010e962ab1bca74"},
+    {file = "gravitasml-0.1.4.tar.gz", hash = "sha256:35d0d9fec7431817482d53d9c976e375557c3e041d1eb6928e809324a8c866e3"},
 ]
 
 [package.dependencies]
@@ -7295,4 +7295,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.14"
-content-hash = "b762806d5d58fcf811220890c4705a16dc62b33387af43e3a29399c62a641098"
+content-hash = "a93ba0cea3b465cb6ec3e3f258b383b09f84ea352ccfdbfa112902cde5653fc6"

autogpt_platform/backend/pyproject.toml

@@ -27,7 +27,7 @@ google-api-python-client = "^2.177.0"
 google-auth-oauthlib = "^1.2.2"
 google-cloud-storage = "^3.2.0"
 googlemaps = "^4.10.0"
-gravitasml = "^0.1.3"
+gravitasml = "^0.1.4"
 groq = "^0.30.0"
 html2text = "^2024.2.26"
 jinja2 = "^3.1.6"
@@ -117,6 +117,7 @@ lint = "linter:lint"
 test = "run_tests:test"
 load-store-agents = "test.load_store_agents:run"
 export-api-schema = "backend.cli.generate_openapi_json:main"
+gen-prisma-stub = "gen_prisma_types_stub:main"
 oauth-tool = "backend.cli.oauth_tool:cli"
 
 [tool.isort]
@@ -134,6 +135,9 @@ ignore_patterns = []
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
+# Disable syrupy plugin to avoid conflict with pytest-snapshot
+# Both provide --snapshot-update argument causing ArgumentError
+addopts = "-p no:syrupy"
 filterwarnings = [
     "ignore:'audioop' is deprecated:DeprecationWarning:discord.player",
     "ignore:invalid escape sequence:DeprecationWarning:tweepy.api",

autogpt_platform/backend/schema.prisma

@@ -1,14 +1,15 @@
 datasource db {
-  provider  = "postgresql"
-  url       = env("DATABASE_URL")
-  directUrl = env("DIRECT_URL")
+  provider   = "postgresql"
+  url        = env("DATABASE_URL")
+  directUrl  = env("DIRECT_URL")
+  extensions = [pgvector(map: "vector")]
 }
 
 generator client {
   provider               = "prisma-client-py"
   recursive_type_depth   = -1
   interface              = "asyncio"
-  previewFeatures        = ["views", "fullTextSearch"]
+  previewFeatures        = ["views", "fullTextSearch", "postgresqlExtensions"]
   partial_type_generator = "backend/data/partial_types.py"
 }
 
@@ -127,8 +128,8 @@ model BuilderSearchHistory {
   updatedAt DateTime @default(now()) @updatedAt
 
   searchQuery String
-  filter      String[]  @default([])
-  byCreator   String[]  @default([])
+  filter      String[] @default([])
+  byCreator   String[] @default([])
 
   userId String
   User   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
@@ -721,26 +722,25 @@ view StoreAgent {
   storeListingVersionId String
   updated_at            DateTime
 
-  slug                String
-  agent_name          String
-  agent_video         String?
-  agent_output_demo   String?
-  agent_image         String[]
+  slug              String
+  agent_name        String
+  agent_video       String?
+  agent_output_demo String?
+  agent_image       String[]
 
-  featured         Boolean                  @default(false)
-  creator_username String?
-  creator_avatar   String?
-  sub_heading      String
-  description      String
-  categories       String[]
-  search           Unsupported("tsvector")? @default(dbgenerated("''::tsvector"))
-  runs                Int
-  rating              Float
-  versions            String[]
-  agentGraphVersions  String[]
-  agentGraphId        String
-  is_available        Boolean                  @default(true)
-  useForOnboarding    Boolean                  @default(false)
+  featured           Boolean  @default(false)
+  creator_username   String?
+  creator_avatar     String?
+  sub_heading        String
+  description        String
+  categories         String[]
+  runs               Int
+  rating             Float
+  versions           String[]
+  agentGraphVersions String[]
+  agentGraphId       String
+  is_available       Boolean  @default(true)
+  useForOnboarding   Boolean  @default(false)
 
   // Materialized views used (refreshed every 15 minutes via pg_cron):
   // - mv_agent_run_counts - Pre-aggregated agent execution counts by agentGraphId
@@ -856,14 +856,14 @@ model StoreListingVersion {
   AgentGraph        AgentGraph @relation(fields: [agentGraphId, agentGraphVersion], references: [id, version])
 
   // Content fields
-  name                String
-  subHeading          String
-  videoUrl            String?
-  agentOutputDemoUrl  String?
-  imageUrls           String[]
-  description         String
-  instructions        String?
-  categories          String[]
+  name               String
+  subHeading         String
+  videoUrl           String?
+  agentOutputDemoUrl String?
+  imageUrls          String[]
+  description        String
+  instructions       String?
+  categories         String[]
 
   isFeatured Boolean @default(false)
 
@@ -899,6 +899,9 @@ model StoreListingVersion {
   // Reviews for this specific version
   Reviews StoreListingReview[]
 
+  // Note: Embeddings now stored in UnifiedContentEmbedding table
+  // Use contentType=STORE_AGENT and contentId=storeListingVersionId
+
   @@unique([storeListingId, version])
   @@index([storeListingId, submissionStatus, isAvailable])
   @@index([submissionStatus])
@@ -906,6 +909,42 @@ model StoreListingVersion {
   @@index([agentGraphId, agentGraphVersion]) // Non-unique index for efficient lookups
 }
 
+// Content type enum for unified search across store agents, blocks, docs
+// Note: BLOCK/INTEGRATION are file-based (Python classes), not DB records
+// DOCUMENTATION are file-based (.md files), not DB records  
+// Only STORE_AGENT and LIBRARY_AGENT are stored in database
+enum ContentType {
+  STORE_AGENT // Database: StoreListingVersion
+  BLOCK // File-based: Python classes in /backend/blocks/
+  INTEGRATION // File-based: Python classes (blocks with credentials)
+  DOCUMENTATION // File-based: .md/.mdx files
+  LIBRARY_AGENT // Database: User's personal agents
+}
+
+// Unified embeddings table for all searchable content types
+// Supports both public content (userId=null) and user-specific content (userId=userID)
+model UnifiedContentEmbedding {
+  id        String   @id @default(uuid())
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  // Content identification
+  contentType ContentType
+  contentId   String // DB ID (storeListingVersionId) or file identifier (block.id, file_path)
+  userId      String? // NULL for public content (store, blocks, docs), userId for private content (library agents)
+
+  // Search data
+  embedding      Unsupported("vector(1536)") // pgvector embedding (extension in platform schema)
+  searchableText String // Combined text for search and fallback
+  metadata       Json                        @default("{}") // Content-specific metadata
+
+  @@unique([contentType, contentId, userId], map: "UnifiedContentEmbedding_contentType_contentId_userId_key")
+  @@index([contentType])
+  @@index([userId])
+  @@index([contentType, userId])
+  @@index([embedding], map: "UnifiedContentEmbedding_embedding_idx")
+}
+
 model StoreListingReview {
   id        String   @id @default(uuid())
   createdAt DateTime @default(now())
@@ -987,151 +1026,6 @@ enum APIKeyStatus {
 
 ////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////
-/////////////   LLM REGISTRY AND BILLING DATA   /////////////
-////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////
-
-// LlmCostUnit: Defines how LLM MODEL costs are calculated (per run or per token).
-// This is distinct from BlockCostType (in backend/data/block.py) which defines
-// how BLOCK EXECUTION costs are calculated (per run, per byte, or per second).
-// LlmCostUnit is for pricing individual LLM model API calls in the registry,
-// while BlockCostType is for billing platform block executions.
-enum LlmCostUnit {
-  RUN
-  TOKENS
-}
-
-model LlmModelCreator {
-  id        String   @id @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  name        String  @unique // e.g., "openai", "anthropic", "meta"
-  displayName String // e.g., "OpenAI", "Anthropic", "Meta"
-  description String?
-  websiteUrl  String? // Link to creator's website
-  logoUrl     String? // URL to creator's logo
-
-  metadata Json @default("{}")
-
-  Models LlmModel[]
-}
-
-model LlmProvider {
-  id        String   @id @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  name        String @unique
-  displayName String
-  description String?
-
-  defaultCredentialProvider String?
-  defaultCredentialId       String?
-  defaultCredentialType     String?
-
-  supportsTools        Boolean @default(true)
-  supportsJsonOutput   Boolean @default(true)
-  supportsReasoning    Boolean @default(false)
-  supportsParallelTool Boolean @default(false)
-
-  metadata Json @default("{}")
-
-  Models LlmModel[]
-}
-
-model LlmModel {
-  id        String   @id @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  slug        String @unique
-  displayName String
-  description String?
-
-  providerId String
-  Provider   LlmProvider @relation(fields: [providerId], references: [id], onDelete: Restrict)
-
-  // Creator is the organization that created/trained the model (e.g., OpenAI, Meta)
-  // This is distinct from the provider who hosts/serves the model (e.g., OpenRouter)
-  creatorId String?
-  Creator   LlmModelCreator? @relation(fields: [creatorId], references: [id], onDelete: SetNull)
-
-  contextWindow   Int
-  maxOutputTokens Int?
-  isEnabled       Boolean @default(true)
-  isRecommended   Boolean @default(false)
-
-  capabilities Json @default("{}")
-  metadata     Json @default("{}")
-
-  Costs LlmModelCost[]
-
-  @@index([providerId, isEnabled])
-  @@index([creatorId])
-  @@index([slug])
-}
-
-model LlmModelCost {
-  id        String     @id @default(uuid())
-  createdAt DateTime   @default(now())
-  updatedAt DateTime   @updatedAt
-  unit      LlmCostUnit @default(RUN)
-
-  creditCost Int
-
-  credentialProvider String
-  credentialId       String?
-  credentialType     String?
-  currency           String?
-
-  metadata Json @default("{}")
-
-  llmModelId String
-  Model      LlmModel @relation(fields: [llmModelId], references: [id], onDelete: Cascade)
-
-  @@index([llmModelId])
-  @@index([credentialProvider])
-}
-
-// Tracks model migrations for revert capability
-// When a model is disabled with migration, we record which nodes were affected
-// so they can be reverted when the original model is back online
-model LlmModelMigration {
-  id        String   @id @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  sourceModelSlug String // The original model that was disabled
-  targetModelSlug String // The model workflows were migrated to
-  reason          String? // Why the migration happened (e.g., "Provider outage")
-
-  // Track affected nodes as JSON array of node IDs
-  // Format: ["node-uuid-1", "node-uuid-2", ...]
-  migratedNodeIds Json @default("[]")
-  nodeCount       Int // Number of nodes migrated
-
-  // Custom pricing override for migrated workflows during the migration period.
-  // Use case: When migrating users from an expensive model (e.g., GPT-4) to a cheaper
-  // one (e.g., GPT-3.5), you may want to temporarily maintain the original pricing
-  // to avoid billing surprises, or offer a discount during the transition.
-  //
-  // IMPORTANT: This field is intended for integration with the billing system.
-  // When billing calculates costs for nodes affected by this migration, it should
-  // check if customCreditCost is set and use it instead of the target model's cost.
-  // If null, the target model's normal cost applies.
-  //
-  // TODO: Integrate with billing system to apply this override during cost calculation.
-  customCreditCost Int?
-
-  // Revert tracking
-  isReverted Boolean   @default(false)
-  revertedAt DateTime?
-
-  @@index([sourceModelSlug])
-  @@index([targetModelSlug])
-  @@index([isReverted])
-}
 //////////////   OAUTH PROVIDER TABLES    //////////////////
 ////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////
@@ -1143,16 +1037,16 @@ model OAuthApplication {
   updatedAt DateTime @updatedAt
 
   // Application metadata
-  name               String
-  description        String?
-  logoUrl            String? // URL to app logo stored in GCS
-  clientId           String @unique
-  clientSecret       String // Hashed with Scrypt (same as API keys)
-  clientSecretSalt   String // Salt for Scrypt hashing
+  name             String
+  description      String?
+  logoUrl          String? // URL to app logo stored in GCS
+  clientId         String  @unique
+  clientSecret     String // Hashed with Scrypt (same as API keys)
+  clientSecretSalt String // Salt for Scrypt hashing
 
   // OAuth configuration
   redirectUris String[] // Allowed callback URLs
-  grantTypes   String[] @default(["authorization_code", "refresh_token"])
+  grantTypes   String[]           @default(["authorization_code", "refresh_token"])
   scopes       APIKeyPermission[] // Which permissions the app can request
 
   // Application management

autogpt_platform/backend/snapshots/grph_single

@@ -2,6 +2,7 @@
   "created_at": "2025-09-04T13:37:00",
   "credentials_input_schema": {
     "properties": {},
+    "required": [],
     "title": "TestGraphCredentialsInputSchema",
     "type": "object"
   },

autogpt_platform/backend/snapshots/grphs_all

@@ -2,6 +2,7 @@
   {
     "credentials_input_schema": {
       "properties": {},
+      "required": [],
       "title": "TestGraphCredentialsInputSchema",
       "type": "object"
     },

autogpt_platform/backend/snapshots/lib_agts_search

@@ -4,6 +4,7 @@
       "id": "test-agent-1",
       "graph_id": "test-agent-1",
       "graph_version": 1,
+      "owner_user_id": "3e53486c-cf57-477e-ba2a-cb02dc828e1a",
       "image_url": null,
       "creator_name": "Test Creator",
       "creator_image_url": "",
@@ -41,6 +42,7 @@
       "id": "test-agent-2",
       "graph_id": "test-agent-2",
       "graph_version": 1,
+      "owner_user_id": "3e53486c-cf57-477e-ba2a-cb02dc828e1a",
       "image_url": null,
       "creator_name": "Test Creator",
       "creator_image_url": "",

autogpt_platform/backend/snapshots/sub_success

@@ -1,6 +1,7 @@
 {
   "submissions": [
     {
+      "listing_id": "test-listing-id",
       "agent_id": "test-agent-id",
       "agent_version": 1,
       "name": "Test Agent",

autogpt_platform/cloudflare_worker.js

@@ -0,0 +1,146 @@
+/**
+ * Cloudflare Workers Script for docs.agpt.co → agpt.co/docs migration
+ * 
+ * Deploy this script to handle all redirects with a single JavaScript file.
+ * No rule limits, easy to maintain, handles all edge cases.
+ */
+
+// URL mapping for special cases that don't follow patterns
+const SPECIAL_MAPPINGS = {
+  // Root page
+  '/': '/docs/platform',
+  
+  // Special cases that don't follow standard patterns
+  '/platform/d_id/': '/docs/integrations/block-integrations/d-id',
+  '/platform/blocks/blocks/': '/docs/integrations',
+  '/platform/blocks/decoder_block/': '/docs/integrations/block-integrations/text-decoder',
+  '/platform/blocks/http': '/docs/integrations/block-integrations/send-web-request',
+  '/platform/blocks/llm/': '/docs/integrations/block-integrations/ai-and-llm',
+  '/platform/blocks/time_blocks': '/docs/integrations/block-integrations/time-and-date',
+  '/platform/blocks/text_to_speech_block': '/docs/integrations/block-integrations/text-to-speech',
+  '/platform/blocks/ai_shortform_video_block': '/docs/integrations/block-integrations/ai-shortform-video',
+  '/platform/blocks/replicate_flux_advanced': '/docs/integrations/block-integrations/replicate-flux-advanced',
+  '/platform/blocks/flux_kontext': '/docs/integrations/block-integrations/flux-kontext',
+  '/platform/blocks/ai_condition/': '/docs/integrations/block-integrations/ai-condition',
+  '/platform/blocks/email_block': '/docs/integrations/block-integrations/email',
+  '/platform/blocks/google_maps': '/docs/integrations/block-integrations/google-maps',
+  '/platform/blocks/google/gmail': '/docs/integrations/block-integrations/gmail',
+  '/platform/blocks/github/issues/': '/docs/integrations/block-integrations/github-issues',
+  '/platform/blocks/github/repo/': '/docs/integrations/block-integrations/github-repo',
+  '/platform/blocks/github/pull_requests': '/docs/integrations/block-integrations/github-pull-requests',
+  '/platform/blocks/twitter/twitter': '/docs/integrations/block-integrations/twitter',
+  '/classic/setup/': '/docs/classic/setup/setting-up-autogpt-classic',
+  '/code-of-conduct/': '/docs/classic/help-us-improve-autogpt/code-of-conduct',
+  '/contributing/': '/docs/classic/contributing',
+  '/contribute/': '/docs/contribute',
+  '/forge/components/introduction/': '/docs/classic/forge/introduction'
+};
+
+/**
+ * Transform path by replacing underscores with hyphens and removing trailing slashes
+ */
+function transformPath(path) {
+  return path.replace(/_/g, '-').replace(/\/$/, '');
+}
+
+/**
+ * Handle docs.agpt.co redirects
+ */
+function handleDocsRedirect(url) {
+  const pathname = url.pathname;
+  
+  // Check special mappings first
+  if (SPECIAL_MAPPINGS[pathname]) {
+    return `https://agpt.co${SPECIAL_MAPPINGS[pathname]}`;
+  }
+  
+  // Pattern-based redirects
+  
+  // Platform blocks: /platform/blocks/* → /docs/integrations/block-integrations/*
+  if (pathname.startsWith('/platform/blocks/')) {
+    const blockName = pathname.substring('/platform/blocks/'.length);
+    const transformedName = transformPath(blockName);
+    return `https://agpt.co/docs/integrations/block-integrations/${transformedName}`;
+  }
+  
+  // Platform contributing: /platform/contributing/* → /docs/platform/contributing/*
+  if (pathname.startsWith('/platform/contributing/')) {
+    const subPath = pathname.substring('/platform/contributing/'.length);
+    return `https://agpt.co/docs/platform/contributing/${subPath}`;
+  }
+  
+  // Platform general: /platform/* → /docs/platform/* (with underscore→hyphen)
+  if (pathname.startsWith('/platform/')) {
+    const subPath = pathname.substring('/platform/'.length);
+    const transformedPath = transformPath(subPath);
+    return `https://agpt.co/docs/platform/${transformedPath}`;
+  }
+  
+  // Forge components: /forge/components/* → /docs/classic/forge/introduction/*
+  if (pathname.startsWith('/forge/components/')) {
+    const subPath = pathname.substring('/forge/components/'.length);
+    return `https://agpt.co/docs/classic/forge/introduction/${subPath}`;
+  }
+  
+  // Forge general: /forge/* → /docs/classic/forge/*
+  if (pathname.startsWith('/forge/')) {
+    const subPath = pathname.substring('/forge/'.length);
+    return `https://agpt.co/docs/classic/forge/${subPath}`;
+  }
+  
+  // Classic: /classic/* → /docs/classic/*
+  if (pathname.startsWith('/classic/')) {
+    const subPath = pathname.substring('/classic/'.length);
+    return `https://agpt.co/docs/classic/${subPath}`;
+  }
+  
+  // Default fallback
+  return 'https://agpt.co/docs/';
+}
+
+/**
+ * Main Worker function
+ */
+export default {
+  async fetch(request, env, ctx) {
+    const url = new URL(request.url);
+    
+    // Only handle docs.agpt.co requests
+    if (url.hostname === 'docs.agpt.co') {
+      const redirectUrl = handleDocsRedirect(url);
+      
+      return new Response(null, {
+        status: 301,
+        headers: {
+          'Location': redirectUrl,
+          'Cache-Control': 'max-age=300' // Cache redirects for 5 minutes
+        }
+      });
+    }
+    
+    // For non-docs requests, pass through or return 404
+    return new Response('Not Found', { status: 404 });
+  }
+};
+
+// Test function for local development
+export function testRedirects() {
+  const testCases = [
+    'https://docs.agpt.co/',
+    'https://docs.agpt.co/platform/getting-started/',
+    'https://docs.agpt.co/platform/advanced_setup/',
+    'https://docs.agpt.co/platform/blocks/basic/',
+    'https://docs.agpt.co/platform/blocks/ai_condition/',
+    'https://docs.agpt.co/classic/setup/',
+    'https://docs.agpt.co/forge/components/agents/',
+    'https://docs.agpt.co/contributing/',
+    'https://docs.agpt.co/unknown-page'
+  ];
+  
+  console.log('Testing redirects:');
+  testCases.forEach(testUrl => {
+    const url = new URL(testUrl);
+    const result = handleDocsRedirect(url);
+    console.log(`${testUrl} → ${result}`);
+  });
+}

autogpt_platform/docker-compose.platform.yml

@@ -37,7 +37,7 @@ services:
       context: ../
       dockerfile: autogpt_platform/backend/Dockerfile
       target: migrate
-    command: ["sh", "-c", "poetry run prisma generate && poetry run prisma migrate deploy"]
+    command: ["sh", "-c", "poetry run prisma generate && poetry run gen-prisma-stub && poetry run prisma migrate deploy"]
     develop:
       watch:
         - path: ./

autogpt_platform/frontend/package.json

@@ -46,14 +46,15 @@
     "@radix-ui/react-scroll-area": "1.2.10",
     "@radix-ui/react-select": "2.2.6",
     "@radix-ui/react-separator": "1.1.7",
+    "@radix-ui/react-slider": "1.3.6",
     "@radix-ui/react-slot": "1.2.3",
     "@radix-ui/react-switch": "1.2.6",
     "@radix-ui/react-tabs": "1.1.13",
     "@radix-ui/react-toast": "1.2.15",
     "@radix-ui/react-tooltip": "1.2.8",
-    "@rjsf/core": "5.24.13",
-    "@rjsf/utils": "5.24.13",
-    "@rjsf/validator-ajv8": "5.24.13",
+    "@rjsf/core": "6.1.2",
+    "@rjsf/utils": "6.1.2",
+    "@rjsf/validator-ajv8": "6.1.2",
     "@sentry/nextjs": "10.27.0",
     "@supabase/ssr": "0.7.0",
     "@supabase/supabase-js": "2.78.0",
@@ -91,7 +92,6 @@
     "react-currency-input-field": "4.0.3",
     "react-day-picker": "9.11.1",
     "react-dom": "18.3.1",
-    "react-drag-drop-files": "2.4.0",
     "react-hook-form": "7.66.0",
     "react-icons": "5.5.0",
     "react-markdown": "9.0.3",