fix(backend/store): wrap semantic SELECT in subquery to fix UNION ORDER BY

- ORDER BY uce.embedding was applying to UNION result, not just semantic SELECT
- uce table only exists in semantic SELECT, causing 'missing FROM-clause' error
- Wrapped semantic SELECT in subquery so ORDER BY applies within correct scope
- UNION can now properly combine lexical and semantic candidates

Fixes marketplace search completely failing and falling back to lexical-only

.github/workflows/platform-backend-ci.yml

@@ -176,7 +176,7 @@ jobs:
           }
 
       - name: Run Database Migrations
-        run: poetry run prisma migrate dev --name updates
+        run: poetry run prisma migrate deploy
         env:
           DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
           DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}

autogpt_platform/backend/.gitignore

@@ -18,3 +18,4 @@ load-tests/results/
 load-tests/*.json
 load-tests/*.log
 load-tests/node_modules/*
+migrations/*/rollback*.sql

autogpt_platform/backend/backend/api/features/chat/tools/run_agent_test.py

@@ -1,4 +1,5 @@
 import uuid
+from unittest.mock import AsyncMock, patch
 
 import orjson
 import pytest
@@ -17,6 +18,17 @@ setup_test_data = setup_test_data
 setup_firecrawl_test_data = setup_firecrawl_test_data
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 @pytest.mark.asyncio(scope="session")
 async def test_run_agent(setup_test_data):
     """Test that the run_agent tool successfully executes an approved agent"""

autogpt_platform/backend/backend/api/features/store/db.py

@@ -1,8 +1,7 @@
 import asyncio
 import logging
-import typing
 from datetime import datetime, timezone
-from typing import Literal
+from typing import Any, Literal
 
 import fastapi
 import prisma.enums
@@ -10,7 +9,7 @@ import prisma.errors
 import prisma.models
 import prisma.types
 
-from backend.data.db import query_raw_with_schema, transaction
+from backend.data.db import transaction
 from backend.data.graph import (
     GraphMeta,
     GraphModel,
@@ -30,6 +29,8 @@ from backend.util.settings import Settings
 
 from . import exceptions as store_exceptions
 from . import model as store_model
+from .embeddings import ensure_embedding
+from .hybrid_search import hybrid_search
 
 logger = logging.getLogger(__name__)
 settings = Settings()
@@ -50,128 +51,77 @@ async def get_store_agents(
     page_size: int = 20,
 ) -> store_model.StoreAgentsResponse:
     """
-    Get PUBLIC store agents from the StoreAgent view
+    Get PUBLIC store agents from the StoreAgent view.
+
+    Search behavior:
+    - With search_query: Uses hybrid search (semantic + lexical)
+    - Fallback: If embeddings unavailable, gracefully degrades to lexical-only
+    - Rationale: User-facing endpoint prioritizes availability over accuracy
+
+    Note: Admin operations (approval) use fail-fast to prevent inconsistent state.
     """
     logger.debug(
         f"Getting store agents. featured={featured}, creators={creators}, sorted_by={sorted_by}, search={search_query}, category={category}, page={page}"
     )
 
+    search_used_hybrid = False
+    store_agents: list[store_model.StoreAgent] = []
+    agents: list[dict[str, Any]] = []
+    total = 0
+    total_pages = 0
+
     try:
-        # If search_query is provided, use full-text search
+        # If search_query is provided, use hybrid search (embeddings + tsvector)
         if search_query:
-            offset = (page - 1) * page_size
+            # Try hybrid search combining semantic and lexical signals
+            # Falls back to lexical-only if OpenAI unavailable (user-facing, high SLA)
+            try:
+                agents, total = await hybrid_search(
+                    query=search_query,
+                    featured=featured,
+                    creators=creators,
+                    category=category,
+                    sorted_by="relevance",  # Use hybrid scoring for relevance
+                    page=page,
+                    page_size=page_size,
+                )
+                search_used_hybrid = True
+            except Exception as e:
+                # Log error but fall back to lexical search for better UX
+                logger.error(
+                    f"Hybrid search failed (likely OpenAI unavailable), "
+                    f"falling back to lexical search: {e}"
+                )
+                # search_used_hybrid remains False, will use fallback path below
 
-            # Whitelist allowed order_by columns
-            ALLOWED_ORDER_BY = {
-                "rating": "rating DESC, rank DESC",
-                "runs": "runs DESC, rank DESC",
-                "name": "agent_name ASC, rank ASC",
-                "updated_at": "updated_at DESC, rank DESC",
-            }
+            # Convert hybrid search results (dict format) if hybrid succeeded
+            if search_used_hybrid:
+                total_pages = (total + page_size - 1) // page_size
+                store_agents: list[store_model.StoreAgent] = []
+                for agent in agents:
+                    try:
+                        store_agent = store_model.StoreAgent(
+                            slug=agent["slug"],
+                            agent_name=agent["agent_name"],
+                            agent_image=(
+                                agent["agent_image"][0] if agent["agent_image"] else ""
+                            ),
+                            creator=agent["creator_username"] or "Needs Profile",
+                            creator_avatar=agent["creator_avatar"] or "",
+                            sub_heading=agent["sub_heading"],
+                            description=agent["description"],
+                            runs=agent["runs"],
+                            rating=agent["rating"],
+                        )
+                        store_agents.append(store_agent)
+                    except Exception as e:
+                        logger.error(
+                            f"Error parsing Store agent from hybrid search results: {e}"
+                        )
+                        continue
 
-            # Validate and get order clause
-            if sorted_by and sorted_by in ALLOWED_ORDER_BY:
-                order_by_clause = ALLOWED_ORDER_BY[sorted_by]
-            else:
-                order_by_clause = "updated_at DESC, rank DESC"
-
-            # Build WHERE conditions and parameters list
-            where_parts: list[str] = []
-            params: list[typing.Any] = [search_query]  # $1 - search term
-            param_index = 2  # Start at $2 for next parameter
-
-            # Always filter for available agents
-            where_parts.append("is_available = true")
-
-            if featured:
-                where_parts.append("featured = true")
-
-            if creators and creators:
-                # Use ANY with array parameter
-                where_parts.append(f"creator_username = ANY(${param_index})")
-                params.append(creators)
-                param_index += 1
-
-            if category and category:
-                where_parts.append(f"${param_index} = ANY(categories)")
-                params.append(category)
-                param_index += 1
-
-            sql_where_clause: str = " AND ".join(where_parts) if where_parts else "1=1"
-
-            # Add pagination params
-            params.extend([page_size, offset])
-            limit_param = f"${param_index}"
-            offset_param = f"${param_index + 1}"
-
-            # Execute full-text search query with parameterized values
-            sql_query = f"""
-                SELECT
-                    slug,
-                    agent_name,
-                    agent_image,
-                    creator_username,
-                    creator_avatar,
-                    sub_heading,
-                    description,
-                    runs,
-                    rating,
-                    categories,
-                    featured,
-                    is_available,
-                    updated_at,
-                    ts_rank_cd(search, query) AS rank
-                FROM {{schema_prefix}}"StoreAgent",
-                    plainto_tsquery('english', $1) AS query
-                WHERE {sql_where_clause}
-                    AND search @@ query
-                ORDER BY {order_by_clause}
-                LIMIT {limit_param} OFFSET {offset_param}
-            """
-
-            # Count query for pagination - only uses search term parameter
-            count_query = f"""
-                SELECT COUNT(*) as count
-                FROM {{schema_prefix}}"StoreAgent",
-                    plainto_tsquery('english', $1) AS query
-                WHERE {sql_where_clause}
-                    AND search @@ query
-            """
-
-            # Execute both queries with parameters
-            agents = await query_raw_with_schema(sql_query, *params)
-
-            # For count, use params without pagination (last 2 params)
-            count_params = params[:-2]
-            count_result = await query_raw_with_schema(count_query, *count_params)
-
-            total = count_result[0]["count"] if count_result else 0
-            total_pages = (total + page_size - 1) // page_size
-
-            # Convert raw results to StoreAgent models
-            store_agents: list[store_model.StoreAgent] = []
-            for agent in agents:
-                try:
-                    store_agent = store_model.StoreAgent(
-                        slug=agent["slug"],
-                        agent_name=agent["agent_name"],
-                        agent_image=(
-                            agent["agent_image"][0] if agent["agent_image"] else ""
-                        ),
-                        creator=agent["creator_username"] or "Needs Profile",
-                        creator_avatar=agent["creator_avatar"] or "",
-                        sub_heading=agent["sub_heading"],
-                        description=agent["description"],
-                        runs=agent["runs"],
-                        rating=agent["rating"],
-                    )
-                    store_agents.append(store_agent)
-                except Exception as e:
-                    logger.error(f"Error parsing Store agent from search results: {e}")
-                    continue
-
-        else:
-            # Non-search query path (original logic)
+        if not search_used_hybrid:
+            # Fallback path - use basic search or no search
             where_clause: prisma.types.StoreAgentWhereInput = {"is_available": True}
             if featured:
                 where_clause["featured"] = featured
@@ -180,6 +130,14 @@ async def get_store_agents(
             if category:
                 where_clause["categories"] = {"has": category}
 
+            # Add basic text search if search_query provided but hybrid failed
+            if search_query:
+                where_clause["OR"] = [
+                    {"agent_name": {"contains": search_query, "mode": "insensitive"}},
+                    {"sub_heading": {"contains": search_query, "mode": "insensitive"}},
+                    {"description": {"contains": search_query, "mode": "insensitive"}},
+                ]
+
             order_by = []
             if sorted_by == "rating":
                 order_by.append({"rating": "desc"})
@@ -188,7 +146,7 @@ async def get_store_agents(
             elif sorted_by == "name":
                 order_by.append({"agent_name": "asc"})
 
-            agents = await prisma.models.StoreAgent.prisma().find_many(
+            db_agents = await prisma.models.StoreAgent.prisma().find_many(
                 where=where_clause,
                 order=order_by,
                 skip=(page - 1) * page_size,
@@ -199,7 +157,7 @@ async def get_store_agents(
             total_pages = (total + page_size - 1) // page_size
 
             store_agents: list[store_model.StoreAgent] = []
-            for agent in agents:
+            for agent in db_agents:
                 try:
                     # Create the StoreAgent object safely
                     store_agent = store_model.StoreAgent(
@@ -1577,7 +1535,7 @@ async def review_store_submission(
                 )
 
                 # Update the AgentGraph with store listing data
-                await prisma.models.AgentGraph.prisma().update(
+                await prisma.models.AgentGraph.prisma(tx).update(
                     where={
                         "graphVersionId": {
                             "id": store_listing_version.agentGraphId,
@@ -1592,6 +1550,23 @@ async def review_store_submission(
                     },
                 )
 
+                # Generate embedding for approved listing (blocking - admin operation)
+                # Inside transaction: if embedding fails, entire transaction rolls back
+                embedding_success = await ensure_embedding(
+                    version_id=store_listing_version_id,
+                    name=store_listing_version.name,
+                    description=store_listing_version.description,
+                    sub_heading=store_listing_version.subHeading,
+                    categories=store_listing_version.categories or [],
+                    tx=tx,
+                )
+                if not embedding_success:
+                    raise ValueError(
+                        f"Failed to generate embedding for listing {store_listing_version_id}. "
+                        "This is likely due to OpenAI API being unavailable. "
+                        "Please try again later or contact support if the issue persists."
+                    )
+
                 await prisma.models.StoreListing.prisma(tx).update(
                     where={"id": store_listing_version.StoreListing.id},
                     data={

autogpt_platform/backend/backend/api/features/store/embeddings.py

@@ -0,0 +1,566 @@
+"""
+Unified Content Embeddings Service
+
+Handles generation and storage of OpenAI embeddings for all content types
+(store listings, blocks, documentation, library agents) to enable semantic/hybrid search.
+"""
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+import prisma
+from prisma.enums import ContentType
+from tiktoken import encoding_for_model
+
+from backend.data.db import execute_raw_with_schema, query_raw_with_schema
+from backend.util.clients import get_openai_client
+from backend.util.json import dumps
+
+logger = logging.getLogger(__name__)
+
+
+# OpenAI embedding model configuration
+EMBEDDING_MODEL = "text-embedding-3-small"
+# OpenAI embedding token limit (8,191 with 1 token buffer for safety)
+EMBEDDING_MAX_TOKENS = 8191
+
+
+def build_searchable_text(
+    name: str,
+    description: str,
+    sub_heading: str,
+    categories: list[str],
+) -> str:
+    """
+    Build searchable text from listing version fields.
+
+    Combines relevant fields into a single string for embedding.
+    """
+    parts = []
+
+    # Name is important - include it
+    if name:
+        parts.append(name)
+
+    # Sub-heading provides context
+    if sub_heading:
+        parts.append(sub_heading)
+
+    # Description is the main content
+    if description:
+        parts.append(description)
+
+    # Categories help with semantic matching
+    if categories:
+        parts.append(" ".join(categories))
+
+    return " ".join(parts)
+
+
+async def generate_embedding(text: str) -> list[float] | None:
+    """
+    Generate embedding for text using OpenAI API.
+
+    Returns None if embedding generation fails.
+    Fail-fast: no retries to maintain consistency with approval flow.
+    """
+    try:
+        client = get_openai_client()
+        if not client:
+            logger.error("openai_internal_api_key not set, cannot generate embedding")
+            return None
+
+        # Truncate text to token limit using tiktoken
+        # Character-based truncation is insufficient because token ratios vary by content type
+        enc = encoding_for_model(EMBEDDING_MODEL)
+        tokens = enc.encode(text)
+        if len(tokens) > EMBEDDING_MAX_TOKENS:
+            tokens = tokens[:EMBEDDING_MAX_TOKENS]
+            truncated_text = enc.decode(tokens)
+            logger.info(
+                f"Truncated text from {len(enc.encode(text))} to {len(tokens)} tokens"
+            )
+        else:
+            truncated_text = text
+
+        start_time = time.time()
+        response = await client.embeddings.create(
+            model=EMBEDDING_MODEL,
+            input=truncated_text,
+        )
+        latency_ms = (time.time() - start_time) * 1000
+
+        embedding = response.data[0].embedding
+        logger.info(
+            f"Generated embedding: {len(embedding)} dims, "
+            f"{len(tokens)} tokens, {latency_ms:.0f}ms"
+        )
+        return embedding
+
+    except Exception as e:
+        logger.error(f"Failed to generate embedding: {e}")
+        return None
+
+
+async def store_embedding(
+    version_id: str,
+    embedding: list[float],
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Store embedding in the database.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    DEPRECATED: Use ensure_embedding() instead (includes searchable_text).
+    """
+    return await store_content_embedding(
+        content_type=ContentType.STORE_AGENT,
+        content_id=version_id,
+        embedding=embedding,
+        searchable_text="",  # Empty for backward compat; ensure_embedding() populates this
+        metadata=None,
+        user_id=None,  # Store agents are public
+        tx=tx,
+    )
+
+
+async def store_content_embedding(
+    content_type: ContentType,
+    content_id: str,
+    embedding: list[float],
+    searchable_text: str,
+    metadata: dict | None = None,
+    user_id: str | None = None,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Store embedding in the unified content embeddings table.
+
+    New function for unified content embedding storage.
+    Uses raw SQL since Prisma doesn't natively support pgvector.
+    """
+    try:
+        client = tx if tx else prisma.get_client()
+
+        # Convert embedding to PostgreSQL vector format
+        embedding_str = embedding_to_vector_string(embedding)
+        metadata_json = dumps(metadata or {})
+
+        # Upsert the embedding
+        # WHERE clause in DO UPDATE prevents PostgreSQL 15 bug with NULLS NOT DISTINCT
+        await execute_raw_with_schema(
+            """
+            INSERT INTO {schema_prefix}"UnifiedContentEmbedding" (
+                "id", "contentType", "contentId", "userId", "embedding", "searchableText", "metadata", "createdAt", "updatedAt"
+            )
+            VALUES (gen_random_uuid()::text, $1::{schema_prefix}"ContentType", $2, $3, $4::vector, $5, $6::jsonb, NOW(), NOW())
+            ON CONFLICT ("contentType", "contentId", "userId")
+            DO UPDATE SET
+                "embedding" = $4::vector,
+                "searchableText" = $5,
+                "metadata" = $6::jsonb,
+                "updatedAt" = NOW()
+            WHERE {schema_prefix}"UnifiedContentEmbedding"."contentType" = $1::{schema_prefix}"ContentType"
+                AND {schema_prefix}"UnifiedContentEmbedding"."contentId" = $2
+                AND ({schema_prefix}"UnifiedContentEmbedding"."userId" = $3 OR ($3 IS NULL AND {schema_prefix}"UnifiedContentEmbedding"."userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+            embedding_str,
+            searchable_text,
+            metadata_json,
+            client=client,
+        )
+
+        logger.info(f"Stored embedding for {content_type}:{content_id}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to store embedding for {content_type}:{content_id}: {e}")
+        return False
+
+
+async def get_embedding(version_id: str) -> dict[str, Any] | None:
+    """
+    Retrieve embedding record for a listing version.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    Returns dict with storeListingVersionId, embedding, timestamps or None if not found.
+    """
+    result = await get_content_embedding(
+        ContentType.STORE_AGENT, version_id, user_id=None
+    )
+    if result:
+        # Transform to old format for backward compatibility
+        return {
+            "storeListingVersionId": result["contentId"],
+            "embedding": result["embedding"],
+            "createdAt": result["createdAt"],
+            "updatedAt": result["updatedAt"],
+        }
+    return None
+
+
+async def get_content_embedding(
+    content_type: ContentType, content_id: str, user_id: str | None = None
+) -> dict[str, Any] | None:
+    """
+    Retrieve embedding record for any content type.
+
+    New function for unified content embedding retrieval.
+    Returns dict with contentType, contentId, embedding, timestamps or None if not found.
+    """
+    try:
+        result = await query_raw_with_schema(
+            """
+            SELECT
+                "contentType",
+                "contentId",
+                "userId",
+                "embedding"::text as "embedding",
+                "searchableText",
+                "metadata",
+                "createdAt",
+                "updatedAt"
+            FROM {schema_prefix}"UnifiedContentEmbedding"
+            WHERE "contentType" = $1::{schema_prefix}"ContentType" AND "contentId" = $2 AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+        )
+
+        if result and len(result) > 0:
+            return result[0]
+        return None
+
+    except Exception as e:
+        logger.error(f"Failed to get embedding for {content_type}:{content_id}: {e}")
+        return None
+
+
+async def ensure_embedding(
+    version_id: str,
+    name: str,
+    description: str,
+    sub_heading: str,
+    categories: list[str],
+    force: bool = False,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Ensure an embedding exists for the listing version.
+
+    Creates embedding if missing. Use force=True to regenerate.
+    Backward-compatible wrapper for store listings.
+
+    Args:
+        version_id: The StoreListingVersion ID
+        name: Agent name
+        description: Agent description
+        sub_heading: Agent sub-heading
+        categories: Agent categories
+        force: Force regeneration even if embedding exists
+        tx: Optional transaction client
+
+    Returns:
+        True if embedding exists/was created, False on failure
+    """
+    try:
+        # Check if embedding already exists
+        if not force:
+            existing = await get_embedding(version_id)
+            if existing and existing.get("embedding"):
+                logger.debug(f"Embedding for version {version_id} already exists")
+                return True
+
+        # Build searchable text for embedding
+        searchable_text = build_searchable_text(
+            name, description, sub_heading, categories
+        )
+
+        # Generate new embedding
+        embedding = await generate_embedding(searchable_text)
+        if embedding is None:
+            logger.warning(f"Could not generate embedding for version {version_id}")
+            return False
+
+        # Store the embedding with metadata using new function
+        metadata = {
+            "name": name,
+            "subHeading": sub_heading,
+            "categories": categories,
+        }
+        return await store_content_embedding(
+            content_type=ContentType.STORE_AGENT,
+            content_id=version_id,
+            embedding=embedding,
+            searchable_text=searchable_text,
+            metadata=metadata,
+            user_id=None,  # Store agents are public
+            tx=tx,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to ensure embedding for version {version_id}: {e}")
+        return False
+
+
+async def delete_embedding(version_id: str) -> bool:
+    """
+    Delete embedding for a listing version.
+
+    BACKWARD COMPATIBILITY: Maintained for existing store listing usage.
+    Note: This is usually handled automatically by CASCADE delete,
+    but provided for manual cleanup if needed.
+    """
+    return await delete_content_embedding(ContentType.STORE_AGENT, version_id)
+
+
+async def delete_content_embedding(
+    content_type: ContentType, content_id: str, user_id: str | None = None
+) -> bool:
+    """
+    Delete embedding for any content type.
+
+    New function for unified content embedding deletion.
+    Note: This is usually handled automatically by CASCADE delete,
+    but provided for manual cleanup if needed.
+
+    Args:
+        content_type: The type of content (STORE_AGENT, LIBRARY_AGENT, etc.)
+        content_id: The unique identifier for the content
+        user_id: Optional user ID. For public content (STORE_AGENT, BLOCK), pass None.
+                 For user-scoped content (LIBRARY_AGENT), pass the user's ID to avoid
+                 deleting embeddings belonging to other users.
+
+    Returns:
+        True if deletion succeeded, False otherwise
+    """
+    try:
+        client = prisma.get_client()
+
+        await execute_raw_with_schema(
+            """
+            DELETE FROM {schema_prefix}"UnifiedContentEmbedding"
+            WHERE "contentType" = $1::{schema_prefix}"ContentType"
+              AND "contentId" = $2
+              AND ("userId" = $3 OR ($3 IS NULL AND "userId" IS NULL))
+            """,
+            content_type,
+            content_id,
+            user_id,
+            client=client,
+        )
+
+        user_str = f" (user: {user_id})" if user_id else ""
+        logger.info(f"Deleted embedding for {content_type}:{content_id}{user_str}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to delete embedding for {content_type}:{content_id}: {e}")
+        return False
+
+
+async def get_embedding_stats() -> dict[str, Any]:
+    """
+    Get statistics about embedding coverage.
+
+    Returns counts of:
+    - Total approved listing versions
+    - Versions with embeddings
+    - Versions without embeddings
+    """
+    try:
+        # Count approved versions
+        approved_result = await query_raw_with_schema(
+            """
+            SELECT COUNT(*) as count
+            FROM {schema_prefix}"StoreListingVersion"
+            WHERE "submissionStatus" = 'APPROVED'
+            AND "isDeleted" = false
+            """
+        )
+        total_approved = approved_result[0]["count"] if approved_result else 0
+
+        # Count versions with embeddings
+        embedded_result = await query_raw_with_schema(
+            """
+            SELECT COUNT(*) as count
+            FROM {schema_prefix}"StoreListingVersion" slv
+            JOIN {schema_prefix}"UnifiedContentEmbedding" uce ON slv.id = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{schema_prefix}"ContentType"
+            WHERE slv."submissionStatus" = 'APPROVED'
+            AND slv."isDeleted" = false
+            """
+        )
+        with_embeddings = embedded_result[0]["count"] if embedded_result else 0
+
+        return {
+            "total_approved": total_approved,
+            "with_embeddings": with_embeddings,
+            "without_embeddings": total_approved - with_embeddings,
+            "coverage_percent": (
+                round(with_embeddings / total_approved * 100, 1)
+                if total_approved > 0
+                else 0
+            ),
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to get embedding stats: {e}")
+        return {
+            "total_approved": 0,
+            "with_embeddings": 0,
+            "without_embeddings": 0,
+            "coverage_percent": 0,
+            "error": str(e),
+        }
+
+
+async def backfill_missing_embeddings(batch_size: int = 10) -> dict[str, Any]:
+    """
+    Generate embeddings for approved listings that don't have them.
+
+    Args:
+        batch_size: Number of embeddings to generate in one call
+
+    Returns:
+        Dict with success/failure counts
+    """
+    try:
+        # Find approved versions without embeddings
+        missing = await query_raw_with_schema(
+            """
+            SELECT
+                slv.id,
+                slv.name,
+                slv.description,
+                slv."subHeading",
+                slv.categories
+            FROM {schema_prefix}"StoreListingVersion" slv
+            LEFT JOIN {schema_prefix}"UnifiedContentEmbedding" uce
+                ON slv.id = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{schema_prefix}"ContentType"
+            WHERE slv."submissionStatus" = 'APPROVED'
+            AND slv."isDeleted" = false
+            AND uce."contentId" IS NULL
+            LIMIT $1
+            """,
+            batch_size,
+        )
+
+        if not missing:
+            return {
+                "processed": 0,
+                "success": 0,
+                "failed": 0,
+                "message": "No missing embeddings",
+            }
+
+        # Process embeddings concurrently for better performance
+        embedding_tasks = [
+            ensure_embedding(
+                version_id=row["id"],
+                name=row["name"],
+                description=row["description"],
+                sub_heading=row["subHeading"],
+                categories=row["categories"] or [],
+            )
+            for row in missing
+        ]
+
+        results = await asyncio.gather(*embedding_tasks, return_exceptions=True)
+
+        success = sum(1 for result in results if result is True)
+        failed = len(results) - success
+
+        return {
+            "processed": len(missing),
+            "success": success,
+            "failed": failed,
+            "message": f"Backfilled {success} embeddings, {failed} failed",
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to backfill embeddings: {e}")
+        return {
+            "processed": 0,
+            "success": 0,
+            "failed": 0,
+            "error": str(e),
+        }
+
+
+async def embed_query(query: str) -> list[float] | None:
+    """
+    Generate embedding for a search query.
+
+    Same as generate_embedding but with clearer intent.
+    """
+    return await generate_embedding(query)
+
+
+def embedding_to_vector_string(embedding: list[float]) -> str:
+    """Convert embedding list to PostgreSQL vector string format."""
+    return "[" + ",".join(str(x) for x in embedding) + "]"
+
+
+async def ensure_content_embedding(
+    content_type: ContentType,
+    content_id: str,
+    searchable_text: str,
+    metadata: dict | None = None,
+    user_id: str | None = None,
+    force: bool = False,
+    tx: prisma.Prisma | None = None,
+) -> bool:
+    """
+    Ensure an embedding exists for any content type.
+
+    Generic function for creating embeddings for store agents, blocks, docs, etc.
+
+    Args:
+        content_type: ContentType enum value (STORE_AGENT, BLOCK, etc.)
+        content_id: Unique identifier for the content
+        searchable_text: Combined text for embedding generation
+        metadata: Optional metadata to store with embedding
+        force: Force regeneration even if embedding exists
+        tx: Optional transaction client
+
+    Returns:
+        True if embedding exists/was created, False on failure
+    """
+    try:
+        # Check if embedding already exists
+        if not force:
+            existing = await get_content_embedding(content_type, content_id, user_id)
+            if existing and existing.get("embedding"):
+                logger.debug(
+                    f"Embedding for {content_type}:{content_id} already exists"
+                )
+                return True
+
+        # Generate new embedding
+        embedding = await generate_embedding(searchable_text)
+        if embedding is None:
+            logger.warning(
+                f"Could not generate embedding for {content_type}:{content_id}"
+            )
+            return False
+
+        # Store the embedding
+        return await store_content_embedding(
+            content_type=content_type,
+            content_id=content_id,
+            embedding=embedding,
+            searchable_text=searchable_text,
+            metadata=metadata or {},
+            user_id=user_id,
+            tx=tx,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to ensure embedding for {content_type}:{content_id}: {e}")
+        return False

autogpt_platform/backend/backend/api/features/store/embeddings_schema_test.py

@@ -0,0 +1,329 @@
+"""
+Integration tests for embeddings with schema handling.
+
+These tests verify that embeddings operations work correctly across different database schemas.
+"""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from prisma.enums import ContentType
+
+from backend.api.features.store import embeddings
+
+# Schema prefix tests removed - functionality moved to db.raw_with_schema() helper
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_store_content_embedding_with_schema():
+    """Test storing embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.store_content_embedding(
+                content_type=ContentType.STORE_AGENT,
+                content_id="test-id",
+                embedding=[0.1] * 1536,
+                searchable_text="test text",
+                metadata={"test": "data"},
+                user_id=None,
+            )
+
+            # Verify the query was called
+            assert mock_client.execute_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.execute_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_get_content_embedding_with_schema():
+    """Test retrieving embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_client.query_raw.return_value = [
+                {
+                    "contentType": "STORE_AGENT",
+                    "contentId": "test-id",
+                    "userId": None,
+                    "embedding": "[0.1, 0.2]",
+                    "searchableText": "test",
+                    "metadata": {},
+                    "createdAt": "2024-01-01",
+                    "updatedAt": "2024-01-01",
+                }
+            ]
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.get_content_embedding(
+                ContentType.STORE_AGENT,
+                "test-id",
+                user_id=None,
+            )
+
+            # Verify the query was called
+            assert mock_client.query_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.query_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is not None
+            assert result["contentId"] == "test-id"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_delete_content_embedding_with_schema():
+    """Test deleting embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.delete_content_embedding(
+                ContentType.STORE_AGENT,
+                "test-id",
+            )
+
+            # Verify the query was called
+            assert mock_client.execute_raw.called
+
+            # Get the SQL query that was executed
+            call_args = mock_client.execute_raw.call_args
+            sql_query = call_args[0][0]
+
+            # Verify schema prefix is in the query
+            assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+            # Verify result
+            assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_get_embedding_stats_with_schema():
+    """Test embedding statistics with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            # Mock both query results
+            mock_client.query_raw.side_effect = [
+                [{"count": 100}],  # total_approved
+                [{"count": 80}],  # with_embeddings
+            ]
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.get_embedding_stats()
+
+            # Verify both queries were called
+            assert mock_client.query_raw.call_count == 2
+
+            # Get both SQL queries
+            first_call = mock_client.query_raw.call_args_list[0]
+            second_call = mock_client.query_raw.call_args_list[1]
+
+            first_sql = first_call[0][0]
+            second_sql = second_call[0][0]
+
+            # Verify schema prefix in both queries
+            assert '"platform"."StoreListingVersion"' in first_sql
+            assert '"platform"."StoreListingVersion"' in second_sql
+            assert '"platform"."UnifiedContentEmbedding"' in second_sql
+
+            # Verify results
+            assert result["total_approved"] == 100
+            assert result["with_embeddings"] == 80
+            assert result["without_embeddings"] == 20
+            assert result["coverage_percent"] == 80.0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backfill_missing_embeddings_with_schema():
+    """Test backfilling embeddings with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            # Mock missing embeddings query
+            mock_client.query_raw.return_value = [
+                {
+                    "id": "version-1",
+                    "name": "Test Agent",
+                    "description": "Test description",
+                    "subHeading": "Test heading",
+                    "categories": ["test"],
+                }
+            ]
+            mock_get_client.return_value = mock_client
+
+            with patch(
+                "backend.api.features.store.embeddings.ensure_embedding"
+            ) as mock_ensure:
+                mock_ensure.return_value = True
+
+                result = await embeddings.backfill_missing_embeddings(batch_size=10)
+
+                # Verify the query was called
+                assert mock_client.query_raw.called
+
+                # Get the SQL query
+                call_args = mock_client.query_raw.call_args
+                sql_query = call_args[0][0]
+
+                # Verify schema prefix in query
+                assert '"platform"."StoreListingVersion"' in sql_query
+                assert '"platform"."UnifiedContentEmbedding"' in sql_query
+
+                # Verify ensure_embedding was called
+                assert mock_ensure.called
+
+                # Verify results
+                assert result["processed"] == 1
+                assert result["success"] == 1
+                assert result["failed"] == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_ensure_content_embedding_with_schema():
+    """Test ensuring embeddings exist with proper schema handling."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch(
+            "backend.api.features.store.embeddings.get_content_embedding"
+        ) as mock_get:
+            # Simulate no existing embedding
+            mock_get.return_value = None
+
+            with patch(
+                "backend.api.features.store.embeddings.generate_embedding"
+            ) as mock_generate:
+                mock_generate.return_value = [0.1] * 1536
+
+                with patch(
+                    "backend.api.features.store.embeddings.store_content_embedding"
+                ) as mock_store:
+                    mock_store.return_value = True
+
+                    result = await embeddings.ensure_content_embedding(
+                        content_type=ContentType.STORE_AGENT,
+                        content_id="test-id",
+                        searchable_text="test text",
+                        metadata={"test": "data"},
+                        user_id=None,
+                        force=False,
+                    )
+
+                    # Verify the flow
+                    assert mock_get.called
+                    assert mock_generate.called
+                    assert mock_store.called
+                    assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backward_compatibility_store_embedding():
+    """Test backward compatibility wrapper for store_embedding."""
+    with patch(
+        "backend.api.features.store.embeddings.store_content_embedding"
+    ) as mock_store:
+        mock_store.return_value = True
+
+        result = await embeddings.store_embedding(
+            version_id="test-version-id",
+            embedding=[0.1] * 1536,
+            tx=None,
+        )
+
+        # Verify it calls the new function with correct parameters
+        assert mock_store.called
+        call_args = mock_store.call_args
+
+        assert call_args[1]["content_type"] == ContentType.STORE_AGENT
+        assert call_args[1]["content_id"] == "test-version-id"
+        assert call_args[1]["user_id"] is None
+        assert result is True
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_backward_compatibility_get_embedding():
+    """Test backward compatibility wrapper for get_embedding."""
+    with patch(
+        "backend.api.features.store.embeddings.get_content_embedding"
+    ) as mock_get:
+        mock_get.return_value = {
+            "contentType": "STORE_AGENT",
+            "contentId": "test-version-id",
+            "embedding": "[0.1, 0.2]",
+            "createdAt": "2024-01-01",
+            "updatedAt": "2024-01-01",
+        }
+
+        result = await embeddings.get_embedding("test-version-id")
+
+        # Verify it calls the new function
+        assert mock_get.called
+
+        # Verify it transforms to old format
+        assert result is not None
+        assert result["storeListingVersionId"] == "test-version-id"
+        assert "embedding" in result
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_schema_handling_error_cases():
+    """Test error handling in schema-aware operations."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch("prisma.get_client") as mock_get_client:
+            mock_client = AsyncMock()
+            mock_client.execute_raw.side_effect = Exception("Database error")
+            mock_get_client.return_value = mock_client
+
+            result = await embeddings.store_content_embedding(
+                content_type=ContentType.STORE_AGENT,
+                content_id="test-id",
+                embedding=[0.1] * 1536,
+                searchable_text="test",
+                metadata=None,
+                user_id=None,
+            )
+
+            # Should return False on error, not raise
+            assert result is False
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "-s"])

autogpt_platform/backend/backend/api/features/store/embeddings_test.py

@@ -0,0 +1,380 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import prisma
+import pytest
+from prisma import Prisma
+from prisma.enums import ContentType
+
+from backend.api.features.store import embeddings
+
+
+@pytest.fixture(autouse=True)
+async def setup_prisma():
+    """Setup Prisma client for tests."""
+    try:
+        Prisma()
+    except prisma.errors.ClientAlreadyRegisteredError:
+        pass
+    yield
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_build_searchable_text():
+    """Test searchable text building from listing fields."""
+    result = embeddings.build_searchable_text(
+        name="AI Assistant",
+        description="A helpful AI assistant for productivity",
+        sub_heading="Boost your productivity",
+        categories=["AI", "Productivity"],
+    )
+
+    expected = "AI Assistant Boost your productivity A helpful AI assistant for productivity AI Productivity"
+    assert result == expected
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_build_searchable_text_empty_fields():
+    """Test searchable text building with empty fields."""
+    result = embeddings.build_searchable_text(
+        name="", description="Test description", sub_heading="", categories=[]
+    )
+
+    assert result == "Test description"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_success():
+    """Test successful embedding generation."""
+    # Mock OpenAI response
+    mock_client = MagicMock()
+    mock_response = MagicMock()
+    mock_response.data = [MagicMock()]
+    mock_response.data[0].embedding = [0.1, 0.2, 0.3] * 512  # 1536 dimensions
+
+    # Use AsyncMock for async embeddings.create method
+    mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is not None
+        assert len(result) == 1536
+        assert result[0] == 0.1
+
+        mock_client.embeddings.create.assert_called_once_with(
+            model="text-embedding-3-small", input="test text"
+        )
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_no_api_key():
+    """Test embedding generation without API key."""
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = None
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_api_error():
+    """Test embedding generation with API error."""
+    mock_client = MagicMock()
+    mock_client.embeddings.create = AsyncMock(side_effect=Exception("API Error"))
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        result = await embeddings.generate_embedding("test text")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_generate_embedding_text_truncation():
+    """Test that long text is properly truncated using tiktoken."""
+    from tiktoken import encoding_for_model
+
+    mock_client = MagicMock()
+    mock_response = MagicMock()
+    mock_response.data = [MagicMock()]
+    mock_response.data[0].embedding = [0.1] * 1536
+
+    # Use AsyncMock for async embeddings.create method
+    mock_client.embeddings.create = AsyncMock(return_value=mock_response)
+
+    # Patch at the point of use in embeddings.py
+    with patch(
+        "backend.api.features.store.embeddings.get_openai_client"
+    ) as mock_get_client:
+        mock_get_client.return_value = mock_client
+
+        # Create text that will exceed 8191 tokens
+        # Use varied characters to ensure token-heavy text: each word is ~1 token
+        words = [f"word{i}" for i in range(10000)]
+        long_text = " ".join(words)  # ~10000 tokens
+
+        await embeddings.generate_embedding(long_text)
+
+        # Verify text was truncated to 8191 tokens
+        call_args = mock_client.embeddings.create.call_args
+        truncated_text = call_args.kwargs["input"]
+
+        # Count actual tokens in truncated text
+        enc = encoding_for_model("text-embedding-3-small")
+        actual_tokens = len(enc.encode(truncated_text))
+
+        # Should be at or just under 8191 tokens
+        assert actual_tokens <= 8191
+        # Should be close to the limit (not over-truncated)
+        assert actual_tokens >= 8100
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_embedding_success(mocker):
+    """Test successful embedding storage."""
+    mock_client = mocker.AsyncMock()
+    mock_client.execute_raw = mocker.AsyncMock()
+
+    embedding = [0.1, 0.2, 0.3]
+
+    result = await embeddings.store_embedding(
+        version_id="test-version-id", embedding=embedding, tx=mock_client
+    )
+
+    assert result is True
+    mock_client.execute_raw.assert_called_once()
+    call_args = mock_client.execute_raw.call_args[0]
+    assert "test-version-id" in call_args
+    assert "[0.1,0.2,0.3]" in call_args
+    assert None in call_args  # userId should be None for store agents
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_store_embedding_database_error(mocker):
+    """Test embedding storage with database error."""
+    mock_client = mocker.AsyncMock()
+    mock_client.execute_raw.side_effect = Exception("Database error")
+
+    embedding = [0.1, 0.2, 0.3]
+
+    result = await embeddings.store_embedding(
+        version_id="test-version-id", embedding=embedding, tx=mock_client
+    )
+
+    assert result is False
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_success(mocker):
+    """Test successful embedding retrieval."""
+    mock_client = mocker.AsyncMock()
+    mock_result = [
+        {
+            "contentType": "STORE_AGENT",
+            "contentId": "test-version-id",
+            "embedding": "[0.1,0.2,0.3]",
+            "searchableText": "Test text",
+            "metadata": {},
+            "createdAt": "2024-01-01T00:00:00Z",
+            "updatedAt": "2024-01-01T00:00:00Z",
+        }
+    ]
+    mock_client.query_raw.return_value = mock_result
+
+    with patch("prisma.get_client", return_value=mock_client):
+        result = await embeddings.get_embedding("test-version-id")
+
+        assert result is not None
+        assert result["storeListingVersionId"] == "test-version-id"
+        assert result["embedding"] == "[0.1,0.2,0.3]"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_not_found(mocker):
+    """Test embedding retrieval when not found."""
+    mock_client = mocker.AsyncMock()
+    mock_client.query_raw.return_value = []
+
+    with patch("prisma.get_client", return_value=mock_client):
+        result = await embeddings.get_embedding("test-version-id")
+
+        assert result is None
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.store_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_already_exists(mock_get, mock_store, mock_generate):
+    """Test ensure_embedding when embedding already exists."""
+    mock_get.return_value = {"embedding": "[0.1,0.2,0.3]"}
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is True
+    mock_generate.assert_not_called()
+    mock_store.assert_not_called()
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.store_content_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_create_new(mock_get, mock_store, mock_generate):
+    """Test ensure_embedding creating new embedding."""
+    mock_get.return_value = None
+    mock_generate.return_value = [0.1, 0.2, 0.3]
+    mock_store.return_value = True
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is True
+    mock_generate.assert_called_once_with("Test Test heading Test description test")
+    mock_store.assert_called_once_with(
+        content_type=ContentType.STORE_AGENT,
+        content_id="test-id",
+        embedding=[0.1, 0.2, 0.3],
+        searchable_text="Test Test heading Test description test",
+        metadata={"name": "Test", "subHeading": "Test heading", "categories": ["test"]},
+        user_id=None,
+        tx=None,
+    )
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.generate_embedding")
+@patch("backend.api.features.store.embeddings.get_embedding")
+async def test_ensure_embedding_generation_fails(mock_get, mock_generate):
+    """Test ensure_embedding when generation fails."""
+    mock_get.return_value = None
+    mock_generate.return_value = None
+
+    result = await embeddings.ensure_embedding(
+        version_id="test-id",
+        name="Test",
+        description="Test description",
+        sub_heading="Test heading",
+        categories=["test"],
+    )
+
+    assert result is False
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_get_embedding_stats(mocker):
+    """Test embedding statistics retrieval."""
+    mock_client = mocker.AsyncMock()
+
+    # Mock approved count query
+    mock_approved_result = [{"count": 100}]
+    # Mock embedded count query
+    mock_embedded_result = [{"count": 75}]
+
+    mock_client.query_raw.side_effect = [mock_approved_result, mock_embedded_result]
+
+    with patch("prisma.get_client", return_value=mock_client):
+        result = await embeddings.get_embedding_stats()
+
+        assert result["total_approved"] == 100
+        assert result["with_embeddings"] == 75
+        assert result["without_embeddings"] == 25
+        assert result["coverage_percent"] == 75.0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@patch("backend.api.features.store.embeddings.ensure_embedding")
+async def test_backfill_missing_embeddings_success(mock_ensure, mocker):
+    """Test backfill with successful embedding generation."""
+    mock_client = mocker.AsyncMock()
+
+    # Mock missing embeddings query
+    mock_missing = [
+        {
+            "id": "version-1",
+            "name": "Agent 1",
+            "description": "Description 1",
+            "subHeading": "Heading 1",
+            "categories": ["AI"],
+        },
+        {
+            "id": "version-2",
+            "name": "Agent 2",
+            "description": "Description 2",
+            "subHeading": "Heading 2",
+            "categories": ["Productivity"],
+        },
+    ]
+    mock_client.query_raw.return_value = mock_missing
+
+    # Mock ensure_embedding to succeed for first, fail for second
+    mock_ensure.side_effect = [True, False]
+
+    with patch("prisma.get_client", return_value=mock_client):
+        result = await embeddings.backfill_missing_embeddings(batch_size=5)
+
+        assert result["processed"] == 2
+        assert result["success"] == 1
+        assert result["failed"] == 1
+        assert mock_ensure.call_count == 2
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_backfill_missing_embeddings_no_missing(mocker):
+    """Test backfill when no embeddings are missing."""
+    mock_client = mocker.AsyncMock()
+    mock_client.query_raw.return_value = []
+
+    with patch("prisma.get_client", return_value=mock_client):
+        result = await embeddings.backfill_missing_embeddings(batch_size=5)
+
+        assert result["processed"] == 0
+        assert result["success"] == 0
+        assert result["failed"] == 0
+        assert result["message"] == "No missing embeddings"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_embedding_to_vector_string():
+    """Test embedding to PostgreSQL vector string conversion."""
+    embedding = [0.1, 0.2, 0.3, -0.4]
+    result = embeddings.embedding_to_vector_string(embedding)
+    assert result == "[0.1,0.2,0.3,-0.4]"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+async def test_embed_query():
+    """Test embed_query function (alias for generate_embedding)."""
+    with patch(
+        "backend.api.features.store.embeddings.generate_embedding"
+    ) as mock_generate:
+        mock_generate.return_value = [0.1, 0.2, 0.3]
+
+        result = await embeddings.embed_query("test query")
+
+        assert result == [0.1, 0.2, 0.3]
+        mock_generate.assert_called_once_with("test query")

autogpt_platform/backend/backend/api/features/store/hybrid_search.py

@@ -0,0 +1,391 @@
+"""
+Hybrid Search for Store Agents
+
+Combines semantic (embedding) search with lexical (tsvector) search
+for improved relevance in marketplace agent discovery.
+"""
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+from backend.api.features.store.embeddings import (
+    embed_query,
+    embedding_to_vector_string,
+)
+from backend.data.db import query_raw_with_schema
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class HybridSearchWeights:
+    """Weights for combining search signals."""
+
+    semantic: float = 0.30  # Embedding cosine similarity
+    lexical: float = 0.30  # tsvector ts_rank_cd score
+    category: float = 0.20  # Category match boost
+    recency: float = 0.10  # Newer agents ranked higher
+    popularity: float = 0.10  # Agent usage/runs (PageRank-like)
+
+    def __post_init__(self):
+        """Validate weights are non-negative and sum to approximately 1.0."""
+        total = (
+            self.semantic
+            + self.lexical
+            + self.category
+            + self.recency
+            + self.popularity
+        )
+
+        if any(
+            w < 0
+            for w in [
+                self.semantic,
+                self.lexical,
+                self.category,
+                self.recency,
+                self.popularity,
+            ]
+        ):
+            raise ValueError("All weights must be non-negative")
+
+        if not (0.99 <= total <= 1.01):
+            raise ValueError(f"Weights must sum to ~1.0, got {total:.3f}")
+
+
+DEFAULT_WEIGHTS = HybridSearchWeights()
+
+# Minimum relevance score threshold - agents below this are filtered out
+# With weights (0.30 semantic + 0.30 lexical + 0.20 category + 0.10 recency + 0.10 popularity):
+# - 0.20 means at least ~60% semantic match OR strong lexical match required
+# - Ensures only genuinely relevant results are returned
+# - Recency/popularity alone (0.10 each) won't pass the threshold
+DEFAULT_MIN_SCORE = 0.20
+
+
+@dataclass
+class HybridSearchResult:
+    """A single search result with score breakdown."""
+
+    slug: str
+    agent_name: str
+    agent_image: str
+    creator_username: str
+    creator_avatar: str
+    sub_heading: str
+    description: str
+    runs: int
+    rating: float
+    categories: list[str]
+    featured: bool
+    is_available: bool
+    updated_at: datetime
+
+    # Score breakdown (for debugging/tuning)
+    combined_score: float
+    semantic_score: float = 0.0
+    lexical_score: float = 0.0
+    category_score: float = 0.0
+    recency_score: float = 0.0
+    popularity_score: float = 0.0
+
+
+async def hybrid_search(
+    query: str,
+    featured: bool = False,
+    creators: list[str] | None = None,
+    category: str | None = None,
+    sorted_by: (
+        Literal["relevance", "rating", "runs", "name", "updated_at"] | None
+    ) = None,
+    page: int = 1,
+    page_size: int = 20,
+    weights: HybridSearchWeights | None = None,
+    min_score: float | None = None,
+) -> tuple[list[dict[str, Any]], int]:
+    """
+    Perform hybrid search combining semantic and lexical signals.
+
+    Args:
+        query: Search query string
+        featured: Filter for featured agents only
+        creators: Filter by creator usernames
+        category: Filter by category
+        sorted_by: Sort order (relevance uses hybrid scoring)
+        page: Page number (1-indexed)
+        page_size: Results per page
+        weights: Custom weights for search signals
+        min_score: Minimum relevance score threshold (0-1). Results below
+                   this score are filtered out. Defaults to DEFAULT_MIN_SCORE.
+
+    Returns:
+        Tuple of (results list, total count). Returns empty list if no
+        results meet the minimum relevance threshold.
+    """
+    # Validate inputs
+    query = query.strip()
+    if not query:
+        return [], 0  # Empty query returns no results
+
+    if page < 1:
+        page = 1
+    if page_size < 1:
+        page_size = 1
+    if page_size > 100:  # Cap at reasonable limit to prevent performance issues
+        page_size = 100
+
+    if weights is None:
+        weights = DEFAULT_WEIGHTS
+    if min_score is None:
+        min_score = DEFAULT_MIN_SCORE
+
+    offset = (page - 1) * page_size
+
+    # Generate query embedding
+    query_embedding = await embed_query(query)
+
+    # Build WHERE clause conditions
+    where_parts: list[str] = ["sa.is_available = true"]
+    params: list[Any] = []
+    param_index = 1
+
+    # Add search query for lexical matching
+    params.append(query)
+    query_param = f"${param_index}"
+    param_index += 1
+
+    # Add lowercased query for category matching
+    params.append(query.lower())
+    query_lower_param = f"${param_index}"
+    param_index += 1
+
+    if featured:
+        where_parts.append("sa.featured = true")
+
+    if creators:
+        where_parts.append(f"sa.creator_username = ANY(${param_index})")
+        params.append(creators)
+        param_index += 1
+
+    if category:
+        where_parts.append(f"${param_index} = ANY(sa.categories)")
+        params.append(category)
+        param_index += 1
+
+    # Safe: where_parts only contains hardcoded strings with $N parameter placeholders
+    # No user input is concatenated directly into the SQL string
+    where_clause = " AND ".join(where_parts)
+
+    # Embedding is required for hybrid search - fail fast if unavailable
+    if query_embedding is None or not query_embedding:
+        # Log detailed error server-side
+        logger.error(
+            "Failed to generate query embedding. "
+            "Check that openai_internal_api_key is configured and OpenAI API is accessible."
+        )
+        # Raise generic error to client
+        raise ValueError("Search service temporarily unavailable")
+
+    # Add embedding parameter
+    embedding_str = embedding_to_vector_string(query_embedding)
+    params.append(embedding_str)
+    embedding_param = f"${param_index}"
+    param_index += 1
+
+    # Add weight parameters for SQL calculation
+    params.append(weights.semantic)
+    weight_semantic_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.lexical)
+    weight_lexical_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.category)
+    weight_category_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.recency)
+    weight_recency_param = f"${param_index}"
+    param_index += 1
+
+    params.append(weights.popularity)
+    weight_popularity_param = f"${param_index}"
+    param_index += 1
+
+    # Add min_score parameter
+    params.append(min_score)
+    min_score_param = f"${param_index}"
+    param_index += 1
+
+    # Optimized hybrid search query:
+    # 1. Direct join to UnifiedContentEmbedding via contentId=storeListingVersionId (no redundant JOINs)
+    # 2. UNION approach (deduplicates agents matching both branches)
+    # 3. COUNT(*) OVER() to get total count in single query
+    # 4. Optimized category matching with EXISTS + unnest
+    # 5. Pre-calculated max values for lexical and popularity normalization
+    # 6. Simplified recency calculation with linear decay
+    # 7. Logarithmic popularity scaling to prevent viral agents from dominating
+    sql_query = f"""
+            WITH candidates AS (
+                -- Lexical matches (uses GIN index on search column)
+                SELECT sa."storeListingVersionId"
+                FROM {{schema_prefix}}"StoreAgent" sa
+                WHERE {where_clause}
+                AND sa.search @@ plainto_tsquery('english', {query_param})
+
+                UNION
+
+                -- Semantic matches (uses HNSW index on embedding with KNN)
+                SELECT "storeListingVersionId"
+                FROM (
+                    SELECT sa."storeListingVersionId", uce.embedding
+                    FROM {{schema_prefix}}"StoreAgent" sa
+                    INNER JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce
+                        ON sa."storeListingVersionId" = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{{schema_prefix}}"ContentType"
+                    WHERE {where_clause}
+                    ORDER BY uce.embedding <=> {embedding_param}::vector
+                    LIMIT 200
+                ) semantic_results
+            ),
+            search_scores AS (
+                SELECT
+                    sa.slug,
+                    sa.agent_name,
+                    sa.agent_image,
+                    sa.creator_username,
+                    sa.creator_avatar,
+                    sa.sub_heading,
+                    sa.description,
+                    sa.runs,
+                    sa.rating,
+                    sa.categories,
+                    sa.featured,
+                    sa.is_available,
+                    sa.updated_at,
+                    -- Semantic score: cosine similarity (1 - distance)
+                    COALESCE(1 - (uce.embedding <=> {embedding_param}::vector), 0) as semantic_score,
+                    -- Lexical score: ts_rank_cd (will be normalized later)
+                    COALESCE(ts_rank_cd(sa.search, plainto_tsquery('english', {query_param})), 0) as lexical_raw,
+                    -- Category match: optimized with unnest for better performance
+                    CASE
+                        WHEN EXISTS (
+                            SELECT 1 FROM unnest(sa.categories) cat
+                            WHERE LOWER(cat) LIKE '%' || {query_lower_param} || '%'
+                        )
+                        THEN 1.0
+                        ELSE 0.0
+                    END as category_score,
+                    -- Recency score: linear decay over 90 days (simpler than exponential)
+                    GREATEST(0, 1 - EXTRACT(EPOCH FROM (NOW() - sa.updated_at)) / (90 * 24 * 3600)) as recency_score,
+                    -- Popularity raw: agent runs count (will be normalized with log scaling)
+                    sa.runs as popularity_raw
+                FROM candidates c
+                INNER JOIN {{schema_prefix}}"StoreAgent" sa
+                    ON c."storeListingVersionId" = sa."storeListingVersionId"
+                LEFT JOIN {{schema_prefix}}"UnifiedContentEmbedding" uce
+                    ON sa."storeListingVersionId" = uce."contentId" AND uce."contentType" = 'STORE_AGENT'::{{schema_prefix}}"ContentType"
+            ),
+            max_lexical AS (
+                SELECT MAX(lexical_raw) as max_val FROM search_scores
+            ),
+            max_popularity AS (
+                SELECT MAX(popularity_raw) as max_val FROM search_scores
+            ),
+            normalized AS (
+                SELECT
+                    ss.*,
+                    -- Normalize lexical score by pre-calculated max
+                    CASE
+                        WHEN ml.max_val > 0
+                        THEN ss.lexical_raw / ml.max_val
+                        ELSE 0
+                    END as lexical_score,
+                    -- Normalize popularity with logarithmic scaling to prevent viral agents from dominating
+                    -- LOG(1 + runs) / LOG(1 + max_runs) ensures score is 0-1 range
+                    CASE
+                        WHEN mp.max_val > 0 AND ss.popularity_raw > 0
+                        THEN LN(1 + ss.popularity_raw) / LN(1 + mp.max_val)
+                        ELSE 0
+                    END as popularity_score
+                FROM search_scores ss
+                CROSS JOIN max_lexical ml
+                CROSS JOIN max_popularity mp
+            ),
+            scored AS (
+                SELECT
+                    slug,
+                    agent_name,
+                    agent_image,
+                    creator_username,
+                    creator_avatar,
+                    sub_heading,
+                    description,
+                    runs,
+                    rating,
+                    categories,
+                    featured,
+                    is_available,
+                    updated_at,
+                    semantic_score,
+                    lexical_score,
+                    category_score,
+                    recency_score,
+                    popularity_score,
+                    (
+                        {weight_semantic_param} * semantic_score +
+                        {weight_lexical_param} * lexical_score +
+                        {weight_category_param} * category_score +
+                        {weight_recency_param} * recency_score +
+                        {weight_popularity_param} * popularity_score
+                    ) as combined_score
+                FROM normalized
+            ),
+            filtered AS (
+                SELECT
+                    *,
+                    COUNT(*) OVER () as total_count
+                FROM scored
+                WHERE combined_score >= {min_score_param}
+            )
+            SELECT * FROM filtered
+            ORDER BY combined_score DESC
+            LIMIT ${param_index} OFFSET ${param_index + 1}
+    """
+
+    # Add pagination params
+    params.extend([page_size, offset])
+
+    # Execute search query - includes total_count via window function
+    results = await query_raw_with_schema(sql_query, *params)
+
+    # Extract total count from first result (all rows have same count)
+    total = results[0]["total_count"] if results else 0
+
+    # Remove total_count from results before returning
+    for result in results:
+        result.pop("total_count", None)
+
+    # Log without sensitive query content
+    logger.info(f"Hybrid search: {len(results)} results, {total} total")
+
+    return results, total
+
+
+async def hybrid_search_simple(
+    query: str,
+    page: int = 1,
+    page_size: int = 20,
+) -> tuple[list[dict[str, Any]], int]:
+    """
+    Simplified hybrid search for common use cases.
+
+    Uses default weights and no filters.
+    """
+    return await hybrid_search(
+        query=query,
+        page=page,
+        page_size=page_size,
+    )

autogpt_platform/backend/backend/api/features/store/hybrid_search_test.py

@@ -0,0 +1,334 @@
+"""
+Integration tests for hybrid search with schema handling.
+
+These tests verify that hybrid search works correctly across different database schemas.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from backend.api.features.store.hybrid_search import HybridSearchWeights, hybrid_search
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_schema_handling():
+    """Test that hybrid search correctly handles database schema prefixes."""
+    # Test with a mock query to ensure schema handling works
+    query = "test agent"
+
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Mock the query result
+        mock_query.return_value = [
+            {
+                "slug": "test/agent",
+                "agent_name": "Test Agent",
+                "agent_image": "test.png",
+                "creator_username": "test",
+                "creator_avatar": "avatar.png",
+                "sub_heading": "Test sub-heading",
+                "description": "Test description",
+                "runs": 10,
+                "rating": 4.5,
+                "categories": ["test"],
+                "featured": False,
+                "is_available": True,
+                "updated_at": "2024-01-01T00:00:00Z",
+                "combined_score": 0.8,
+                "semantic_score": 0.7,
+                "lexical_score": 0.6,
+                "category_score": 0.5,
+                "recency_score": 0.4,
+                "total_count": 1,
+            }
+        ]
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536  # Mock embedding
+
+            results, total = await hybrid_search(
+                query=query,
+                page=1,
+                page_size=20,
+            )
+
+            # Verify the query was called
+            assert mock_query.called
+            # Verify the SQL template uses schema_prefix placeholder
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            assert "{schema_prefix}" in sql_template
+
+            # Verify results
+            assert len(results) == 1
+            assert total == 1
+            assert results[0]["slug"] == "test/agent"
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_public_schema():
+    """Test hybrid search when using public schema (no prefix needed)."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "public"
+
+        with patch(
+            "backend.api.features.store.hybrid_search.query_raw_with_schema"
+        ) as mock_query:
+            mock_query.return_value = []
+
+            with patch(
+                "backend.api.features.store.hybrid_search.embed_query"
+            ) as mock_embed:
+                mock_embed.return_value = [0.1] * 1536
+
+                results, total = await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+                # Verify the mock was set up correctly
+                assert mock_schema.return_value == "public"
+
+                # Results should work even with empty results
+                assert results == []
+                assert total == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_custom_schema():
+    """Test hybrid search when using custom schema (e.g., 'platform')."""
+    with patch("backend.data.db.get_database_schema") as mock_schema:
+        mock_schema.return_value = "platform"
+
+        with patch(
+            "backend.api.features.store.hybrid_search.query_raw_with_schema"
+        ) as mock_query:
+            mock_query.return_value = []
+
+            with patch(
+                "backend.api.features.store.hybrid_search.embed_query"
+            ) as mock_embed:
+                mock_embed.return_value = [0.1] * 1536
+
+                results, total = await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+                # Verify the mock was set up correctly
+                assert mock_schema.return_value == "platform"
+
+                assert results == []
+                assert total == 0
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_without_embeddings():
+    """Test hybrid search fails fast when embeddings are unavailable."""
+    # Patch where the function is used, not where it's defined
+    with patch("backend.api.features.store.hybrid_search.embed_query") as mock_embed:
+        # Simulate embedding failure
+        mock_embed.return_value = None
+
+        # Should raise ValueError with helpful message
+        with pytest.raises(ValueError) as exc_info:
+            await hybrid_search(
+                query="test",
+                page=1,
+                page_size=20,
+            )
+
+        # Verify error message is generic (doesn't leak implementation details)
+        assert "Search service temporarily unavailable" in str(exc_info.value)
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_with_filters():
+    """Test hybrid search with various filters."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test with featured filter
+            results, total = await hybrid_search(
+                query="test",
+                featured=True,
+                creators=["user1", "user2"],
+                category="productivity",
+                page=1,
+                page_size=10,
+            )
+
+            # Verify filters were applied in the query
+            call_args = mock_query.call_args
+            params = call_args[0][1:]  # Skip SQL template
+
+            # Should have query, query_lower, creators array, category
+            assert len(params) >= 4
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_weights():
+    """Test hybrid search with custom weights."""
+    custom_weights = HybridSearchWeights(
+        semantic=0.5,
+        lexical=0.3,
+        category=0.1,
+        recency=0.1,
+        popularity=0.0,
+    )
+
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            results, total = await hybrid_search(
+                query="test",
+                weights=custom_weights,
+                page=1,
+                page_size=20,
+            )
+
+            # Verify custom weights were used in the query
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            params = call_args[0][1:]  # Get all parameters passed
+
+            # Check that SQL uses parameterized weights (not f-string interpolation)
+            assert "$" in sql_template  # Verify parameterization is used
+
+            # Check that custom weights are in the params
+            assert 0.5 in params  # semantic weight
+            assert 0.3 in params  # lexical weight
+            assert 0.1 in params  # category and recency weights
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_min_score_filtering():
+    """Test hybrid search minimum score threshold."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Return results with varying scores
+        mock_query.return_value = [
+            {
+                "slug": "high-score/agent",
+                "agent_name": "High Score Agent",
+                "combined_score": 0.8,
+                "total_count": 1,
+                # ... other fields
+            }
+        ]
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test with custom min_score
+            results, total = await hybrid_search(
+                query="test",
+                min_score=0.5,  # High threshold
+                page=1,
+                page_size=20,
+            )
+
+            # Verify min_score was applied in query
+            call_args = mock_query.call_args
+            sql_template = call_args[0][0]
+            params = call_args[0][1:]  # Get all parameters
+
+            # Check that SQL uses parameterized min_score
+            assert "combined_score >=" in sql_template
+            assert "$" in sql_template  # Verify parameterization
+
+            # Check that custom min_score is in the params
+            assert 0.5 in params
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_pagination():
+    """Test hybrid search pagination."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        mock_query.return_value = []
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Test page 2 with page_size 10
+            results, total = await hybrid_search(
+                query="test",
+                page=2,
+                page_size=10,
+            )
+
+            # Verify pagination parameters
+            call_args = mock_query.call_args
+            params = call_args[0]
+
+            # Last two params should be LIMIT and OFFSET
+            limit = params[-2]
+            offset = params[-1]
+
+            assert limit == 10  # page_size
+            assert offset == 10  # (page - 1) * page_size = (2 - 1) * 10
+
+
+@pytest.mark.asyncio(loop_scope="session")
+@pytest.mark.integration
+async def test_hybrid_search_error_handling():
+    """Test hybrid search error handling."""
+    with patch(
+        "backend.api.features.store.hybrid_search.query_raw_with_schema"
+    ) as mock_query:
+        # Simulate database error
+        mock_query.side_effect = Exception("Database connection error")
+
+        with patch(
+            "backend.api.features.store.hybrid_search.embed_query"
+        ) as mock_embed:
+            mock_embed.return_value = [0.1] * 1536
+
+            # Should raise exception
+            with pytest.raises(Exception) as exc_info:
+                await hybrid_search(
+                    query="test",
+                    page=1,
+                    page_size=20,
+                )
+
+            assert "Database connection error" in str(exc_info.value)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "-s"])

autogpt_platform/backend/backend/blocks/claude_code.py

@@ -1,602 +0,0 @@
-import shlex
-from typing import Literal
-
-from e2b import AsyncSandbox as BaseAsyncSandbox
-from pydantic import BaseModel, SecretStr
-
-from backend.data.block import (
-    Block,
-    BlockCategory,
-    BlockOutput,
-    BlockSchemaInput,
-    BlockSchemaOutput,
-)
-from backend.data.model import (
-    APIKeyCredentials,
-    CredentialsField,
-    CredentialsMetaInput,
-    SchemaField,
-)
-from backend.integrations.providers import ProviderName
-
-# Test credentials for E2B
-TEST_E2B_CREDENTIALS = APIKeyCredentials(
-    id="01234567-89ab-cdef-0123-456789abcdef",
-    provider="e2b",
-    api_key=SecretStr("mock-e2b-api-key"),
-    title="Mock E2B API key",
-    expires_at=None,
-)
-TEST_E2B_CREDENTIALS_INPUT = {
-    "provider": TEST_E2B_CREDENTIALS.provider,
-    "id": TEST_E2B_CREDENTIALS.id,
-    "type": TEST_E2B_CREDENTIALS.type,
-    "title": TEST_E2B_CREDENTIALS.title,
-}
-
-# Test credentials for Anthropic
-TEST_ANTHROPIC_CREDENTIALS = APIKeyCredentials(
-    id="2e568a2b-b2ea-475a-8564-9a676bf31c56",
-    provider="anthropic",
-    api_key=SecretStr("mock-anthropic-api-key"),
-    title="Mock Anthropic API key",
-    expires_at=None,
-)
-TEST_ANTHROPIC_CREDENTIALS_INPUT = {
-    "provider": TEST_ANTHROPIC_CREDENTIALS.provider,
-    "id": TEST_ANTHROPIC_CREDENTIALS.id,
-    "type": TEST_ANTHROPIC_CREDENTIALS.type,
-    "title": TEST_ANTHROPIC_CREDENTIALS.title,
-}
-
-
-class ClaudeCodeBlock(Block):
-    """
-    Execute tasks using Claude Code (Anthropic's AI coding assistant) in an E2B sandbox.
-
-    Claude Code can create files, install tools, run commands, and perform complex
-    coding tasks autonomously within a secure sandbox environment.
-    """
-
-    # Use base template - we'll install Claude Code ourselves for latest version
-    DEFAULT_TEMPLATE = "base"
-
-    class Input(BlockSchemaInput):
-        e2b_credentials: CredentialsMetaInput[
-            Literal[ProviderName.E2B], Literal["api_key"]
-        ] = CredentialsField(
-            description=(
-                "API key for the E2B platform to create the sandbox. "
-                "Get one at https://e2b.dev/docs"
-            ),
-        )
-
-        anthropic_credentials: CredentialsMetaInput[
-            Literal[ProviderName.ANTHROPIC], Literal["api_key"]
-        ] = CredentialsField(
-            description=(
-                "API key for Anthropic to power Claude Code. "
-                "Get one at https://console.anthropic.com/"
-            ),
-        )
-
-        prompt: str = SchemaField(
-            description=(
-                "The task or instruction for Claude Code to execute. "
-                "Claude Code can create files, install packages, run commands, "
-                "and perform complex coding tasks."
-            ),
-            placeholder="Create a hello world index.html file",
-            default="",
-            advanced=False,
-        )
-
-        timeout: int = SchemaField(
-            description=(
-                "Sandbox timeout in seconds. Claude Code tasks can take "
-                "a while, so set this appropriately for your task complexity. "
-                "Note: This only applies when creating a new sandbox. "
-                "When reconnecting to an existing sandbox via sandbox_id, "
-                "the original timeout is retained."
-            ),
-            default=300,  # 5 minutes default
-            advanced=True,
-        )
-
-        setup_commands: list[str] = SchemaField(
-            description=(
-                "Optional shell commands to run before executing Claude Code. "
-                "Useful for installing dependencies or setting up the environment."
-            ),
-            default_factory=list,
-            advanced=True,
-        )
-
-        working_directory: str = SchemaField(
-            description="Working directory for Claude Code to operate in.",
-            default="/home/user",
-            advanced=True,
-        )
-
-        # Session/continuation support
-        session_id: str = SchemaField(
-            description=(
-                "Session ID to resume a previous conversation. "
-                "Leave empty for a new conversation. "
-                "Use the session_id from a previous run to continue that conversation."
-            ),
-            default="",
-            advanced=True,
-        )
-
-        sandbox_id: str = SchemaField(
-            description=(
-                "Sandbox ID to reconnect to an existing sandbox. "
-                "Required when resuming a session (along with session_id). "
-                "Use the sandbox_id from a previous run where dispose_sandbox was False."
-            ),
-            default="",
-            advanced=True,
-        )
-
-        conversation_history: str = SchemaField(
-            description=(
-                "Previous conversation history to continue from. "
-                "Use this to restore context on a fresh sandbox if the previous one timed out. "
-                "Pass the conversation_history output from a previous run."
-            ),
-            default="",
-            advanced=True,
-        )
-
-        dispose_sandbox: bool = SchemaField(
-            description=(
-                "Whether to dispose of the sandbox immediately after execution. "
-                "Set to False if you want to continue the conversation later "
-                "(you'll need both sandbox_id and session_id from the output)."
-            ),
-            default=True,
-            advanced=True,
-        )
-
-    class FileOutput(BaseModel):
-        """A file extracted from the sandbox."""
-
-        path: str
-        relative_path: str  # Path relative to working directory (for GitHub, etc.)
-        name: str
-        content: str
-
-    class Output(BlockSchemaOutput):
-        response: str = SchemaField(
-            description="The output/response from Claude Code execution"
-        )
-        files: list["ClaudeCodeBlock.FileOutput"] = SchemaField(
-            description=(
-                "List of files created/modified by Claude Code. "
-                "Each file has 'path', 'name', and 'content' fields."
-            )
-        )
-        conversation_history: str = SchemaField(
-            description=(
-                "Full conversation history including this turn. "
-                "Pass this to conversation_history input to continue on a fresh sandbox "
-                "if the previous sandbox timed out."
-            )
-        )
-        session_id: str = SchemaField(
-            description=(
-                "Session ID for this conversation. "
-                "Pass this back along with sandbox_id to continue the conversation."
-            )
-        )
-        sandbox_id: str = SchemaField(
-            description=(
-                "ID of the sandbox instance. "
-                "Pass this back along with session_id to continue the conversation "
-                "(only available if dispose_sandbox was False)."
-            )
-        )
-
-    def __init__(self):
-        super().__init__(
-            id="4e34f4a5-9b89-4326-ba77-2dd6750b7194",
-            description=(
-                "Execute tasks using Claude Code in an E2B sandbox. "
-                "Claude Code can create files, install tools, run commands, "
-                "and perform complex coding tasks autonomously."
-            ),
-            categories={BlockCategory.DEVELOPER_TOOLS, BlockCategory.AI},
-            input_schema=ClaudeCodeBlock.Input,
-            output_schema=ClaudeCodeBlock.Output,
-            test_credentials={
-                "e2b_credentials": TEST_E2B_CREDENTIALS,
-                "anthropic_credentials": TEST_ANTHROPIC_CREDENTIALS,
-            },
-            test_input={
-                "e2b_credentials": TEST_E2B_CREDENTIALS_INPUT,
-                "anthropic_credentials": TEST_ANTHROPIC_CREDENTIALS_INPUT,
-                "prompt": "Create a hello world HTML file",
-                "timeout": 300,
-                "setup_commands": [],
-                "working_directory": "/home/user",
-                "session_id": "",
-                "sandbox_id": "",
-                "conversation_history": "",
-                "dispose_sandbox": True,
-            },
-            test_output=[
-                ("response", "Created index.html with hello world content"),
-                (
-                    "files",
-                    [
-                        {
-                            "path": "/home/user/index.html",
-                            "relative_path": "index.html",
-                            "name": "index.html",
-                            "content": "<html>Hello World</html>",
-                        }
-                    ],
-                ),
-                (
-                    "conversation_history",
-                    "User: Create a hello world HTML file\n"
-                    "Claude: Created index.html with hello world content",
-                ),
-                ("session_id", str),
-            ],
-            test_mock={
-                "execute_claude_code": lambda *args, **kwargs: (
-                    "Created index.html with hello world content",  # response
-                    [
-                        ClaudeCodeBlock.FileOutput(
-                            path="/home/user/index.html",
-                            relative_path="index.html",
-                            name="index.html",
-                            content="<html>Hello World</html>",
-                        )
-                    ],  # files
-                    "User: Create a hello world HTML file\n"
-                    "Claude: Created index.html with hello world content",  # conversation_history
-                    "test-session-id",  # session_id
-                    "sandbox_id",  # sandbox_id
-                ),
-            },
-        )
-
-    async def execute_claude_code(
-        self,
-        e2b_api_key: str,
-        anthropic_api_key: str,
-        prompt: str,
-        timeout: int,
-        setup_commands: list[str],
-        working_directory: str,
-        session_id: str,
-        existing_sandbox_id: str,
-        conversation_history: str,
-        dispose_sandbox: bool,
-    ) -> tuple[str, list["ClaudeCodeBlock.FileOutput"], str, str, str]:
-        """
-        Execute Claude Code in an E2B sandbox.
-
-        Returns:
-            Tuple of (response, files, conversation_history, session_id, sandbox_id)
-        """
-        import json
-        import uuid
-
-        # Validate that sandbox_id is provided when resuming a session
-        if session_id and not existing_sandbox_id:
-            raise ValueError(
-                "sandbox_id is required when resuming a session with session_id. "
-                "The session state is stored in the original sandbox. "
-                "If the sandbox has timed out, use conversation_history instead "
-                "to restore context on a fresh sandbox."
-            )
-
-        sandbox = None
-
-        try:
-            # Either reconnect to existing sandbox or create a new one
-            if existing_sandbox_id:
-                # Reconnect to existing sandbox for conversation continuation
-                sandbox = await BaseAsyncSandbox.connect(
-                    sandbox_id=existing_sandbox_id,
-                    api_key=e2b_api_key,
-                )
-            else:
-                # Create new sandbox
-                sandbox = await BaseAsyncSandbox.create(
-                    template=self.DEFAULT_TEMPLATE,
-                    api_key=e2b_api_key,
-                    timeout=timeout,
-                    envs={"ANTHROPIC_API_KEY": anthropic_api_key},
-                )
-
-                # Install Claude Code from npm (ensures we get the latest version)
-                install_result = await sandbox.commands.run(
-                    "npm install -g @anthropic-ai/claude-code@latest",
-                    timeout=120,  # 2 min timeout for install
-                )
-                if install_result.exit_code != 0:
-                    raise Exception(
-                        f"Failed to install Claude Code: {install_result.stderr}"
-                    )
-
-                # Run any user-provided setup commands
-                for cmd in setup_commands:
-                    setup_result = await sandbox.commands.run(cmd)
-                    if setup_result.exit_code != 0:
-                        raise Exception(
-                            f"Setup command failed: {cmd}\n"
-                            f"Exit code: {setup_result.exit_code}\n"
-                            f"Stdout: {setup_result.stdout}\n"
-                            f"Stderr: {setup_result.stderr}"
-                        )
-
-            # Generate or use provided session ID
-            current_session_id = session_id if session_id else str(uuid.uuid4())
-
-            # Build base Claude flags
-            base_flags = "-p --dangerously-skip-permissions --output-format json"
-
-            # Add conversation history context if provided (for fresh sandbox continuation)
-            history_flag = ""
-            if conversation_history and not session_id:
-                # Inject previous conversation as context via system prompt
-                # Use consistent escaping via _escape_prompt helper
-                escaped_history = self._escape_prompt(
-                    f"Previous conversation context: {conversation_history}"
-                )
-                history_flag = f" --append-system-prompt {escaped_history}"
-
-            # Build Claude command based on whether we're resuming or starting new
-            # Use shlex.quote for working_directory and session IDs to prevent injection
-            safe_working_dir = shlex.quote(working_directory)
-            if session_id:
-                # Resuming existing session (sandbox still alive)
-                safe_session_id = shlex.quote(session_id)
-                claude_command = (
-                    f"cd {safe_working_dir} && "
-                    f"echo {self._escape_prompt(prompt)} | "
-                    f"claude --resume {safe_session_id} {base_flags}"
-                )
-            else:
-                # New session with specific ID
-                safe_current_session_id = shlex.quote(current_session_id)
-                claude_command = (
-                    f"cd {safe_working_dir} && "
-                    f"echo {self._escape_prompt(prompt)} | "
-                    f"claude --session-id {safe_current_session_id} {base_flags}{history_flag}"
-                )
-
-            result = await sandbox.commands.run(
-                claude_command,
-                timeout=0,  # No command timeout - let sandbox timeout handle it
-            )
-
-            # Check for command failure
-            if result.exit_code != 0:
-                error_msg = result.stderr or result.stdout or "Unknown error"
-                raise Exception(
-                    f"Claude Code command failed with exit code {result.exit_code}:\n"
-                    f"{error_msg}"
-                )
-
-            raw_output = result.stdout or ""
-            sandbox_id = sandbox.sandbox_id
-
-            # Parse JSON output to extract response and build conversation history
-            response = ""
-            new_conversation_history = conversation_history or ""
-
-            try:
-                # The JSON output contains the result
-                output_data = json.loads(raw_output)
-                response = output_data.get("result", raw_output)
-
-                # Build conversation history entry
-                turn_entry = f"User: {prompt}\nClaude: {response}"
-                if new_conversation_history:
-                    new_conversation_history = (
-                        f"{new_conversation_history}\n\n{turn_entry}"
-                    )
-                else:
-                    new_conversation_history = turn_entry
-
-            except json.JSONDecodeError:
-                # If not valid JSON, use raw output
-                response = raw_output
-                turn_entry = f"User: {prompt}\nClaude: {response}"
-                if new_conversation_history:
-                    new_conversation_history = (
-                        f"{new_conversation_history}\n\n{turn_entry}"
-                    )
-                else:
-                    new_conversation_history = turn_entry
-
-            # Extract files from the working directory
-            files = await self._extract_files(sandbox, working_directory)
-
-            return (
-                response,
-                files,
-                new_conversation_history,
-                current_session_id,
-                sandbox_id,
-            )
-
-        finally:
-            if dispose_sandbox and sandbox:
-                await sandbox.kill()
-
-    async def _extract_files(
-        self,
-        sandbox: BaseAsyncSandbox,
-        working_directory: str,
-    ) -> list["ClaudeCodeBlock.FileOutput"]:
-        """
-        Extract files from the sandbox working directory.
-
-        Returns:
-            List of FileOutput objects with path, name, and content
-        """
-        files: list[ClaudeCodeBlock.FileOutput] = []
-
-        # Text file extensions we can safely read
-        text_extensions = {
-            ".txt",
-            ".md",
-            ".html",
-            ".htm",
-            ".css",
-            ".js",
-            ".ts",
-            ".jsx",
-            ".tsx",
-            ".json",
-            ".xml",
-            ".yaml",
-            ".yml",
-            ".toml",
-            ".ini",
-            ".cfg",
-            ".conf",
-            ".py",
-            ".rb",
-            ".php",
-            ".java",
-            ".c",
-            ".cpp",
-            ".h",
-            ".hpp",
-            ".cs",
-            ".go",
-            ".rs",
-            ".swift",
-            ".kt",
-            ".scala",
-            ".sh",
-            ".bash",
-            ".zsh",
-            ".sql",
-            ".graphql",
-            ".env",
-            ".gitignore",
-            ".dockerfile",
-            "Dockerfile",
-            ".vue",
-            ".svelte",
-            ".astro",
-            ".mdx",
-            ".rst",
-            ".tex",
-            ".csv",
-            ".log",
-        }
-
-        try:
-            # List files recursively using find command
-            # Exclude node_modules and .git directories, but allow hidden files
-            # like .env and .gitignore (they're filtered by text_extensions later)
-            safe_working_dir = shlex.quote(working_directory)
-            find_result = await sandbox.commands.run(
-                f"find {safe_working_dir} -type f "
-                f"-not -path '*/node_modules/*' "
-                f"-not -path '*/.git/*' "
-                f"2>/dev/null | head -100"
-            )
-
-            if find_result.stdout:
-                for file_path in find_result.stdout.strip().split("\n"):
-                    if not file_path:
-                        continue
-
-                    # Check if it's a text file we can read
-                    is_text = any(
-                        file_path.endswith(ext) for ext in text_extensions
-                    ) or file_path.endswith("Dockerfile")
-
-                    if is_text:
-                        try:
-                            content = await sandbox.files.read(file_path)
-                            # Handle bytes or string
-                            if isinstance(content, bytes):
-                                content = content.decode("utf-8", errors="replace")
-
-                            # Extract filename from path
-                            file_name = file_path.split("/")[-1]
-
-                            # Calculate relative path by stripping working directory
-                            relative_path = file_path
-                            if file_path.startswith(working_directory):
-                                relative_path = file_path[len(working_directory) :]
-                                # Remove leading slash if present
-                                if relative_path.startswith("/"):
-                                    relative_path = relative_path[1:]
-
-                            files.append(
-                                ClaudeCodeBlock.FileOutput(
-                                    path=file_path,
-                                    relative_path=relative_path,
-                                    name=file_name,
-                                    content=content,
-                                )
-                            )
-                        except Exception:
-                            # Skip files that can't be read
-                            pass
-
-        except Exception:
-            # If file extraction fails, return empty results
-            pass
-
-        return files
-
-    def _escape_prompt(self, prompt: str) -> str:
-        """Escape the prompt for safe shell execution."""
-        # Use single quotes and escape any single quotes in the prompt
-        escaped = prompt.replace("'", "'\"'\"'")
-        return f"'{escaped}'"
-
-    async def run(
-        self,
-        input_data: Input,
-        *,
-        e2b_credentials: APIKeyCredentials,
-        anthropic_credentials: APIKeyCredentials,
-        **kwargs,
-    ) -> BlockOutput:
-        try:
-            (
-                response,
-                files,
-                conversation_history,
-                session_id,
-                sandbox_id,
-            ) = await self.execute_claude_code(
-                e2b_api_key=e2b_credentials.api_key.get_secret_value(),
-                anthropic_api_key=anthropic_credentials.api_key.get_secret_value(),
-                prompt=input_data.prompt,
-                timeout=input_data.timeout,
-                setup_commands=input_data.setup_commands,
-                working_directory=input_data.working_directory,
-                session_id=input_data.session_id,
-                existing_sandbox_id=input_data.sandbox_id,
-                conversation_history=input_data.conversation_history,
-                dispose_sandbox=input_data.dispose_sandbox,
-            )
-
-            yield "response", response
-            # Always yield files (empty list if none) to match Output schema
-            yield "files", [f.model_dump() for f in files]
-            # Always yield conversation_history so user can restore context on fresh sandbox
-            yield "conversation_history", conversation_history
-            # Always yield session_id so user can continue conversation
-            yield "session_id", session_id
-            if not input_data.dispose_sandbox and sandbox_id:
-                yield "sandbox_id", sandbox_id
-
-        except Exception as e:
-            yield "error", str(e)

autogpt_platform/backend/backend/data/db.py

@@ -38,6 +38,20 @@ POOL_TIMEOUT = os.getenv("DB_POOL_TIMEOUT")
 if POOL_TIMEOUT:
     DATABASE_URL = add_param(DATABASE_URL, "pool_timeout", POOL_TIMEOUT)
 
+# Add public schema to search_path for pgvector type access
+# The vector extension is in public schema, but search_path is determined by schema parameter
+# Extract the schema from DATABASE_URL or default to 'platform'
+parsed_url = urlparse(DATABASE_URL)
+url_params = dict(parse_qsl(parsed_url.query))
+db_schema = url_params.get("schema", "platform")
+# Build search_path, avoiding duplicates if db_schema is already 'public'
+search_path_schemas = list(
+    dict.fromkeys([db_schema, "public"])
+)  # Preserves order, removes duplicates
+search_path = ",".join(search_path_schemas)
+# This allows using ::vector without schema qualification
+DATABASE_URL = add_param(DATABASE_URL, "options", f"-c search_path={search_path}")
+
 HTTP_TIMEOUT = int(POOL_TIMEOUT) if POOL_TIMEOUT else None
 
 prisma = Prisma(
@@ -108,21 +122,84 @@ def get_database_schema() -> str:
     return query_params.get("schema", "public")
 
 
-async def query_raw_with_schema(query_template: str, *args) -> list[dict]:
-    """Execute raw SQL query with proper schema handling."""
+async def _raw_with_schema(
+    query_template: str,
+    *args,
+    execute: bool = False,
+    client: Prisma | None = None,
+) -> list[dict] | int:
+    """Internal: Execute raw SQL with proper schema handling.
+
+    Use query_raw_with_schema() or execute_raw_with_schema() instead.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+        execute: If False, executes SELECT query. If True, executes INSERT/UPDATE/DELETE.
+        client: Optional Prisma client for transactions (only used when execute=True).
+
+    Returns:
+        - list[dict] if execute=False (query results)
+        - int if execute=True (number of affected rows)
+    """
     schema = get_database_schema()
     schema_prefix = f'"{schema}".' if schema != "public" else ""
     formatted_query = query_template.format(schema_prefix=schema_prefix)
 
     import prisma as prisma_module
 
-    result = await prisma_module.get_client().query_raw(
-        formatted_query, *args  # type: ignore
-    )
+    db_client = client if client else prisma_module.get_client()
+
+    if execute:
+        result = await db_client.execute_raw(formatted_query, *args)  # type: ignore
+    else:
+        result = await db_client.query_raw(formatted_query, *args)  # type: ignore
 
     return result
 
 
+async def query_raw_with_schema(query_template: str, *args) -> list[dict]:
+    """Execute raw SQL SELECT query with proper schema handling.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+
+    Returns:
+        List of result rows as dictionaries
+
+    Example:
+        results = await query_raw_with_schema(
+            'SELECT * FROM {schema_prefix}"User" WHERE id = $1',
+            user_id
+        )
+    """
+    return await _raw_with_schema(query_template, *args, execute=False)  # type: ignore
+
+
+async def execute_raw_with_schema(
+    query_template: str, *args, client: Prisma | None = None
+) -> int:
+    """Execute raw SQL command (INSERT/UPDATE/DELETE) with proper schema handling.
+
+    Args:
+        query_template: SQL query with {schema_prefix} placeholder
+        *args: Query parameters
+        client: Optional Prisma client for transactions
+
+    Returns:
+        Number of affected rows
+
+    Example:
+        await execute_raw_with_schema(
+            'INSERT INTO {schema_prefix}"User" (id, name) VALUES ($1, $2)',
+            user_id, name,
+            client=tx  # Optional transaction client
+        )
+    """
+    return await _raw_with_schema(query_template, *args, execute=True, client=client)  # type: ignore
+
+
 class BaseDbModel(BaseModel):
     id: str = Field(default_factory=lambda: str(uuid4()))
 

autogpt_platform/backend/backend/data/graph_test.py

@@ -1,5 +1,6 @@
 import json
 from typing import Any
+from unittest.mock import AsyncMock, patch
 from uuid import UUID
 
 import fastapi.exceptions
@@ -18,6 +19,17 @@ from backend.usecases.sample import create_test_user
 from backend.util.test import SpinTestServer
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 @pytest.mark.asyncio(loop_scope="session")
 async def test_graph_creation(server: SpinTestServer, snapshot: Snapshot):
     """

autogpt_platform/backend/backend/executor/database.py

@@ -7,6 +7,10 @@ from backend.api.features.library.db import (
     list_library_agents,
 )
 from backend.api.features.store.db import get_store_agent_details, get_store_agents
+from backend.api.features.store.embeddings import (
+    backfill_missing_embeddings,
+    get_embedding_stats,
+)
 from backend.data import db
 from backend.data.analytics import (
     get_accuracy_trends_and_alerts,
@@ -208,6 +212,10 @@ class DatabaseManager(AppService):
     get_store_agents = _(get_store_agents)
     get_store_agent_details = _(get_store_agent_details)
 
+    # Store Embeddings
+    get_embedding_stats = _(get_embedding_stats)
+    backfill_missing_embeddings = _(backfill_missing_embeddings)
+
     # Summary data - async
     get_user_execution_summary_data = _(get_user_execution_summary_data)
 
@@ -259,6 +267,10 @@ class DatabaseManagerClient(AppServiceClient):
     get_store_agents = _(d.get_store_agents)
     get_store_agent_details = _(d.get_store_agent_details)
 
+    # Store Embeddings
+    get_embedding_stats = _(d.get_embedding_stats)
+    backfill_missing_embeddings = _(d.backfill_missing_embeddings)
+
 
 class DatabaseManagerAsyncClient(AppServiceClient):
     d = DatabaseManager

autogpt_platform/backend/backend/executor/manager_test.py

@@ -1,4 +1,5 @@
 import logging
+from unittest.mock import AsyncMock, patch
 
 import fastapi.responses
 import pytest
@@ -19,6 +20,17 @@ from backend.util.test import SpinTestServer, wait_execution
 logger = logging.getLogger(__name__)
 
 
+@pytest.fixture(scope="session", autouse=True)
+def mock_embedding_functions():
+    """Mock embedding functions for all tests to avoid database/API dependencies."""
+    with patch(
+        "backend.api.features.store.db.ensure_embedding",
+        new_callable=AsyncMock,
+        return_value=True,
+    ):
+        yield
+
+
 async def create_graph(s: SpinTestServer, g: graph.Graph, u: User) -> graph.Graph:
     logger.info(f"Creating graph for user {u.id}")
     return await s.agent_server.test_create_graph(CreateGraph(graph=g), u.id)

autogpt_platform/backend/backend/executor/scheduler.py

@@ -2,6 +2,7 @@ import asyncio
 import logging
 import os
 import threading
+import time
 import uuid
 from enum import Enum
 from typing import Optional
@@ -37,7 +38,7 @@ from backend.monitoring import (
     report_execution_accuracy_alerts,
     report_late_executions,
 )
-from backend.util.clients import get_scheduler_client
+from backend.util.clients import get_database_manager_client, get_scheduler_client
 from backend.util.cloud_storage import cleanup_expired_files_async
 from backend.util.exceptions import (
     GraphNotFoundError,
@@ -254,6 +255,74 @@ def execution_accuracy_alerts():
     return report_execution_accuracy_alerts()
 
 
+def ensure_embeddings_coverage():
+    """
+    Ensure approved store agents have embeddings for hybrid search.
+
+    Processes ALL missing embeddings in batches of 10 until 100% coverage.
+    Missing embeddings = agents invisible in hybrid search.
+
+    Schedule: Runs every 6 hours (balanced between coverage and API costs).
+    - Catches agents approved between scheduled runs
+    - Batch size 10: gradual processing to avoid rate limits
+    - Manual trigger available via execute_ensure_embeddings_coverage endpoint
+    """
+    db_client = get_database_manager_client()
+    stats = db_client.get_embedding_stats()
+
+    # Check for error from get_embedding_stats() first
+    if "error" in stats:
+        logger.error(
+            f"Failed to get embedding stats: {stats['error']} - skipping backfill"
+        )
+        return {"processed": 0, "success": 0, "failed": 0, "error": stats["error"]}
+
+    if stats["without_embeddings"] == 0:
+        logger.info("All approved agents have embeddings, skipping backfill")
+        return {"processed": 0, "success": 0, "failed": 0}
+
+    logger.info(
+        f"Found {stats['without_embeddings']} agents without embeddings "
+        f"({stats['coverage_percent']}% coverage) - processing all"
+    )
+
+    total_processed = 0
+    total_success = 0
+    total_failed = 0
+
+    # Process in batches until no more missing embeddings
+    while True:
+        result = db_client.backfill_missing_embeddings(batch_size=10)
+
+        total_processed += result["processed"]
+        total_success += result["success"]
+        total_failed += result["failed"]
+
+        if result["processed"] == 0:
+            # No more missing embeddings
+            break
+
+        if result["success"] == 0 and result["processed"] > 0:
+            # All attempts in this batch failed - stop to avoid infinite loop
+            logger.error(
+                f"All {result['processed']} embedding attempts failed - stopping backfill"
+            )
+            break
+
+        # Small delay between batches to avoid rate limits
+        time.sleep(1)
+
+    logger.info(
+        f"Embedding backfill completed: {total_success}/{total_processed} succeeded, "
+        f"{total_failed} failed"
+    )
+    return {
+        "processed": total_processed,
+        "success": total_success,
+        "failed": total_failed,
+    }
+
+
 # Monitoring functions are now imported from monitoring module
 
 
@@ -475,6 +544,19 @@ class Scheduler(AppService):
                 jobstore=Jobstores.EXECUTION.value,
             )
 
+            # Embedding Coverage - Every 6 hours
+            # Ensures all approved agents have embeddings for hybrid search
+            # Critical: missing embeddings = agents invisible in search
+            self.scheduler.add_job(
+                ensure_embeddings_coverage,
+                id="ensure_embeddings_coverage",
+                trigger="interval",
+                hours=6,
+                replace_existing=True,
+                max_instances=1,  # Prevent overlapping runs
+                jobstore=Jobstores.EXECUTION.value,
+            )
+
         self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
         self.scheduler.add_listener(job_missed_listener, EVENT_JOB_MISSED)
         self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)
@@ -632,6 +714,11 @@ class Scheduler(AppService):
         """Manually trigger execution accuracy alert checking."""
         return execution_accuracy_alerts()
 
+    @expose
+    def execute_ensure_embeddings_coverage(self):
+        """Manually trigger embedding backfill for approved store agents."""
+        return ensure_embeddings_coverage()
+
 
 class SchedulerClient(AppServiceClient):
     @classmethod

autogpt_platform/backend/backend/util/clients.py

@@ -10,6 +10,7 @@ from backend.util.settings import Settings
 settings = Settings()
 
 if TYPE_CHECKING:
+    from openai import AsyncOpenAI
     from supabase import AClient, Client
 
     from backend.data.execution import (
@@ -139,6 +140,24 @@ async def get_async_supabase() -> "AClient":
     )
 
 
+# ============ OpenAI Client ============ #
+
+
+@cached(ttl_seconds=3600)
+def get_openai_client() -> "AsyncOpenAI | None":
+    """
+    Get a process-cached async OpenAI client for embeddings.
+
+    Returns None if API key is not configured.
+    """
+    from openai import AsyncOpenAI
+
+    api_key = settings.secrets.openai_internal_api_key
+    if not api_key:
+        return None
+    return AsyncOpenAI(api_key=api_key)
+
+
 # ============ Notification Queue Helpers ============ #
 
 

autogpt_platform/backend/migrations/20260109181714_add_docs_embedding/migration.sql

@@ -0,0 +1,46 @@
+-- CreateExtension
+-- Supabase: pgvector must be enabled via Dashboard → Database → Extensions first
+-- Create in public schema so vector type is available across all schemas
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "vector" WITH SCHEMA "public";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'vector extension not available or already exists, skipping';
+END $$;
+
+-- CreateEnum
+CREATE TYPE "ContentType" AS ENUM ('STORE_AGENT', 'BLOCK', 'INTEGRATION', 'DOCUMENTATION', 'LIBRARY_AGENT');
+
+-- CreateTable
+CREATE TABLE "UnifiedContentEmbedding" (
+    "id" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "contentType" "ContentType" NOT NULL,
+    "contentId" TEXT NOT NULL,
+    "userId" TEXT,
+    "embedding" public.vector(1536) NOT NULL,
+    "searchableText" TEXT NOT NULL,
+    "metadata" JSONB NOT NULL DEFAULT '{}',
+
+    CONSTRAINT "UnifiedContentEmbedding_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_contentType_idx" ON "UnifiedContentEmbedding"("contentType");
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_userId_idx" ON "UnifiedContentEmbedding"("userId");
+
+-- CreateIndex
+CREATE INDEX "UnifiedContentEmbedding_contentType_userId_idx" ON "UnifiedContentEmbedding"("contentType", "userId");
+
+-- CreateIndex
+-- NULLS NOT DISTINCT ensures only one public (NULL userId) embedding per contentType+contentId
+-- Requires PostgreSQL 15+. Supabase uses PostgreSQL 15+.
+CREATE UNIQUE INDEX "UnifiedContentEmbedding_contentType_contentId_userId_key" ON "UnifiedContentEmbedding"("contentType", "contentId", "userId") NULLS NOT DISTINCT;
+
+-- CreateIndex
+-- HNSW index for fast vector similarity search on embeddings
+-- Uses cosine distance operator (<=>), which matches the query in hybrid_search.py
+CREATE INDEX "UnifiedContentEmbedding_embedding_idx" ON "UnifiedContentEmbedding" USING hnsw ("embedding" public.vector_cosine_ops);

autogpt_platform/backend/migrations/20260112173500_add_supabase_extensions_to_platform_schema/migration.sql

@@ -0,0 +1,71 @@
+-- Acknowledge Supabase-managed extensions to prevent drift warnings
+-- These extensions are pre-installed by Supabase in specific schemas
+-- This migration ensures they exist where available (Supabase) or skips gracefully (CI)
+
+-- Create schemas (safe in both CI and Supabase)
+CREATE SCHEMA IF NOT EXISTS "extensions";
+
+-- Extensions that exist in both CI and Supabase
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pgcrypto" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgcrypto extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'uuid-ossp extension not available, skipping';
+END $$;
+
+-- Supabase-specific extensions (skip gracefully in CI)
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pg_stat_statements" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_stat_statements extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pg_net" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_net extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE EXTENSION IF NOT EXISTS "pgjwt" WITH SCHEMA "extensions";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgjwt extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "graphql";
+    CREATE EXTENSION IF NOT EXISTS "pg_graphql" WITH SCHEMA "graphql";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pg_graphql extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "pgsodium";
+    CREATE EXTENSION IF NOT EXISTS "pgsodium" WITH SCHEMA "pgsodium";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'pgsodium extension not available, skipping';
+END $$;
+
+DO $$
+BEGIN
+    CREATE SCHEMA IF NOT EXISTS "vault";
+    CREATE EXTENSION IF NOT EXISTS "supabase_vault" WITH SCHEMA "vault";
+EXCEPTION WHEN OTHERS THEN
+    RAISE NOTICE 'supabase_vault extension not available, skipping';
+END $$;
+
+
+-- Return to platform
+CREATE SCHEMA IF NOT EXISTS "platform";

autogpt_platform/backend/schema.prisma

@@ -1,14 +1,15 @@
 datasource db {
-  provider  = "postgresql"
-  url       = env("DATABASE_URL")
-  directUrl = env("DIRECT_URL")
+  provider   = "postgresql"
+  url        = env("DATABASE_URL")
+  directUrl  = env("DIRECT_URL")
+  extensions = [pgvector(map: "vector")]
 }
 
 generator client {
   provider               = "prisma-client-py"
   recursive_type_depth   = -1
   interface              = "asyncio"
-  previewFeatures        = ["views", "fullTextSearch"]
+  previewFeatures        = ["views", "fullTextSearch", "postgresqlExtensions"]
   partial_type_generator = "backend/data/partial_types.py"
 }
 
@@ -127,8 +128,8 @@ model BuilderSearchHistory {
   updatedAt DateTime @default(now()) @updatedAt
 
   searchQuery String
-  filter      String[]  @default([])
-  byCreator   String[]  @default([])
+  filter      String[] @default([])
+  byCreator   String[] @default([])
 
   userId String
   User   User   @relation(fields: [userId], references: [id], onDelete: Cascade)
@@ -721,26 +722,25 @@ view StoreAgent {
   storeListingVersionId String
   updated_at            DateTime
 
-  slug                String
-  agent_name          String
-  agent_video         String?
-  agent_output_demo   String?
-  agent_image         String[]
+  slug              String
+  agent_name        String
+  agent_video       String?
+  agent_output_demo String?
+  agent_image       String[]
 
-  featured         Boolean                  @default(false)
-  creator_username String?
-  creator_avatar   String?
-  sub_heading      String
-  description      String
-  categories       String[]
-  search           Unsupported("tsvector")? @default(dbgenerated("''::tsvector"))
-  runs                Int
-  rating              Float
-  versions            String[]
-  agentGraphVersions  String[]
-  agentGraphId        String
-  is_available        Boolean                  @default(true)
-  useForOnboarding    Boolean                  @default(false)
+  featured           Boolean  @default(false)
+  creator_username   String?
+  creator_avatar     String?
+  sub_heading        String
+  description        String
+  categories         String[]
+  runs               Int
+  rating             Float
+  versions           String[]
+  agentGraphVersions String[]
+  agentGraphId       String
+  is_available       Boolean  @default(true)
+  useForOnboarding   Boolean  @default(false)
 
   // Materialized views used (refreshed every 15 minutes via pg_cron):
   // - mv_agent_run_counts - Pre-aggregated agent execution counts by agentGraphId
@@ -856,14 +856,14 @@ model StoreListingVersion {
   AgentGraph        AgentGraph @relation(fields: [agentGraphId, agentGraphVersion], references: [id, version])
 
   // Content fields
-  name                String
-  subHeading          String
-  videoUrl            String?
-  agentOutputDemoUrl  String?
-  imageUrls           String[]
-  description         String
-  instructions        String?
-  categories          String[]
+  name               String
+  subHeading         String
+  videoUrl           String?
+  agentOutputDemoUrl String?
+  imageUrls          String[]
+  description        String
+  instructions       String?
+  categories         String[]
 
   isFeatured Boolean @default(false)
 
@@ -899,6 +899,9 @@ model StoreListingVersion {
   // Reviews for this specific version
   Reviews StoreListingReview[]
 
+  // Note: Embeddings now stored in UnifiedContentEmbedding table
+  // Use contentType=STORE_AGENT and contentId=storeListingVersionId
+
   @@unique([storeListingId, version])
   @@index([storeListingId, submissionStatus, isAvailable])
   @@index([submissionStatus])
@@ -906,6 +909,42 @@ model StoreListingVersion {
   @@index([agentGraphId, agentGraphVersion]) // Non-unique index for efficient lookups
 }
 
+// Content type enum for unified search across store agents, blocks, docs
+// Note: BLOCK/INTEGRATION are file-based (Python classes), not DB records
+// DOCUMENTATION are file-based (.md files), not DB records  
+// Only STORE_AGENT and LIBRARY_AGENT are stored in database
+enum ContentType {
+  STORE_AGENT // Database: StoreListingVersion
+  BLOCK // File-based: Python classes in /backend/blocks/
+  INTEGRATION // File-based: Python classes (blocks with credentials)
+  DOCUMENTATION // File-based: .md/.mdx files
+  LIBRARY_AGENT // Database: User's personal agents
+}
+
+// Unified embeddings table for all searchable content types
+// Supports both public content (userId=null) and user-specific content (userId=userID)
+model UnifiedContentEmbedding {
+  id        String   @id @default(uuid())
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  // Content identification
+  contentType ContentType
+  contentId   String // DB ID (storeListingVersionId) or file identifier (block.id, file_path)
+  userId      String? // NULL for public content (store, blocks, docs), userId for private content (library agents)
+
+  // Search data
+  embedding      Unsupported("vector(1536)") // pgvector embedding (extension in platform schema)
+  searchableText String // Combined text for search and fallback
+  metadata       Json                        @default("{}") // Content-specific metadata
+
+  @@unique([contentType, contentId, userId], map: "UnifiedContentEmbedding_contentType_contentId_userId_key")
+  @@index([contentType])
+  @@index([userId])
+  @@index([contentType, userId])
+  @@index([embedding], map: "UnifiedContentEmbedding_embedding_idx")
+}
+
 model StoreListingReview {
   id        String   @id @default(uuid())
   createdAt DateTime @default(now())
@@ -998,16 +1037,16 @@ model OAuthApplication {
   updatedAt DateTime @updatedAt
 
   // Application metadata
-  name               String
-  description        String?
-  logoUrl            String? // URL to app logo stored in GCS
-  clientId           String @unique
-  clientSecret       String // Hashed with Scrypt (same as API keys)
-  clientSecretSalt   String // Salt for Scrypt hashing
+  name             String
+  description      String?
+  logoUrl          String? // URL to app logo stored in GCS
+  clientId         String  @unique
+  clientSecret     String // Hashed with Scrypt (same as API keys)
+  clientSecretSalt String // Salt for Scrypt hashing
 
   // OAuth configuration
   redirectUris String[] // Allowed callback URLs
-  grantTypes   String[] @default(["authorization_code", "refresh_token"])
+  grantTypes   String[]           @default(["authorization_code", "refresh_token"])
   scopes       APIKeyPermission[] // Which permissions the app can request
 
   // Application management