Add litellm_extra_body metadata for V1 conversations (#12266)

Co-authored-by: openhands <openhands@all-hands.dev>

openhands/app_server/app_conversation/live_status_app_conversation_service.py

@@ -72,6 +72,10 @@ from openhands.app_server.user.user_models import UserInfo
 from openhands.app_server.utils.docker_utils import (
     replace_localhost_hostname_for_docker,
 )
+from openhands.app_server.utils.llm_metadata import (
+    get_llm_metadata,
+    should_set_litellm_extra_body,
+)
 from openhands.experiments.experiment_manager import ExperimentManagerImpl
 from openhands.integrations.provider import ProviderType
 from openhands.sdk import Agent, AgentContext, LocalWorkspace
@@ -892,6 +896,63 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
 
         return agent
 
+    def _update_agent_with_llm_metadata(
+        self,
+        agent: Agent,
+        conversation_id: UUID,
+        user_id: str | None,
+    ) -> Agent:
+        """Update agent's LLM and condenser LLM with litellm_extra_body metadata.
+
+        This adds tracing metadata (conversation_id, user_id, etc.) to the LLM
+        for analytics and debugging purposes. Only applies to openhands/ models.
+
+        Args:
+            agent: The agent to update
+            conversation_id: The conversation ID
+            user_id: The user ID (can be None)
+
+        Returns:
+            Updated agent with LLM metadata
+        """
+        updates: dict[str, Any] = {}
+
+        # Update main LLM if it's an openhands model
+        if should_set_litellm_extra_body(agent.llm.model):
+            llm_metadata = get_llm_metadata(
+                model_name=agent.llm.model,
+                llm_type=agent.llm.usage_id or 'agent',
+                conversation_id=conversation_id,
+                user_id=user_id,
+            )
+            updated_llm = agent.llm.model_copy(
+                update={'litellm_extra_body': {'metadata': llm_metadata}}
+            )
+            updates['llm'] = updated_llm
+
+        # Update condenser LLM if it exists and is an openhands model
+        if agent.condenser and hasattr(agent.condenser, 'llm'):
+            condenser_llm = agent.condenser.llm
+            if should_set_litellm_extra_body(condenser_llm.model):
+                condenser_metadata = get_llm_metadata(
+                    model_name=condenser_llm.model,
+                    llm_type=condenser_llm.usage_id or 'condenser',
+                    conversation_id=conversation_id,
+                    user_id=user_id,
+                )
+                updated_condenser_llm = condenser_llm.model_copy(
+                    update={'litellm_extra_body': {'metadata': condenser_metadata}}
+                )
+                updated_condenser = agent.condenser.model_copy(
+                    update={'llm': updated_condenser_llm}
+                )
+                updates['condenser'] = updated_condenser
+
+        # Return updated agent if there are changes
+        if updates:
+            return agent.model_copy(update=updates)
+        return agent
+
     async def _finalize_conversation_request(
         self,
         agent: Agent,
@@ -930,6 +991,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
             user.id, conversation_id, agent
         )
 
+        # Update agent's LLM with litellm_extra_body metadata for tracing
+        # This is done after experiment variants to ensure the final LLM config is used
+        agent = self._update_agent_with_llm_metadata(agent, conversation_id, user.id)
+
         # Load and merge skills if remote workspace is available
         if remote_workspace:
             try:

openhands/app_server/utils/llm_metadata.py

@@ -0,0 +1,73 @@
+"""Utility functions for LLM metadata in OpenHands V1 conversations."""
+
+import os
+from typing import Any
+from uuid import UUID
+
+import openhands
+
+
+def should_set_litellm_extra_body(model_name: str) -> bool:
+    """Determine if litellm_extra_body should be set based on the model name.
+
+    Only set litellm_extra_body for openhands models to avoid issues
+    with providers that don't support extra_body parameters.
+
+    The SDK internally translates "openhands/" prefix to "litellm_proxy/"
+    when making API calls, so we check for both.
+
+    Args:
+        model_name: Name of the LLM model
+
+    Returns:
+        True if litellm_extra_body should be set, False otherwise
+    """
+    return 'openhands/' in model_name or 'litellm_proxy/' in model_name
+
+
+def get_llm_metadata(
+    model_name: str,
+    llm_type: str,
+    conversation_id: UUID | str | None = None,
+    user_id: str | None = None,
+) -> dict[str, Any]:
+    """Generate LLM metadata for OpenHands V1 conversations.
+
+    This metadata is passed to the LiteLLM proxy for tracing and analytics.
+
+    Args:
+        model_name: Name of the LLM model
+        llm_type: Type of LLM usage (e.g., 'agent', 'condenser', 'planning_condenser')
+        conversation_id: Optional conversation identifier
+        user_id: Optional user identifier
+
+    Returns:
+        Dictionary containing metadata for LLM initialization
+    """
+    openhands_version = openhands.__version__
+
+    metadata: dict[str, Any] = {
+        'trace_version': openhands_version,
+        'tags': [
+            'app:openhands',
+            f'model:{model_name}',
+            f'type:{llm_type}',
+            f'web_host:{os.environ.get("WEB_HOST", "unspecified")}',
+            f'openhands_version:{openhands_version}',
+            'conversation_version:V1',
+        ],
+    }
+
+    if conversation_id is not None:
+        # Convert UUID to string if needed
+        session_id = (
+            str(conversation_id)
+            if isinstance(conversation_id, UUID)
+            else conversation_id
+        )
+        metadata['session_id'] = session_id
+
+    if user_id is not None:
+        metadata['trace_user_id'] = user_id
+
+    return metadata

tests/unit/app_server/test_live_status_app_conversation_service.py

@@ -793,7 +793,15 @@ class TestLiveStatusAppConversationService:
         """Test _finalize_conversation_request with skills loading."""
         # Arrange
         mock_agent = Mock(spec=Agent)
+
+        # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+        mock_llm = Mock(spec=LLM)
+        mock_llm.model = 'gpt-4'  # Non-openhands model, so no metadata update
+        mock_llm.usage_id = 'agent'
+
         mock_updated_agent = Mock(spec=Agent)
+        mock_updated_agent.llm = mock_llm
+        mock_updated_agent.condenser = None  # No condenser
         mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
             mock_updated_agent
         )
@@ -852,7 +860,15 @@ class TestLiveStatusAppConversationService:
         """Test _finalize_conversation_request without remote workspace (no skills)."""
         # Arrange
         mock_agent = Mock(spec=Agent)
+
+        # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+        mock_llm = Mock(spec=LLM)
+        mock_llm.model = 'gpt-4'  # Non-openhands model, so no metadata update
+        mock_llm.usage_id = 'agent'
+
         mock_updated_agent = Mock(spec=Agent)
+        mock_updated_agent.llm = mock_llm
+        mock_updated_agent.condenser = None  # No condenser
         mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
             mock_updated_agent
         )
@@ -890,7 +906,15 @@ class TestLiveStatusAppConversationService:
         """Test _finalize_conversation_request when skills loading fails."""
         # Arrange
         mock_agent = Mock(spec=Agent)
+
+        # Create mock LLM with required attributes for _update_agent_with_llm_metadata
+        mock_llm = Mock(spec=LLM)
+        mock_llm.model = 'gpt-4'  # Non-openhands model, so no metadata update
+        mock_llm.usage_id = 'agent'
+
         mock_updated_agent = Mock(spec=Agent)
+        mock_updated_agent.llm = mock_llm
+        mock_updated_agent.condenser = None  # No condenser
         mock_experiment_manager.run_agent_variant_tests__v1.return_value = (
             mock_updated_agent
         )

tests/unit/experiments/test_experiment_manager.py

@@ -139,6 +139,7 @@ class TestExperimentManagerIntegration:
 
         mock_agent = Mock(spec=Agent)
         mock_agent.llm = mock_llm
+        mock_agent.condenser = None  # No condenser for this test
         mock_agent.system_prompt_filename = 'default_system_prompt.j2'
         mock_agent.model_copy = Mock(return_value=mock_agent)