From a8098505c291ea159299135da2eb7d143b6328a2 Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Mon, 5 Jan 2026 14:27:06 -0500 Subject: [PATCH] Add litellm_extra_body metadata for V1 conversations (#12266) Co-authored-by: openhands --- .../live_status_app_conversation_service.py | 65 +++++++++++++++++ openhands/app_server/utils/llm_metadata.py | 73 +++++++++++++++++++ ...st_live_status_app_conversation_service.py | 24 ++++++ .../experiments/test_experiment_manager.py | 1 + 4 files changed, 163 insertions(+) create mode 100644 openhands/app_server/utils/llm_metadata.py diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py index 887c4dfeb4..ac4609e648 100644 --- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py +++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py @@ -72,6 +72,10 @@ from openhands.app_server.user.user_models import UserInfo from openhands.app_server.utils.docker_utils import ( replace_localhost_hostname_for_docker, ) +from openhands.app_server.utils.llm_metadata import ( + get_llm_metadata, + should_set_litellm_extra_body, +) from openhands.experiments.experiment_manager import ExperimentManagerImpl from openhands.integrations.provider import ProviderType from openhands.sdk import Agent, AgentContext, LocalWorkspace @@ -892,6 +896,63 @@ class LiveStatusAppConversationService(AppConversationServiceBase): return agent + def _update_agent_with_llm_metadata( + self, + agent: Agent, + conversation_id: UUID, + user_id: str | None, + ) -> Agent: + """Update agent's LLM and condenser LLM with litellm_extra_body metadata. + + This adds tracing metadata (conversation_id, user_id, etc.) to the LLM + for analytics and debugging purposes. Only applies to openhands/ models. + + Args: + agent: The agent to update + conversation_id: The conversation ID + user_id: The user ID (can be None) + + Returns: + Updated agent with LLM metadata + """ + updates: dict[str, Any] = {} + + # Update main LLM if it's an openhands model + if should_set_litellm_extra_body(agent.llm.model): + llm_metadata = get_llm_metadata( + model_name=agent.llm.model, + llm_type=agent.llm.usage_id or 'agent', + conversation_id=conversation_id, + user_id=user_id, + ) + updated_llm = agent.llm.model_copy( + update={'litellm_extra_body': {'metadata': llm_metadata}} + ) + updates['llm'] = updated_llm + + # Update condenser LLM if it exists and is an openhands model + if agent.condenser and hasattr(agent.condenser, 'llm'): + condenser_llm = agent.condenser.llm + if should_set_litellm_extra_body(condenser_llm.model): + condenser_metadata = get_llm_metadata( + model_name=condenser_llm.model, + llm_type=condenser_llm.usage_id or 'condenser', + conversation_id=conversation_id, + user_id=user_id, + ) + updated_condenser_llm = condenser_llm.model_copy( + update={'litellm_extra_body': {'metadata': condenser_metadata}} + ) + updated_condenser = agent.condenser.model_copy( + update={'llm': updated_condenser_llm} + ) + updates['condenser'] = updated_condenser + + # Return updated agent if there are changes + if updates: + return agent.model_copy(update=updates) + return agent + async def _finalize_conversation_request( self, agent: Agent, @@ -930,6 +991,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase): user.id, conversation_id, agent ) + # Update agent's LLM with litellm_extra_body metadata for tracing + # This is done after experiment variants to ensure the final LLM config is used + agent = self._update_agent_with_llm_metadata(agent, conversation_id, user.id) + # Load and merge skills if remote workspace is available if remote_workspace: try: diff --git a/openhands/app_server/utils/llm_metadata.py b/openhands/app_server/utils/llm_metadata.py new file mode 100644 index 0000000000..8413b437e3 --- /dev/null +++ b/openhands/app_server/utils/llm_metadata.py @@ -0,0 +1,73 @@ +"""Utility functions for LLM metadata in OpenHands V1 conversations.""" + +import os +from typing import Any +from uuid import UUID + +import openhands + + +def should_set_litellm_extra_body(model_name: str) -> bool: + """Determine if litellm_extra_body should be set based on the model name. + + Only set litellm_extra_body for openhands models to avoid issues + with providers that don't support extra_body parameters. + + The SDK internally translates "openhands/" prefix to "litellm_proxy/" + when making API calls, so we check for both. + + Args: + model_name: Name of the LLM model + + Returns: + True if litellm_extra_body should be set, False otherwise + """ + return 'openhands/' in model_name or 'litellm_proxy/' in model_name + + +def get_llm_metadata( + model_name: str, + llm_type: str, + conversation_id: UUID | str | None = None, + user_id: str | None = None, +) -> dict[str, Any]: + """Generate LLM metadata for OpenHands V1 conversations. + + This metadata is passed to the LiteLLM proxy for tracing and analytics. + + Args: + model_name: Name of the LLM model + llm_type: Type of LLM usage (e.g., 'agent', 'condenser', 'planning_condenser') + conversation_id: Optional conversation identifier + user_id: Optional user identifier + + Returns: + Dictionary containing metadata for LLM initialization + """ + openhands_version = openhands.__version__ + + metadata: dict[str, Any] = { + 'trace_version': openhands_version, + 'tags': [ + 'app:openhands', + f'model:{model_name}', + f'type:{llm_type}', + f'web_host:{os.environ.get("WEB_HOST", "unspecified")}', + f'openhands_version:{openhands_version}', + 'conversation_version:V1', + ], + } + + if conversation_id is not None: + # Convert UUID to string if needed + session_id = ( + str(conversation_id) + if isinstance(conversation_id, UUID) + else conversation_id + ) + metadata['session_id'] = session_id + + if user_id is not None: + metadata['trace_user_id'] = user_id + + return metadata diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py index 126e54c69c..f37a7894b8 100644 --- a/tests/unit/app_server/test_live_status_app_conversation_service.py +++ b/tests/unit/app_server/test_live_status_app_conversation_service.py @@ -793,7 +793,15 @@ class TestLiveStatusAppConversationService: """Test _finalize_conversation_request with skills loading.""" # Arrange mock_agent = Mock(spec=Agent) + + # Create mock LLM with required attributes for _update_agent_with_llm_metadata + mock_llm = Mock(spec=LLM) + mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update + mock_llm.usage_id = 'agent' + mock_updated_agent = Mock(spec=Agent) + mock_updated_agent.llm = mock_llm + mock_updated_agent.condenser = None # No condenser mock_experiment_manager.run_agent_variant_tests__v1.return_value = ( mock_updated_agent ) @@ -852,7 +860,15 @@ class TestLiveStatusAppConversationService: """Test _finalize_conversation_request without remote workspace (no skills).""" # Arrange mock_agent = Mock(spec=Agent) + + # Create mock LLM with required attributes for _update_agent_with_llm_metadata + mock_llm = Mock(spec=LLM) + mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update + mock_llm.usage_id = 'agent' + mock_updated_agent = Mock(spec=Agent) + mock_updated_agent.llm = mock_llm + mock_updated_agent.condenser = None # No condenser mock_experiment_manager.run_agent_variant_tests__v1.return_value = ( mock_updated_agent ) @@ -890,7 +906,15 @@ class TestLiveStatusAppConversationService: """Test _finalize_conversation_request when skills loading fails.""" # Arrange mock_agent = Mock(spec=Agent) + + # Create mock LLM with required attributes for _update_agent_with_llm_metadata + mock_llm = Mock(spec=LLM) + mock_llm.model = 'gpt-4' # Non-openhands model, so no metadata update + mock_llm.usage_id = 'agent' + mock_updated_agent = Mock(spec=Agent) + mock_updated_agent.llm = mock_llm + mock_updated_agent.condenser = None # No condenser mock_experiment_manager.run_agent_variant_tests__v1.return_value = ( mock_updated_agent ) diff --git a/tests/unit/experiments/test_experiment_manager.py b/tests/unit/experiments/test_experiment_manager.py index 70cd6c5d07..6a9d5e4a32 100644 --- a/tests/unit/experiments/test_experiment_manager.py +++ b/tests/unit/experiments/test_experiment_manager.py @@ -139,6 +139,7 @@ class TestExperimentManagerIntegration: mock_agent = Mock(spec=Agent) mock_agent.llm = mock_llm + mock_agent.condenser = None # No condenser for this test mock_agent.system_prompt_filename = 'default_system_prompt.j2' mock_agent.model_copy = Mock(return_value=mock_agent)