From 368a0248e33c73f5249f4d42cace2935cbb921b2 Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Wed, 13 Aug 2025 14:41:28 -0400 Subject: [PATCH] Modify experiment manager defaults for nested runtimes (#10269) Co-authored-by: openhands --- openhands/experiments/experiment_manager.py | 38 +++++++++++++++++-- openhands/server/constants.py | 3 ++ .../docker_nested_conversation_manager.py | 3 +- .../standalone_conversation_manager.py | 3 +- .../server/routes/manage_conversations.py | 27 +++++++++++++ openhands/server/session/session.py | 7 ++-- openhands/storage/locations.py | 4 ++ 7 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 openhands/server/constants.py diff --git a/openhands/experiments/experiment_manager.py b/openhands/experiments/experiment_manager.py index 2e84542e9a..96a81b9652 100644 --- a/openhands/experiments/experiment_manager.py +++ b/openhands/experiments/experiment_manager.py @@ -1,11 +1,33 @@ import os +from pydantic import BaseModel + from openhands.core.config.openhands_config import OpenHandsConfig from openhands.core.logger import openhands_logger as logger from openhands.server.session.conversation_init_data import ConversationInitData +from openhands.server.shared import file_store +from openhands.storage.locations import get_experiment_config_filename from openhands.utils.import_utils import get_impl +class ExperimentConfig(BaseModel): + config: dict[str, str] | None = None + + +def load_experiment_config(conversation_id: str) -> ExperimentConfig | None: + try: + file_path = get_experiment_config_filename(conversation_id) + exp_config = file_store.read(file_path) + return ExperimentConfig.model_validate_json(exp_config) + + except FileNotFoundError: + pass + except Exception as e: + logger.warning(f'Failed to load experiment config: {e}') + + return None + + class ExperimentManager: @staticmethod def run_conversation_variant_test( @@ -17,9 +39,19 @@ class ExperimentManager: def run_config_variant_test( user_id: str, conversation_id: str, config: OpenHandsConfig ) -> OpenHandsConfig: - logger.debug( - f'Running agent config variant test for user_id={user_id}, conversation_id={conversation_id}' - ) + exp_config = load_experiment_config(conversation_id) + if exp_config and exp_config.config: + agent_cfg = config.get_agent_config(config.default_agent) + try: + for attr, value in exp_config.config.items(): + if hasattr(agent_cfg, attr): + logger.info( + f'Set attrib {attr} to {value} for {conversation_id}' + ) + setattr(agent_cfg, attr, value) + except Exception as e: + logger.warning(f'Error processing exp config: {e}') + return config diff --git a/openhands/server/constants.py b/openhands/server/constants.py new file mode 100644 index 0000000000..2f09bab748 --- /dev/null +++ b/openhands/server/constants.py @@ -0,0 +1,3 @@ +"""Server constants.""" + +ROOM_KEY = 'room:{sid}' diff --git a/openhands/server/conversation_manager/docker_nested_conversation_manager.py b/openhands/server/conversation_manager/docker_nested_conversation_manager.py index 275e9a4a8d..82db976a4f 100644 --- a/openhands/server/conversation_manager/docker_nested_conversation_manager.py +++ b/openhands/server/conversation_manager/docker_nested_conversation_manager.py @@ -25,6 +25,7 @@ from openhands.llm.llm import LLM from openhands.runtime import get_runtime_cls from openhands.runtime.impl.docker.docker_runtime import DockerRuntime from openhands.server.config.server_config import ServerConfig +from openhands.server.constants import ROOM_KEY from openhands.server.conversation_manager.conversation_manager import ( ConversationManager, ) @@ -32,7 +33,7 @@ from openhands.server.data_models.agent_loop_info import AgentLoopInfo from openhands.server.monitoring import MonitoringListener from openhands.server.session.conversation import ServerConversation from openhands.server.session.conversation_init_data import ConversationInitData -from openhands.server.session.session import ROOM_KEY, Session +from openhands.server.session.session import Session from openhands.storage.conversation.conversation_store import ConversationStore from openhands.storage.data_models.conversation_metadata import ConversationMetadata from openhands.storage.data_models.conversation_status import ConversationStatus diff --git a/openhands/server/conversation_manager/standalone_conversation_manager.py b/openhands/server/conversation_manager/standalone_conversation_manager.py index ad8dc5b0b5..b646d40f70 100644 --- a/openhands/server/conversation_manager/standalone_conversation_manager.py +++ b/openhands/server/conversation_manager/standalone_conversation_manager.py @@ -14,11 +14,12 @@ from openhands.events.action import MessageAction from openhands.events.stream import EventStreamSubscriber, session_exists from openhands.runtime import get_runtime_cls from openhands.server.config.server_config import ServerConfig +from openhands.server.constants import ROOM_KEY from openhands.server.data_models.agent_loop_info import AgentLoopInfo from openhands.server.monitoring import MonitoringListener from openhands.server.session.agent_session import WAIT_TIME_BEFORE_CLOSE, AgentSession from openhands.server.session.conversation import ServerConversation -from openhands.server.session.session import ROOM_KEY, Session +from openhands.server.session.session import Session from openhands.storage.conversation.conversation_store import ConversationStore from openhands.storage.data_models.conversation_metadata import ConversationMetadata from openhands.storage.data_models.conversation_status import ConversationStatus diff --git a/openhands/server/routes/manage_conversations.py b/openhands/server/routes/manage_conversations.py index 4d5dbc077d..4da6b7e310 100644 --- a/openhands/server/routes/manage_conversations.py +++ b/openhands/server/routes/manage_conversations.py @@ -22,6 +22,7 @@ from openhands.events.observation import ( AgentStateChangedObservation, NullObservation, ) +from openhands.experiments.experiment_manager import ExperimentConfig from openhands.integrations.provider import ( PROVIDER_TOKEN_TYPE, ProviderHandler, @@ -71,6 +72,7 @@ from openhands.storage.data_models.conversation_metadata import ( from openhands.storage.data_models.conversation_status import ConversationStatus from openhands.storage.data_models.settings import Settings from openhands.storage.data_models.user_secrets import UserSecrets +from openhands.storage.locations import get_experiment_config_filename from openhands.storage.settings.settings_store import SettingsStore from openhands.utils.async_utils import wait_all from openhands.utils.conversation_summary import get_default_conversation_title @@ -707,3 +709,28 @@ async def update_conversation( }, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + + +@app.post('/conversations/{conversation_id}/exp-config') +def add_experiment_config_for_conversation( + conversation_id: str, exp_config: ExperimentConfig +) -> bool: + exp_config_filepath = get_experiment_config_filename(conversation_id) + exists = False + try: + file_store.read(exp_config_filepath) + exists = True + except FileNotFoundError: + pass + + # Don't modify again if it already exists + if exists: + return False + + try: + file_store.write(exp_config_filepath, exp_config.model_dump_json()) + except Exception as e: + logger.info(f'Failed to write experiment config for {conversation_id}: {e}') + return True + + return False diff --git a/openhands/server/session/session.py b/openhands/server/session/session.py index 5f2836322e..458f45abe2 100644 --- a/openhands/server/session/session.py +++ b/openhands/server/session/session.py @@ -28,16 +28,14 @@ from openhands.events.observation.agent import RecallObservation from openhands.events.observation.error import ErrorObservation from openhands.events.serialization import event_from_dict, event_to_dict from openhands.events.stream import EventStreamSubscriber -from openhands.experiments.experiment_manager import ExperimentManagerImpl from openhands.llm.llm import LLM from openhands.runtime.runtime_status import RuntimeStatus +from openhands.server.constants import ROOM_KEY from openhands.server.session.agent_session import AgentSession from openhands.server.session.conversation_init_data import ConversationInitData from openhands.storage.data_models.settings import Settings from openhands.storage.files import FileStore -ROOM_KEY = 'room:{sid}' - class Session: sid: str @@ -75,6 +73,9 @@ class Session: ) # Copying this means that when we update variables they are not applied to the shared global configuration! self.config = deepcopy(config) + # Lazy import to avoid circular dependency + from openhands.experiments.experiment_manager import ExperimentManagerImpl + self.config = ExperimentManagerImpl.run_config_variant_test( user_id, sid, self.config ) diff --git a/openhands/storage/locations.py b/openhands/storage/locations.py index 43e1661b91..192721da86 100644 --- a/openhands/storage/locations.py +++ b/openhands/storage/locations.py @@ -28,3 +28,7 @@ def get_conversation_init_data_filename(sid: str, user_id: str | None = None) -> def get_conversation_agent_state_filename(sid: str, user_id: str | None = None) -> str: return f'{get_conversation_dir(sid, user_id)}agent_state.pkl' + + +def get_experiment_config_filename(sid: str, user_id: str | None = None) -> str: + return f'{get_conversation_dir(sid, user_id)}exp_config.json'