diff --git a/openhands/experiments/experiment_manager.py b/openhands/experiments/experiment_manager.py index 39c6af6284..2e84542e9a 100644 --- a/openhands/experiments/experiment_manager.py +++ b/openhands/experiments/experiment_manager.py @@ -1,5 +1,7 @@ import os +from openhands.core.config.openhands_config import OpenHandsConfig +from openhands.core.logger import openhands_logger as logger from openhands.server.session.conversation_init_data import ConversationInitData from openhands.utils.import_utils import get_impl @@ -11,6 +13,15 @@ class ExperimentManager: ) -> ConversationInitData: return conversation_settings + @staticmethod + def run_config_variant_test( + user_id: str, conversation_id: str, config: OpenHandsConfig + ) -> OpenHandsConfig: + logger.debug( + f'Running agent config variant test for user_id={user_id}, conversation_id={conversation_id}' + ) + return config + experiment_manager_cls = os.environ.get( 'OPENHANDS_EXPERIMENT_MANAGER_CLS', diff --git a/openhands/server/conversation_manager/docker_nested_conversation_manager.py b/openhands/server/conversation_manager/docker_nested_conversation_manager.py index bbd9366e7a..275e9a4a8d 100644 --- a/openhands/server/conversation_manager/docker_nested_conversation_manager.py +++ b/openhands/server/conversation_manager/docker_nested_conversation_manager.py @@ -19,6 +19,7 @@ from openhands.core.logger import openhands_logger as logger from openhands.events.action import MessageAction from openhands.events.nested_event_store import NestedEventStore from openhands.events.stream import EventStream +from openhands.experiments.experiment_manager import ExperimentManagerImpl from openhands.integrations.provider import PROVIDER_TOKEN_TYPE, ProviderHandler from openhands.llm.llm import LLM from openhands.runtime import get_runtime_cls @@ -468,24 +469,30 @@ class DockerNestedConversationManager(ConversationManager): ) -> DockerRuntime: # This session is created here only because it is the easiest way to get a runtime, which # is the easiest way to create the needed docker container + + # Run experiment manager variant test before creating session + config: OpenHandsConfig = ExperimentManagerImpl.run_config_variant_test( + user_id, sid, self.config + ) + session = Session( sid=sid, file_store=self.file_store, - config=self.config, + config=config, sio=self.sio, user_id=user_id, ) - agent_cls = settings.agent or self.config.default_agent + agent_cls = settings.agent or config.default_agent agent_name = agent_cls if agent_cls is not None else 'agent' llm = LLM( - config=self.config.get_llm_config_from_agent(agent_name), + config=config.get_llm_config_from_agent(agent_name), retry_listener=session._notify_on_llm_retry, ) llm = session._create_llm(agent_cls) - agent_config = self.config.get_agent_config(agent_cls) + agent_config = config.get_agent_config(agent_cls) agent = Agent.get_cls(agent_cls)(llm, agent_config) - config = self.config.model_copy(deep=True) + config = config.model_copy(deep=True) env_vars = config.sandbox.runtime_startup_env_vars env_vars['CONVERSATION_MANAGER_CLASS'] = ( 'openhands.server.conversation_manager.standalone_conversation_manager.StandaloneConversationManager' diff --git a/openhands/server/session/session.py b/openhands/server/session/session.py index 566b3f0b5c..9ad93f2360 100644 --- a/openhands/server/session/session.py +++ b/openhands/server/session/session.py @@ -28,6 +28,7 @@ from openhands.events.observation.agent import RecallObservation from openhands.events.observation.error import ErrorObservation from openhands.events.serialization import event_from_dict, event_to_dict from openhands.events.stream import EventStreamSubscriber +from openhands.experiments.experiment_manager import ExperimentManagerImpl from openhands.llm.llm import LLM from openhands.runtime.runtime_status import RuntimeStatus from openhands.server.session.agent_session import AgentSession @@ -74,6 +75,9 @@ class Session: ) # Copying this means that when we update variables they are not applied to the shared global configuration! self.config = deepcopy(config) + self.config = ExperimentManagerImpl.run_config_variant_test( + user_id, sid, self.config + ) self.loop = asyncio.get_event_loop() self.user_id = user_id