[eval]: disable MCP for SWE-Bench evaluation (#8574)

Co-authored-by: openhands <openhands@all-hands.dev>
Co-authored-by: Engel Nyst <enyst@users.noreply.github.com>
Co-authored-by: Engel Nyst <engel.nyst@gmail.com>

evaluation/benchmarks/swe_bench/run_infer.py

@@ -261,6 +261,7 @@ def get_config(
         enable_jupyter=False,
         enable_browsing=RUN_WITH_BROWSING,
         enable_llm_editor=False,
+        enable_mcp=False,
         condenser=metadata.condenser_config,
         enable_prompt_extensions=False,
     )

openhands/cli/main.py

@@ -251,7 +251,8 @@ async def run_session(
     )
 
     # Add MCP tools to the agent
-    await add_mcp_tools_to_agent(agent, runtime, memory, config.mcp)
+    if agent.config.enable_mcp:
+        await add_mcp_tools_to_agent(agent, runtime, memory, config.mcp)
 
     # Clear loading animation
     is_loaded.set()

openhands/core/config/agent_config.py

@@ -28,6 +28,8 @@ class AgentConfig(BaseModel):
     """Whether to enable finish tool"""
     enable_prompt_extensions: bool = Field(default=True)
     """Whether to enable prompt extensions"""
+    enable_mcp: bool = Field(default=True)
+    """Whether to enable MCP tools"""
     disabled_microagents: list[str] = Field(default_factory=list)
     """A list of microagents to disable (by name, without .py extension, e.g. ["github", "lint"]). Default is None."""
     enable_history_truncation: bool = Field(default=True)

openhands/core/main.py

@@ -129,7 +129,8 @@ async def run_controller(
         )
 
     # Add MCP tools to the agent
-    await add_mcp_tools_to_agent(agent, runtime, memory, config.mcp)
+    if agent.config.enable_mcp:
+        await add_mcp_tools_to_agent(agent, runtime, memory, config.mcp)
 
     replay_events: list[Event] | None = None
     if config.replay_trajectory_path:

openhands/server/session/agent_session.py

@@ -149,7 +149,7 @@ class AgentSession:
 
             # NOTE: this needs to happen before controller is created
             # so MCP tools can be included into the SystemMessageAction
-            if self.runtime and runtime_connected:
+            if self.runtime and runtime_connected and agent.config.enable_mcp:
                 await add_mcp_tools_to_agent(agent, self.runtime, self.memory, config.mcp)
 
             if replay_json:

tests/unit/test_agent_controller.py

@@ -57,6 +57,10 @@ def mock_agent():
     agent.llm.metrics = Metrics()
     agent.llm.config = AppConfig().get_llm_config()
 
+    # Add config with enable_mcp attribute
+    agent.config = MagicMock(spec=AgentConfig)
+    agent.config.enable_mcp = True
+
     # Add a proper system message mock
     system_message = SystemMessageAction(
         content='Test system message', tools=['test_tool']

tests/unit/test_agent_session.py

@@ -35,6 +35,7 @@ def mock_agent():
 
     # Configure the agent config
     agent_config.disabled_microagents = []
+    agent_config.enable_mcp = True
 
     # Set up the chain of mocks
     llm.metrics = metrics