github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-09 14:57:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove remaining global config (#3099)

Browse Source 
* Remove global config from memory

* Remove runtime global config

* Remove from storage

* Remove global config

* Fix event stream tests

* Fix sandbox issue

* Change config

* Removed transferred tests

* Add swe env box

* Fixes on testing

* Fixed some tests

* Fix typing

* Fix ipython test

* Revive function

* Make temp_dir fixture

* Remove test to avoid circular import
This commit is contained in:
Graham Neubig
2024-07-26 14:43:32 -04:00
committed by GitHub
parent 3301beffec
commit 275ea706cf
41 changed files with 279 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
tests/integration/test_agent.py
View File

@@ -7,7 +7,7 @@ import pytest
from opendevin.controller.agent import Agent
from opendevin.controller.state.state import State
from opendevin.core.config import LLMConfig, parse_arguments
from opendevin.core.config import LLMConfig
from opendevin.core.main import run_agent_controller
from opendevin.core.schema import AgentState
from opendevin.events.action import (
@@ -74,10 +74,9 @@ def validate_final_state(final_state: State | None, test_name: str):
)
def test_write_simple_script(current_test_name: str) -> None:
task = "Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point."
args = parse_arguments()
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
final_state: State | None = asyncio.run(
run_agent_controller(
@@ -121,7 +120,6 @@ def test_write_simple_script(current_test_name: str) -> None:
reason='local sandbox shows environment-dependent absolute path for pwd command',
)
def test_edits(current_test_name: str):
args = parse_arguments()
# Copy workspace artifacts to workspace_base location
source_dir = os.path.join(os.path.dirname(__file__), 'workspace/test_edits/')
files = os.listdir(source_dir)
@@ -132,7 +130,7 @@ def test_edits(current_test_name: str):
shutil.copy(os.path.join(source_dir, file), dest_file)
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
# Execute the task
task = 'Fix typos in bad.txt. Do not ask me for confirmation at any point.'
@@ -164,10 +162,8 @@ Enjoy!
reason='Currently, only ssh sandbox supports stateful tasks',
)
def test_ipython(current_test_name: str):
args = parse_arguments()
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
# Execute the task
task = "Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'. Do not ask me for confirmation at any point."
@@ -199,10 +195,8 @@ def test_ipython(current_test_name: str):
reason='FIXME: local sandbox does not capture stderr',
)
def test_simple_task_rejection(current_test_name: str):
args = parse_arguments()
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
# Give an impossible task to do: cannot write a commit message because
# the workspace is not a git repo
@@ -224,10 +218,8 @@ def test_simple_task_rejection(current_test_name: str):
reason='Currently, only ssh sandbox supports stateful tasks',
)
def test_ipython_module(current_test_name: str):
args = parse_arguments()
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
# Execute the task
task = "Install and import pymsgbox==1.0.9 and print it's version in /workspace/test.txt. Do not ask me for confirmation at any point."
@@ -265,10 +257,8 @@ def test_ipython_module(current_test_name: str):
reason='CodeActAgent/CodeActSWEAgent only supports ssh sandbox which is stateful',
)
def test_browse_internet(http_server, current_test_name: str):
args = parse_arguments()
# Create the agent
agent = Agent.get_cls(args.agent_cls)(llm=LLM(LLMConfig()))
agent = Agent.get_cls(os.getenv('DEFAULT_AGENT'))(llm=LLM(LLMConfig()))
# Execute the task
task = 'Browse localhost:8000, and tell me the ultimate answer to life. Do not ask me for confirmation at any point.'

34
tests/unit/test_event_stream.py
View File

@@ -1,31 +1,39 @@
import json
import pathlib
import tempfile
import pytest
from opendevin.events import EventSource, EventStream
from opendevin.events.action import NullAction
from opendevin.events.action import (
NullAction,
)
from opendevin.events.observation import NullObservation
from opendevin.storage import get_file_store
@pytest.fixture
def event_stream():
event_stream = EventStream('abc')
yield event_stream
# clear after each test
event_stream.clear()
def temp_dir(monkeypatch):
# get a temporary directory
with tempfile.TemporaryDirectory() as temp_dir:
pathlib.Path().mkdir(parents=True, exist_ok=True)
yield temp_dir
def collect_events(stream):
return [event for event in stream.get_events()]
def test_basic_flow(event_stream: EventStream):
def test_basic_flow(temp_dir: str):
file_store = get_file_store('local', temp_dir)
event_stream = EventStream('abc', file_store)
event_stream.add_event(NullAction(), EventSource.AGENT)
assert len(collect_events(event_stream)) == 1
def test_stream_storage(event_stream: EventStream):
def test_stream_storage(temp_dir: str):
file_store = get_file_store('local', temp_dir)
event_stream = EventStream('abc', file_store)
event_stream.add_event(NullObservation(''), EventSource.AGENT)
assert len(collect_events(event_stream)) == 1
content = event_stream._file_store.read('sessions/abc/events/0.json')
@@ -43,15 +51,17 @@ def test_stream_storage(event_stream: EventStream):
}
def test_rehydration(event_stream: EventStream):
def test_rehydration(temp_dir: str):
file_store = get_file_store('local', temp_dir)
event_stream = EventStream('abc', file_store)
event_stream.add_event(NullObservation('obs1'), EventSource.AGENT)
event_stream.add_event(NullObservation('obs2'), EventSource.AGENT)
assert len(collect_events(event_stream)) == 2
stream2 = EventStream('es2')
stream2 = EventStream('es2', file_store)
assert len(collect_events(stream2)) == 0
stream1rehydrated = EventStream('abc')
stream1rehydrated = EventStream('abc', file_store)
events = collect_events(stream1rehydrated)
assert len(events) == 2
assert events[0].content == 'obs1'

6
tests/unit/test_ipython.py
View File

@@ -4,7 +4,7 @@ from unittest.mock import MagicMock, call, patch
import pytest
from opendevin.core.config import SandboxConfig
from opendevin.core.config import AppConfig, SandboxConfig
from opendevin.events.action import IPythonRunCellAction
from opendevin.events.observation import IPythonRunCellObservation
from opendevin.runtime.server.runtime import ServerRuntime
@@ -42,7 +42,9 @@ async def test_run_python_backticks():
):
# Initialize the runtime with the mock event_stream
runtime = ServerRuntime(
sandbox_config=SandboxConfig(box_type='ssh', persist_sandbox=False),
config=AppConfig(
persist_sandbox=False, sandbox=SandboxConfig(box_type='ssh')
),
event_stream=mock_event_stream,
)

12
tests/unit/test_is_stuck.py
View File

@@ -1,4 +1,5 @@
import logging
import tempfile
from unittest.mock import Mock, patch
import pytest
@@ -17,6 +18,7 @@ from opendevin.events.observation.empty import NullObservation
from opendevin.events.observation.error import ErrorObservation
from opendevin.events.stream import EventSource, EventStream
from opendevin.memory.history import ShortTermHistory
from opendevin.storage import get_file_store
def collect_events(stream):
@@ -28,11 +30,13 @@ logging.basicConfig(level=logging.DEBUG)
@pytest.fixture
def event_stream():
event_stream = EventStream('asdf')
yield event_stream
with tempfile.TemporaryDirectory() as temp_dir:
file_store = get_file_store('local', temp_dir)
event_stream = EventStream('asdf', file_store)
yield event_stream
# clear after each test
event_stream.clear()
# clear after each test
event_stream.clear()
class TestStuckDetector:

12
tests/unit/test_micro_agents.py
View File

@@ -1,5 +1,6 @@
import json
import os
import tempfile
from unittest.mock import MagicMock
import pytest
@@ -12,15 +13,18 @@ from opendevin.events import EventSource
from opendevin.events.action import MessageAction
from opendevin.events.stream import EventStream
from opendevin.memory.history import ShortTermHistory
from opendevin.storage import get_file_store
@pytest.fixture
def event_stream():
event_stream = EventStream('asdf')
yield event_stream
with tempfile.TemporaryDirectory() as temp_dir:
file_store = get_file_store('local', temp_dir)
event_stream = EventStream('asdf', file_store)
yield event_stream
# clear after each test
event_stream.clear()
# clear after each test
event_stream.clear()
def test_all_agents_are_loaded():

67
tests/unit/test_runtime.py
View File

@@ -9,7 +9,7 @@ from unittest.mock import patch
import pytest
from opendevin.core.config import SandboxConfig
from opendevin.core.config import AppConfig, SandboxConfig
from opendevin.core.logger import opendevin_logger as logger
from opendevin.events import EventStream
from opendevin.events.action import (
@@ -21,6 +21,7 @@ from opendevin.events.observation import (
from opendevin.runtime.client.runtime import EventStreamRuntime
from opendevin.runtime.plugins import AgentSkillsRequirement, JupyterRequirement
from opendevin.runtime.server.runtime import ServerRuntime
from opendevin.storage import get_file_store
@pytest.fixture(autouse=True)
@@ -46,23 +47,31 @@ def box_class(request):
return request.param
async def _load_runtime(box_class, event_stream):
async def _load_runtime(temp_dir, box_class):
sid = 'test'
cli_session = 'main_test'
plugins = [JupyterRequirement(), AgentSkillsRequirement()]
sandbox_config = SandboxConfig(
use_host_network=True,
config = AppConfig(
workspace_base=temp_dir,
workspace_mount_path=temp_dir,
sandbox=SandboxConfig(
use_host_network=True,
),
)
container_image = sandbox_config.container_image
file_store = get_file_store(config.file_store, config.file_store_path)
event_stream = EventStream(cli_session, file_store)
container_image = config.sandbox.container_image
# NOTE: we will use the default container image specified in the config.sandbox
# if it is an official od_runtime image.
if 'od_runtime' not in container_image:
container_image = 'ubuntu:22.04'
logger.warning(
f'`{sandbox_config.container_image}` is not an od_runtime image. Will use `{container_image}` as the container image for testing.'
f'`{config.sandbox.container_image}` is not an od_runtime image. Will use `{container_image}` as the container image for testing.'
)
if box_class == EventStreamRuntime:
runtime = EventStreamRuntime(
sandbox_config=sandbox_config,
config=config,
event_stream=event_stream,
sid=sid,
# NOTE: we probably don't have a default container image `/sandbox` for the event stream runtime
@@ -72,9 +81,7 @@ async def _load_runtime(box_class, event_stream):
)
await runtime.ainit()
elif box_class == ServerRuntime:
runtime = ServerRuntime(
sandbox_config=sandbox_config, event_stream=event_stream, sid=sid
)
runtime = ServerRuntime(config=config, event_stream=event_stream, sid=sid)
await runtime.ainit()
runtime.init_sandbox_plugins(plugins)
runtime.init_runtime_tools(
@@ -89,12 +96,9 @@ async def _load_runtime(box_class, event_stream):
@pytest.mark.asyncio
async def test_env_vars_os_environ(box_class):
async def test_env_vars_os_environ(temp_dir, box_class):
with patch.dict(os.environ, {'SANDBOX_ENV_FOOBAR': 'BAZ'}):
cli_session = 'main_test'
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
runtime = await _load_runtime(temp_dir, box_class)
obs: CmdOutputObservation = await runtime.run_action(
CmdRunAction(command='env')
@@ -115,11 +119,8 @@ async def test_env_vars_os_environ(box_class):
@pytest.mark.asyncio
async def test_env_vars_runtime_add_env_vars(box_class):
cli_session = 'main_test'
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
async def test_env_vars_runtime_add_env_vars(temp_dir, box_class):
runtime = await _load_runtime(temp_dir, box_class)
await runtime.add_env_vars({'QUUX': 'abc"def'})
obs: CmdOutputObservation = await runtime.run_action(
@@ -136,11 +137,8 @@ async def test_env_vars_runtime_add_env_vars(box_class):
@pytest.mark.asyncio
async def test_env_vars_runtime_add_empty_dict(box_class):
cli_session = 'main_test'
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
async def test_env_vars_runtime_add_empty_dict(temp_dir, box_class):
runtime = await _load_runtime(temp_dir, box_class)
prev_obs = await runtime.run_action(CmdRunAction(command='env'))
assert prev_obs.exit_code == 0, 'The exit code should be 0.'
@@ -160,11 +158,8 @@ async def test_env_vars_runtime_add_empty_dict(box_class):
@pytest.mark.asyncio
async def test_env_vars_runtime_add_multiple_env_vars(box_class):
cli_session = 'main_test'
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
async def test_env_vars_runtime_add_multiple_env_vars(temp_dir, box_class):
runtime = await _load_runtime(temp_dir, box_class)
await runtime.add_env_vars({'QUUX': 'abc"def', 'FOOBAR': 'xyz'})
obs: CmdOutputObservation = await runtime.run_action(
@@ -181,12 +176,9 @@ async def test_env_vars_runtime_add_multiple_env_vars(box_class):
@pytest.mark.asyncio
async def test_env_vars_runtime_add_env_vars_overwrite(box_class):
cli_session = 'main_test'
async def test_env_vars_runtime_add_env_vars_overwrite(temp_dir, box_class):
with patch.dict(os.environ, {'SANDBOX_ENV_FOOBAR': 'BAZ'}):
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
runtime = await _load_runtime(temp_dir, box_class)
await runtime.add_env_vars({'FOOBAR': 'xyz'})
obs: CmdOutputObservation = await runtime.run_action(
@@ -204,10 +196,7 @@ async def test_env_vars_runtime_add_env_vars_overwrite(box_class):
@pytest.mark.asyncio
async def test_bash_command_pexcept(temp_dir, box_class):
cli_session = 'main_test'
event_stream = EventStream(cli_session)
runtime = await _load_runtime(box_class, event_stream)
runtime = await _load_runtime(temp_dir, box_class)
# We set env var PS1="\u@\h:\w $"
# and construct the PEXCEPT prompt base on it.

31
tests/unit/test_sandbox.py
View File

@@ -11,14 +11,14 @@ from opendevin.runtime.utils import split_bash_commands
def create_docker_box_from_app_config(
path: str, config: AppConfig = None
path: str, config: AppConfig | None = None
) -> DockerSSHBox:
if config is None:
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
persist_sandbox=False,
)
),
persist_sandbox=False,
)
return DockerSSHBox(
config=config.sandbox,
@@ -305,9 +305,9 @@ def test_sandbox_jupyter_agentskills_fileop_pwd(temp_dir):
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
persist_sandbox=False,
enable_auto_lint=False,
)
),
persist_sandbox=False,
)
assert not config.sandbox.enable_auto_lint
box = create_docker_box_from_app_config(temp_dir, config)
@@ -324,9 +324,9 @@ def test_agnostic_sandbox_jupyter_agentskills_fileop_pwd(temp_dir):
sandbox=SandboxConfig(
box_type='ssh',
container_image=base_sandbox_image,
persist_sandbox=False,
enable_auto_lint=False,
)
),
persist_sandbox=False,
)
assert not config.sandbox.enable_auto_lint
box = create_docker_box_from_app_config(temp_dir, config)
@@ -337,11 +337,20 @@ def test_sandbox_jupyter_plugin_backticks(temp_dir):
config = AppConfig(
sandbox=SandboxConfig(
box_type='ssh',
persist_sandbox=False,
enable_auto_lint=False,
)
),
persist_sandbox=False,
)
box = DockerSSHBox(
config=config.sandbox,
persist_sandbox=config.persist_sandbox,
workspace_mount_path=temp_dir,
sandbox_workspace_dir=config.workspace_mount_path_in_sandbox,
cache_dir=config.cache_dir,
run_as_devin=True,
ssh_hostname=config.ssh_hostname,
ssh_password=config.ssh_password,
ssh_port=config.ssh_port,
)
box = create_docker_box_from_app_config(temp_dir, config)
box.init_plugins([JupyterRequirement])
test_code = "print('Hello, `World`!')"
expected_write_command = (