github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-09 14:57:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring: event stream based agent history (#2709)

Browse Source 
* add to event stream sync

* remove async from tests

* small logging spam fix

* remove swe agent

* arch refactoring: use history from the event stream

* refactor agents

* monologue agent

* ruff

* planner agent

* micro-agents

* refactor history in evaluations

* evals history refactoring

* adapt evals and tests

* unit testing stuck

* testing micro agents, event stream

* fix planner agent

* fix tests

* fix stuck after rename

* fix test

* small clean up

* fix merge

* fix merge issue

* fix integration tests

* Update agenthub/dummy_agent/agent.py

* fix tests

* rename more clearly; add todo; clean up
This commit is contained in:
Engel Nyst
2024-07-07 23:04:23 +02:00
committed by GitHub
parent 9dc2d2c80f
commit d37b2973b2
107 changed files with 1692 additions and 698 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
agenthub/browsing_agent/browsing_agent.py
View File

@@ -15,6 +15,7 @@ from opendevin.events.action import (
)
from opendevin.events.event import EventSource
from opendevin.events.observation import BrowserOutputObservation
from opendevin.events.observation.observation import Observation
from opendevin.llm.llm import LLM
from opendevin.runtime.plugins import (
PluginRequirement,
@@ -146,23 +147,21 @@ class BrowsingAgent(Agent):
last_obs = None
last_action = None
if EVAL_MODE and len(state.history) == 1:
if EVAL_MODE and len(state.history.get_events_as_list()) == 1:
# for webarena and miniwob++ eval, we need to retrieve the initial observation already in browser env
# initialize and retrieve the first observation by issuing an noop OP
# For non-benchmark browsing, the browser env starts with a blank page, and the agent is expected to first navigate to desired websites
return BrowseInteractiveAction(browser_actions='noop()')
for prev_action, obs in state.history:
if isinstance(prev_action, BrowseInteractiveAction):
prev_actions.append(prev_action.browser_actions)
last_obs = obs
last_action = prev_action
elif (
isinstance(prev_action, MessageAction)
and prev_action.source == EventSource.AGENT
):
# agent has responded, task finish.
return AgentFinishAction(outputs={'content': prev_action.content})
for event in state.history.get_events():
if isinstance(event, BrowseInteractiveAction):
prev_actions.append(event.browser_actions)
last_action = event
elif isinstance(event, MessageAction) and event.source == EventSource.AGENT:
# agent has responded, task finished.
return AgentFinishAction(outputs={'content': event.content})
elif isinstance(event, Observation):
last_obs = event
if EVAL_MODE:
prev_actions = prev_actions[1:] # remove the first noop action
@@ -207,7 +206,7 @@ class BrowsingAgent(Agent):
prompt = get_prompt(error_prefix, cur_axtree_txt, prev_action_str)
messages.append({'role': 'user', 'content': prompt})
logger.info(prompt)
logger.debug(prompt)
response = self.llm.completion(
messages=messages,
temperature=0.0,

57
agenthub/codeact_agent/codeact_agent.py
View File

@@ -182,27 +182,14 @@ class CodeActAgent(Agent):
- MessageAction(content) - Message action to run (e.g. ask for clarification)
- AgentFinishAction() - end the interaction
"""
messages: list[dict[str, str]] = [
{'role': 'system', 'content': self.system_message},
{'role': 'user', 'content': self.in_context_example},
]
for prev_action, obs in state.history:
action_message = get_action_message(prev_action)
if action_message:
messages.append(action_message)
# if we're done, go back
latest_user_message = state.history.get_last_user_message()
if latest_user_message and latest_user_message.strip() == '/exit':
return AgentFinishAction()
obs_message = get_observation_message(obs)
if obs_message:
messages.append(obs_message)
latest_user_message = [m for m in messages if m['role'] == 'user'][-1]
if latest_user_message:
if latest_user_message['content'].strip() == '/exit':
return AgentFinishAction()
latest_user_message['content'] += (
f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task. When finished reply with <finish></finish>.'
)
# prepare what we want to send to the LLM
messages: list[dict[str, str]] = self._get_messages(state)
response = self.llm.completion(
messages=messages,
@@ -217,3 +204,35 @@ class CodeActAgent(Agent):
def search_memory(self, query: str) -> list[str]:
raise NotImplementedError('Implement this abstract method')
def _get_messages(self, state: State) -> list[dict[str, str]]:
messages = [
{'role': 'system', 'content': self.system_message},
{'role': 'user', 'content': self.in_context_example},
]
for event in state.history.get_events():
# create a regular message from an event
message = (
get_action_message(event)
if isinstance(event, Action)
else get_observation_message(event)
)
# add regular message
if message:
messages.append(message)
# the latest user message is important:
# we want to remind the agent of the environment constraints
latest_user_message = next(
(m for m in reversed(messages) if m['role'] == 'user'), None
)
# add a reminder to the prompt
if latest_user_message:
latest_user_message['content'] += (
f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task. When finished reply with <finish></finish>'
)
return messages

57
agenthub/codeact_swe_agent/codeact_swe_agent.py
View File

@@ -138,27 +138,14 @@ class CodeActSWEAgent(Agent):
- MessageAction(content) - Message action to run (e.g. ask for clarification)
- AgentFinishAction() - end the interaction
"""
messages: list[dict[str, str]] = [
{'role': 'system', 'content': self.system_message},
{'role': 'user', 'content': self.in_context_example},
]
for prev_action, obs in state.history:
action_message = get_action_message(prev_action)
if action_message:
messages.append(action_message)
# if we're done, go back
latest_user_message = state.history.get_last_user_message()
if latest_user_message and latest_user_message.strip() == '/exit':
return AgentFinishAction()
obs_message = get_observation_message(obs)
if obs_message:
messages.append(obs_message)
latest_user_message = [m for m in messages if m['role'] == 'user'][-1]
if latest_user_message:
if latest_user_message['content'].strip() == '/exit':
return AgentFinishAction()
latest_user_message['content'] += (
f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task.'
)
# prepare what we want to send to the LLM
messages: list[dict[str, str]] = self._get_messages(state)
response = self.llm.completion(
messages=messages,
@@ -173,3 +160,35 @@ class CodeActSWEAgent(Agent):
def search_memory(self, query: str) -> list[str]:
raise NotImplementedError('Implement this abstract method')
def _get_messages(self, state: State) -> list[dict[str, str]]:
messages = [
{'role': 'system', 'content': self.system_message},
{'role': 'user', 'content': self.in_context_example},
]
for event in state.history.get_events():
# create a regular message from an event
message = (
get_action_message(event)
if isinstance(event, Action)
else get_observation_message(event)
)
# add regular message
if message:
messages.append(message)
# the latest user message is important:
# we want to remind the agent of the environment constraints
latest_user_message = next(
(m for m in reversed(messages) if m['role'] == 'user'), None
)
# add a reminder to the prompt
if latest_user_message:
latest_user_message['content'] += (
f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task.'
)
return messages

4
agenthub/delegator_agent/agent.py
View File

@@ -41,7 +41,9 @@ class DelegatorAgent(Agent):
agent='StudyRepoForTaskAgent', inputs={'task': task}
)
last_observation = state.history[-1][1]
# last observation in history should be from the delegate
last_observation = state.history.get_last_observation()
if not isinstance(last_observation, AgentDelegateObservation):
raise Exception('Last observation is not an AgentDelegateObservation')

12
agenthub/dummy_agent/agent.py
View File

@@ -125,11 +125,16 @@ class DummyAgent(Agent):
time.sleep(0.1)
if state.iteration > 0:
prev_step = self.steps[state.iteration - 1]
# a step is (action, observations list)
if 'observations' in prev_step:
# one obs, at most
expected_observations = prev_step['observations']
hist_start = len(state.history) - len(expected_observations)
# check if the history matches the expected observations
hist_events = state.history.get_last_events(len(expected_observations))
for i in range(len(expected_observations)):
hist_obs = event_to_dict(state.history[hist_start + i][1])
hist_obs = event_to_dict(hist_events[i])
expected_obs = event_to_dict(expected_observations[i])
if (
'command_id' in hist_obs['extras']
@@ -143,9 +148,6 @@ class DummyAgent(Agent):
):
del expected_obs['extras']['command_id']
expected_obs['content'] = ''
if hist_obs != expected_obs:
print('\nactual', hist_obs)
print('\nexpect', expected_obs)
assert (
hist_obs == expected_obs
), f'Expected observation {expected_obs}, got {hist_obs}'

25
agenthub/micro/agent.py
View File

@@ -7,6 +7,7 @@ from opendevin.events.action import Action
from opendevin.events.serialization.action import action_from_dict
from opendevin.events.serialization.event import event_to_memory
from opendevin.llm.llm import LLM
from opendevin.memory.history import ShortTermHistory
from .instructions import instructions
from .registry import all_microagents
@@ -27,18 +28,24 @@ def to_json(obj, **kwargs):
return json.dumps(obj, **kwargs)
def history_to_json(obj, **kwargs):
def history_to_json(history: ShortTermHistory, max_events=20, **kwargs):
"""
Serialize and simplify history to str format
"""
if isinstance(obj, list):
# process history, make it simpler.
processed_history = []
for action, observation in obj:
processed_history.append(
(event_to_memory(action), event_to_memory(observation))
)
return json.dumps(processed_history, **kwargs)
processed_history = []
event_count = 0
for event in history.get_events(reverse=True):
if event_count >= max_events:
break
processed_history.append(event_to_memory(event))
event_count += 1
# history is in reverse order, let's fix it
processed_history.reverse()
return json.dumps(processed_history, **kwargs)
class MicroAgent(Agent):

2
agenthub/micro/coder/prompt.md
View File

@@ -21,7 +21,7 @@ Do NOT finish until you have completed the tasks.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
## Format
{{ instructions.format.action }}

2
agenthub/micro/commit_writer/prompt.md
View File

@@ -20,7 +20,7 @@ action with `outputs.answer` set to the answer.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
If the last item in the history is an error, you should try to fix it.

2
agenthub/micro/manager/prompt.md
View File

@@ -27,7 +27,7 @@ you have delegated to, and why they failed).
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
If the last item in the history is an error, you should try to fix it. If you
cannot fix it, call the `reject` action.

2
agenthub/micro/math_agent/prompt.md
View File

@@ -10,7 +10,7 @@ and call the `finish` action with `outputs.answer` set to the answer.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
If the last item in the history is an error, you should try to fix it.

2
agenthub/micro/postgres_agent/prompt.md
View File

@@ -18,7 +18,7 @@ You may take any of the following actions:
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
## Format
{{ instructions.format.action }}

2
agenthub/micro/repo_explorer/prompt.md
View File

@@ -20,7 +20,7 @@ When you're done, put your summary into the output of the `finish` action.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
## Format
{{ instructions.format.action }}

2
agenthub/micro/study_repo_for_task/prompt.md
View File

@@ -24,7 +24,7 @@ implement the solution. If the codebase is empty, you should call the `finish` a
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
## Format
{{ instructions.format.action }}

2
agenthub/micro/typo_fixer_agent/prompt.md
View File

@@ -31,7 +31,7 @@ Do NOT finish until you have fixed all the typos and generated a summary.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-5:]) }}
{{ history_to_json(state.history, max_events=10) }}
## Format
{{ instructions.format.action }}

2
agenthub/micro/verifier/prompt.md
View File

@@ -22,7 +22,7 @@ explaining what the problem is.
## History
{{ instructions.history_truncated }}
{{ history_to_json(state.history[-10:]) }}
{{ history_to_json(state.history, max_events=20) }}
## Format
{{ instructions.format.action }}

29
agenthub/monologue_agent/agent.py
View File

@@ -32,9 +32,6 @@ from opendevin.runtime.tools import RuntimeTool
if config.agent.memory_enabled:
from opendevin.memory.memory import LongTermMemory
MAX_TOKEN_COUNT_PADDING = 512
MAX_OUTPUT_LENGTH = 5000
class MonologueAgent(Agent):
VERSION = '1.0'
@@ -68,7 +65,7 @@ class MonologueAgent(Agent):
Will execute again when called after reset.
Parameters:
- task (str): The initial goal statement provided by the user
- task: The initial goal statement provided by the user
Raises:
- AgentNoInstructionError: If task is not provided
@@ -155,16 +152,20 @@ class MonologueAgent(Agent):
recent_events: list[dict[str, str]] = []
# add the events from state.history
for prev_action, obs in state.history:
if not isinstance(prev_action, NullAction):
recent_events.append(event_to_memory(prev_action))
if not isinstance(obs, NullObservation):
recent_events.append(event_to_memory(obs))
for event in state.history.get_events():
recent_events.append(event_to_memory(event))
# add the last messages to long term memory
if self.memory is not None and state.history and len(state.history) > 0:
self.memory.add_event(event_to_memory(state.history[-1][0]))
self.memory.add_event(event_to_memory(state.history[-1][1]))
if self.memory is not None:
last_action = state.history.get_last_action()
last_observation = state.history.get_last_observation()
# this should still work
# we will need to do this differently: find out if there really is an action or an observation in this step
if last_action:
self.memory.add_event(event_to_memory(last_action))
if last_observation:
self.memory.add_event(event_to_memory(last_observation))
# the action prompt with initial thoughts and recent events
prompt = prompts.get_request_action_prompt(
@@ -188,10 +189,10 @@ class MonologueAgent(Agent):
Uses search to produce top 10 results.
Parameters:
- query (str): The query that we want to find related memories for
- The query that we want to find related memories for
Returns:
- list[str]: A list of top 10 text results that matched the query
- A list of top 10 text results that matched the query
"""
if self.memory is None:
return []

37
agenthub/planner_agent/prompt.py
View File

@@ -6,13 +6,10 @@ from opendevin.events.action import (
Action,
NullAction,
)
from opendevin.events.observation import (
NullObservation,
)
from opendevin.events.serialization.action import action_from_dict
from opendevin.events.serialization.event import event_to_memory
HISTORY_SIZE = 10
HISTORY_SIZE = 20
prompt = """
# Task
@@ -132,18 +129,28 @@ def get_prompt(state: State) -> str:
- str: The formatted string prompt with historical values
"""
# the plan
plan_str = json.dumps(state.root_task.to_dict(), indent=2)
sub_history = state.history[-HISTORY_SIZE:]
# the history
history_dicts = []
latest_action: Action = NullAction()
for action, observation in sub_history:
if not isinstance(action, NullAction):
history_dicts.append(event_to_memory(action))
latest_action = action
if not isinstance(observation, NullObservation):
observation_dict = event_to_memory(observation)
history_dicts.append(observation_dict)
# retrieve the latest HISTORY_SIZE events
for event_count, event in enumerate(state.history.get_events(reverse=True)):
if event_count >= HISTORY_SIZE:
break
if latest_action == NullAction() and isinstance(event, Action):
latest_action = event
history_dicts.append(event_to_memory(event))
# history_dicts is in reverse order, lets fix it
history_dicts.reverse()
# and get it as a JSON string
history_str = json.dumps(history_dicts, indent=2)
# the plan status
current_task = state.root_task.get_current_task()
if current_task is not None:
plan_status = f"You're currently working on this task:\n{current_task.goal}."
@@ -151,9 +158,15 @@ def get_prompt(state: State) -> str:
plan_status += "\nIf it's not achievable AND verifiable with a SINGLE action, you MUST break it down into subtasks NOW."
else:
plan_status = "You're not currently working on any tasks. Your next action MUST be to mark a task as in_progress."
# the hint, based on the last action
hint = get_hint(event_to_memory(latest_action).get('action', ''))
logger.info('HINT:\n' + hint, extra={'msg_type': 'DETAIL'})
# the last relevant user message (the task)
task = state.get_current_user_intent()
# finally, fill in the prompt
return prompt % {
'task': task,
'plan': plan_str,

25
evaluation/EDA/run_infer.py
View File

@@ -25,7 +25,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
game = None
@@ -42,11 +41,14 @@ def cleanup():
def codeact_user_response_eda(state: State) -> str:
global game
model_guess = ''
# retrieve the latest model message from history
if state.history:
for act, _ in reversed(state.history):
if isinstance(act, MessageAction) and act.source == 'agent':
model_guess = act.content
for event in state.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'agent':
model_guess = event.content
break
assert game is not None, 'Game is not initialized.'
msg = game.generate_user_response(model_guess)
game.curr_turn += 1
@@ -149,24 +151,27 @@ def process_instance(
raise ValueError('State should not be None.')
final_message = ''
for act, _ in reversed(state.history):
if isinstance(act, MessageAction) and act.source == 'agent':
final_message = act.content
for event in state.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'agent':
final_message = event.content
break
logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
test_result = game.reward()
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': instance['text'].strip(),
'instance': instance,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': {

9
evaluation/TUTORIAL.md
View File

@@ -100,13 +100,14 @@ def codeact_user_response(state: State) -> str:
'If you think you have modified the code in a way that fixes the issue, please run the following command: <execute_bash> exit </execute_bash>.\n'
'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
)
# check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
if state.history:
user_msgs = [
action
for action, _ in state.history
if isinstance(action, MessageAction) and action.source == 'agent'
event
for event in state.history.get_events()
if isinstance(action, MessageAction) and action.source == 'user'
]
if len(user_msgs) >= 2:
if len(user_msgs) > 2:
# let the agent know that it can give up when it has tried 3 times
return (
msg

25
evaluation/agent_bench/run_infer.py
View File

@@ -27,7 +27,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import CmdRunAction, MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from opendevin.runtime.docker.ssh_box import DockerSSHBox
@@ -145,13 +144,15 @@ def process_instance(
else:
logger.info('Retrieving agent answer from history.')
raw_ans = ''
for act, _ in reversed(state.history):
if isinstance(act, MessageAction) and act.source == 'agent':
raw_ans = act.content
break
if isinstance(act, CmdRunAction) and act.source == 'agent':
raw_ans = act.thought
break
# retrieve the last agent message or thought
for event in state.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'agent':
raw_ans = event.content
elif isinstance(event, CmdRunAction) and event.source == 'agent':
raw_ans = event.thought
# parse the answer for a solution tag
agent_answer = re.findall(r'<solution>(.*?)</solution>', raw_ans)
if len(agent_answer) == 0:
logger.warning(f'Failed to parse model answer: {raw_ans}')
@@ -179,9 +180,11 @@ def process_instance(
)
test_result = compare_results(comparison_method, agent_answer, final_ans)
histories = [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
]
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
metrics = state.metrics.get() if state.metrics else None
# Save the output

10
evaluation/biocoder/run_infer.py
View File

@@ -24,7 +24,6 @@ from opendevin.core.config import config, get_llm_config_arg, parse_arguments
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
@@ -196,6 +195,11 @@ def process_instance(
raise ValueError('State should not be None.')
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'test_case_id': instance.test_case_id,
@@ -203,9 +207,7 @@ def process_instance(
'instruction': instruction,
'generated': test_result['metadata']['1_copy_change_code'],
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

1
evaluation/bird/README.md
View File

@@ -143,7 +143,6 @@ For each problem, OpenDevin is given a set number of iterations to fix the faili
"action": "run",
"args": {
"command": "python3 0.py",
"background": false,
"thought": "The Python code with the SQL query has been written to the file `0.py`. Now, let's run the Python script to execute the SQL query and get the result."
}
},

19
evaluation/bird/run_infer.py
View File

@@ -27,7 +27,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
@@ -46,12 +45,13 @@ def codeact_user_response(state: State) -> str:
'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
)
if state.history:
# check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
user_msgs = [
action
for action, _ in state.history
if isinstance(action, MessageAction) and action.source == 'user'
event
for event in state.history.get_events()
if isinstance(event, MessageAction) and event.source == 'user'
]
if len(user_msgs) >= 2:
if len(user_msgs) > 2:
# let the agent know that it can give up when it has tried 3 times
return (
msg
@@ -245,14 +245,17 @@ def process_instance(
raise ValueError('State should not be None.')
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'task_id': instance.task_id,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

25
evaluation/gaia/run_infer.py
View File

@@ -26,7 +26,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import CmdRunAction, MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
DATASET_CACHE_DIR = '~/.cache/open-devin/evals/gaia'
@@ -136,13 +135,13 @@ def process_instance(
raise ValueError('State should not be None.')
model_answer_raw = ''
for act, _ in reversed(state.history):
if isinstance(act, CmdRunAction) and act.source == 'agent':
model_answer_raw = act.thought
break
elif isinstance(act, MessageAction) and act.source == 'agent':
model_answer_raw = act.content
break
# get the last message or thought from the agent
for event in state.history.get_events(reverse=True):
if isinstance(event, CmdRunAction) and event.source == 'agent':
model_answer_raw = event.thought
elif isinstance(event, MessageAction) and event.source == 'agent':
model_answer_raw = event.content
# attempt to parse model_answer
model_answer = re.findall(r'<solution>(.*?)</solution>', model_answer_raw)
@@ -166,16 +165,18 @@ def process_instance(
}
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': instance['task_id'],
'instance': instance,
'instruction': instance['Question'],
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

32
evaluation/gorilla/run_infer.py
View File

@@ -17,7 +17,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from .utils import encode_question, get_data
@@ -37,13 +36,15 @@ def codeact_user_response(state: State) -> str:
'Please run the following command: <execute_bash> exit </execute_bash>.\n'
#'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
)
# check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
if state.history:
user_msgs = [
action
for action, _ in state.history
if isinstance(action, MessageAction) and action.source == 'user'
event
for event in state.history.get_events()
if isinstance(event, MessageAction) and event.source == 'user'
]
if len(user_msgs) >= 2:
if len(user_msgs) > 2:
# let the agent know that it can give up when it has tried 3 times
return (
msg
@@ -131,10 +132,12 @@ def process_instance(agent, question_id, question, metadata, reset_logger: bool
raise ValueError('State should not be None.')
model_answer_raw = ''
for act, _ in reversed(state.history):
if isinstance(act, MessageAction) and act.source == 'agent':
model_answer_raw = act.content
break
# retrieve the last message from the agent
for event in state.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'agent':
model_answer_raw = event
# attempt to parse model_answer
_, _, ast_eval = get_data(metadata['hub'])
correct, hallucination = ast_eval(question_id, model_answer_raw)
@@ -142,6 +145,12 @@ def process_instance(agent, question_id, question, metadata, reset_logger: bool
logger.info(
f'Final message: {model_answer_raw} | Correctness: {correct} | Hallucination: {hallucination}'
)
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'question_id': question_id,
@@ -151,10 +160,7 @@ def process_instance(agent, question_id, question, metadata, reset_logger: bool
'answer_id': 'None',
'model_id': metadata['model_name'],
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
}

13
evaluation/gpqa/run_infer.py
View File

@@ -42,7 +42,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
@@ -214,7 +213,7 @@ def process_instance(
final_message = next(
(
act.content
for act in reversed(state.history)
for act in state.history.get_events(reverse=True)
if isinstance(act, MessageAction)
),
None,
@@ -231,16 +230,18 @@ def process_instance(
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'task_id': instance.task_id,
'instance_id': instance.instance_id,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

11
evaluation/humanevalfix/run_infer.py
View File

@@ -32,7 +32,6 @@ from opendevin.core.config import config, get_llm_config_arg, parse_arguments
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
IMPORT_HELPER = {
@@ -202,15 +201,17 @@ def process_instance(
raise ValueError('State should not be None.')
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'task_id': instance.task_id,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

23
evaluation/logic_reasoning/run_infer.py
View File

@@ -22,7 +22,6 @@ from opendevin.core.config import config, get_llm_config_arg, get_parser
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
@@ -199,12 +198,12 @@ def process_instance(
final_message = ''
messages = []
for action, obs in reversed(state.history):
# if isinstance(act, MessageAction):
messages.append(obs.content)
# print("obs.content:", obs.content)
if str(obs.content) in ["'A'", "'B'", "'C'"]:
final_message = obs.content
for event in state.history.get_events(reverse=True):
# will this be a MessageAction?
# TODO we can filter for types of events if we know what to expect
messages.append(event.content)
if str(event.content) in ["'A'", "'B'", "'C'"]:
final_message = event.content
break
final_message = final_message.strip("'")
@@ -217,16 +216,18 @@ def process_instance(
)
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'id': instance['id'],
'instance': instance,
'instruction': instruction,
# 'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'final_message': final_message,
'messages': messages,

10
evaluation/miniwob/run_infer.py
View File

@@ -19,7 +19,6 @@ from opendevin.core.config import config, get_llm_config_arg, parse_arguments
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from opendevin.runtime.docker.ssh_box import DockerSSHBox
from opendevin.runtime.tools import RuntimeTool
@@ -110,14 +109,17 @@ def process_instance(
rewards = json.load(f)
reward = max(rewards)
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': env_id,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': reward,

1
evaluation/mint/env.py
View File

@@ -99,7 +99,6 @@ class SimplifiedEnv:
return
content = output.to_str()
# self.state.history.append({"role": "user", "content": content})
self.task_state.latest_output = output.to_dict()
self.task_state.latest_output['content'] = content

12
evaluation/mint/run_infer.py
View File

@@ -21,7 +21,6 @@ from opendevin.core.config import config, get_llm_config_arg, get_parser
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from .datatypes import TaskState
@@ -39,7 +38,7 @@ def codeact_user_response_mint(state: State, task: Task, task_config: Dict[str,
task=task,
task_config=task_config,
)
last_action, _ = state.history[-1]
last_action = state.history.get_last_action()
result_state: TaskState = env.step(last_action.message or '')
state.task_state = result_state
@@ -162,15 +161,18 @@ def process_instance(
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'id': instance.task_id,
'instance': instance.to_dict(),
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': task_state.success if task_state else False,

11
evaluation/ml_bench/run_infer.py
View File

@@ -36,7 +36,6 @@ from opendevin.core.config import config, get_llm_config_arg, get_parser
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from opendevin.runtime.docker.ssh_box import DockerSSHBox
@@ -195,16 +194,18 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
logger.info(f'Output: {eval_output}')
metrics['success'] = 1
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': instance['id'],
'repo': repo_url,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs))
for action, obs in state.history
],
'history': histories,
'eval_script': eval_script_content,
'eval_exit_code': exit_code,
'eval_output': eval_output,

10
evaluation/swe_bench/run_infer.py
View File

@@ -25,7 +25,6 @@ from opendevin.core.config import config, get_llm_config_arg, parse_arguments
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false') == 'true'
@@ -310,6 +309,11 @@ IMPORTANT TIPS:
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': instance.instance_id,
@@ -317,9 +321,7 @@ IMPORTANT TIPS:
'instruction': instruction,
'git_patch': git_patch, # SWE Bench specific
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': test_result,

23
evaluation/toolqa/run_infer.py
View File

@@ -21,7 +21,6 @@ from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.action import MessageAction
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from .utils import download_data, download_tools, encode_question, eval_answer, get_data
@@ -97,14 +96,24 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
raise ValueError('State should not be None.')
model_answer_raw = ''
for act, _ in reversed(state.history):
if isinstance(act, MessageAction) and act.source == 'agent':
model_answer_raw = act.content
# retrieve the last message from the agent
for event in state.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'agent':
model_answer_raw = event.content
break
# attempt to parse model_answer
correct = eval_answer(str(model_answer_raw), str(answer))
metrics = state.metrics.get() if state.metrics else None
logger.info(f'Final message: {model_answer_raw} | Correctness: {correct}')
metrics = state.metrics.get() if state.metrics else None
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'qid': qid,
@@ -113,9 +122,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
'answer_id': 'None',
'model_id': metadata.model_name,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
}

12
evaluation/utils/shared.py
View File

@@ -49,16 +49,20 @@ def codeact_user_response(
f'{encaps_str}'
'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
)
if state.history:
# check if the last action has an answer, if so, early exit
if try_parse is not None:
last_action, _ = state.history[-1]
last_action = state.history.get_last_action()
ans = try_parse(last_action)
if ans is not None:
return '/exit'
# check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
user_msgs = [
action
for action, _ in state.history
if isinstance(action, MessageAction) and action.source == 'user'
event
for event in state.history.get_events()
if isinstance(event, MessageAction) and event.source == 'user'
]
if len(user_msgs) >= 2:
# let the agent know that it can give up when it has tried 3 times

10
evaluation/webarena/run_infer.py
View File

@@ -19,7 +19,6 @@ from opendevin.core.config import config, get_llm_config_arg, parse_arguments
from opendevin.core.logger import get_console_handler
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.main import run_agent_controller
from opendevin.events.serialization.event import event_to_dict
from opendevin.llm.llm import LLM
from opendevin.runtime.docker.ssh_box import DockerSSHBox
from opendevin.runtime.tools import RuntimeTool
@@ -111,14 +110,17 @@ def process_instance(
rewards = json.load(f)
reward = max(rewards)
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
# for compatibility with the existing output format, we can remake the pairs here
# remove when it becomes unnecessary
histories = state.history.compatibility_for_eval_history_pairs()
# Save the output
output = {
'instance_id': env_id,
'instruction': instruction,
'metadata': metadata.model_dump(),
'history': [
(event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
],
'history': histories,
'metrics': metrics,
'error': state.last_error if state and state.last_error else None,
'test_result': reward,

1
opendevin/controller/agent.py
View File

@@ -70,6 +70,7 @@ class Agent(ABC):
to prepare the agent for restarting the instruction or cleaning up before destruction.
"""
# TODO clear history
self._complete = False
@property

126
opendevin/controller/agent_controller.py
View File

@@ -4,6 +4,7 @@ from typing import Optional, Type
from opendevin.controller.agent import Agent
from opendevin.controller.state.state import State, TrafficControlState
from opendevin.controller.stuck import StuckDetector
from opendevin.core.config import config
from opendevin.core.exceptions import (
LLMMalformedActionError,
@@ -94,6 +95,10 @@ class AgentController:
)
self.max_budget_per_task = max_budget_per_task
# stuck helper
self._stuck_detector = StuckDetector(self.state)
if not is_delegate:
self.agent_task = asyncio.create_task(self._start_step_loop())
@@ -114,9 +119,9 @@ class AgentController:
"""
This error will be reported to the user and sent to the LLM next step, in the hope it can self-correct.
This method should be called for a particular type of errors:
- the string message should be user-friendly, it will be shown in the UI
- an ErrorObservation can be sent to the LLM by the agent, with the exception message, so it can self-correct next time
This method should be called for a particular type of errors, which have:
- a user-friendly message, which will be shown in the chat box. This should not be a raw exception message.
- an ErrorObservation that can be sent to the LLM by the agent, with the exception message, so it can self-correct next time.
"""
self.state.last_error = message
if exception:
@@ -126,7 +131,9 @@ class AgentController:
async def add_history(self, action: Action, observation: Observation):
if isinstance(action, NullAction) and isinstance(observation, NullObservation):
return
self.state.history.append((action, observation))
logger.debug(
f'Adding history ({type(action).__name__} with id={action.id}, {type(observation).__name__} with id={observation.id})'
)
async def _start_step_loop(self):
logger.info(f'[Agent Controller {self.id}] Starting step loop...')
@@ -180,12 +187,14 @@ class AgentController:
logger.info(event, extra={'msg_type': 'OBSERVATION'})
elif isinstance(event, AgentDelegateObservation):
await self.add_history(NullAction(), event)
self.state.history.on_event(event)
logger.info(event, extra={'msg_type': 'OBSERVATION'})
elif isinstance(event, ErrorObservation):
await self.add_history(NullAction(), event)
logger.info(event, extra={'msg_type': 'OBSERVATION'})
def reset_task(self):
self.almost_stuck = 0
self.agent.reset()
async def set_agent_state_to(self, new_state: AgentState):
@@ -244,7 +253,6 @@ class AgentController:
await self.delegate.set_agent_state_to(AgentState.RUNNING)
async def _step(self):
logger.debug(f'[Agent Controller {self.id}] Entering step method')
if self.get_agent_state() != AgentState.RUNNING:
await asyncio.sleep(1)
return
@@ -347,12 +355,13 @@ class AgentController:
if action.runnable:
self._pending_action = action
else:
await self.add_history(action, NullObservation(''))
if not isinstance(action, NullAction):
self.event_stream.add_event(action, EventSource.AGENT)
if not action.runnable:
await self.add_history(action, NullObservation(''))
await self.update_state_after_step()
logger.info(action, extra={'msg_type': 'ACTION'})
@@ -373,83 +382,32 @@ class AgentController:
else:
self.state = state
# when restored from a previous session, the State object will have history, start_id, and end_id
# connect it to the event stream
self.state.history.set_event_stream(self.event_stream)
# if start_id was not set in State, we're starting fresh, at the top of the stream
start_id = self.state.start_id
if start_id == -1:
start_id = self.event_stream.get_latest_event_id() + 1
else:
logger.debug(f'AgentController {self.id} restoring from event {start_id}')
# make sure history is in sync
self.state.start_id = start_id
self.state.history.start_id = start_id
# if there was an end_id saved in State, set it in history
# currently not used, later useful for delegates
if self.state.end_id > -1:
self.state.history.end_id = self.state.end_id
def _is_stuck(self):
# check if delegate stuck
if self.delegate and self.delegate._is_stuck():
return True
# filter out MessageAction with source='user' from history
filtered_history = [
_tuple
for _tuple in self.state.history
if not (
isinstance(_tuple[0], MessageAction)
and _tuple[0].source == EventSource.USER
)
]
if len(filtered_history) < 3:
return False
# FIXME rewrite this to be more readable
# Scenario 1: the same (Action, Observation) loop
# 3 pairs of (action, observation) to stop the agent
last_three_tuples = filtered_history[-3:]
if all(
# (Action, Observation) tuples
# compare the last action to the last three actions
self._eq_no_pid(last_three_tuples[-1][0], _tuple[0])
for _tuple in last_three_tuples
) and all(
# compare the last observation to the last three observations
self._eq_no_pid(last_three_tuples[-1][1], _tuple[1])
for _tuple in last_three_tuples
):
logger.warning('Action, Observation loop detected')
return True
if len(filtered_history) < 4:
return False
last_four_tuples = filtered_history[-4:]
# Scenario 2: (action, error) pattern, not necessary identical error
# 4 pairs of (action, error) to stop the agent
if all(
self._eq_no_pid(last_four_tuples[-1][0], _tuple[0])
for _tuple in last_four_tuples
):
# It repeats the same action, give it a chance, but not if:
if all(
isinstance(_tuple[1], ErrorObservation) for _tuple in last_four_tuples
):
logger.warning('Action, ErrorObservation loop detected')
return True
# check if the agent repeats the same (Action, Observation)
# every other step in the last six tuples
# step1 = step3 = step5
# step2 = step4 = step6
if len(filtered_history) >= 6:
last_six_tuples = filtered_history[-6:]
if (
# this pattern is every other step, like:
# (action_1, obs_1), (action_2, obs_2), (action_1, obs_1), (action_2, obs_2),...
self._eq_no_pid(last_six_tuples[-1][0], last_six_tuples[-3][0])
and self._eq_no_pid(last_six_tuples[-1][0], last_six_tuples[-5][0])
and self._eq_no_pid(last_six_tuples[-2][0], last_six_tuples[-4][0])
and self._eq_no_pid(last_six_tuples[-2][0], last_six_tuples[-6][0])
and self._eq_no_pid(last_six_tuples[-1][1], last_six_tuples[-3][1])
and self._eq_no_pid(last_six_tuples[-1][1], last_six_tuples[-5][1])
and self._eq_no_pid(last_six_tuples[-2][1], last_six_tuples[-4][1])
and self._eq_no_pid(last_six_tuples[-2][1], last_six_tuples[-6][1])
):
logger.warning('Action, Observation pattern detected')
return True
return False
return self._stuck_detector.is_stuck()
def __repr__(self):
return (
@@ -458,13 +416,3 @@ class AgentController:
f'state={self.state!r}, agent_task={self.agent_task!r}, '
f'delegate={self.delegate!r}, _pending_action={self._pending_action!r})'
)
def _eq_no_pid(self, obj1, obj2):
if isinstance(obj1, CmdOutputObservation) and isinstance(
obj2, CmdOutputObservation
):
# for loop detection, ignore command_id, which is the pid
return obj1.command == obj2.command and obj1.exit_code == obj2.exit_code
else:
# this is the default comparison
return obj1 == obj2

57
opendevin/controller/state/state.py
View File

@@ -8,12 +8,10 @@ from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.metrics import Metrics
from opendevin.core.schema import AgentState
from opendevin.events.action import (
Action,
MessageAction,
)
from opendevin.events.observation import (
Observation,
)
from opendevin.events.action.agent import AgentFinishAction
from opendevin.memory.history import ShortTermHistory
from opendevin.storage import get_file_store
@@ -41,7 +39,7 @@ class State:
root_task: RootTask = field(default_factory=RootTask)
iteration: int = 0
max_iterations: int = 100
history: list[tuple[Action, Observation]] = field(default_factory=list)
history: ShortTermHistory = field(default_factory=ShortTermHistory)
inputs: dict = field(default_factory=dict)
outputs: dict = field(default_factory=dict)
last_error: str | None = None
@@ -51,10 +49,15 @@ class State:
metrics: Metrics = Metrics()
# root agent has level 0, and every delegate increases the level by one
delegate_level: int = 0
# start_id and end_id track the range of events in history
start_id: int = -1
end_id: int = -1
almost_stuck: int = 0
def save_to_session(self, sid: str):
fs = get_file_store()
pickled = pickle.dumps(self)
logger.debug(f'Saving state to session {sid}:{self.agent_state}')
encoded = base64.b64encode(pickled).decode('utf-8')
try:
fs.write(f'sessions/{sid}/agent_state.pkl', encoded)
@@ -79,10 +82,42 @@ class State:
state.agent_state = AgentState.LOADING
return state
def __getstate__(self):
state = self.__dict__.copy()
# save the relevant data from recent history
# so that we can restore it when the state is restored
if 'history' in state:
state['start_id'] = state['history'].start_id
state['end_id'] = state['history'].end_id
# don't save history object itself
state.pop('history', None)
return state
def __setstate__(self, state):
self.__dict__.update(state)
# recreate the history object
if not hasattr(self, 'history'):
self.history = ShortTermHistory()
# restore the relevant data in history from the state
self.history.start_id = self.start_id
self.history.end_id = self.end_id
# remove the restored data from the state if any
def get_current_user_intent(self):
# TODO: this is used to understand the user's main goal, but it's possible
# the latest message is an interruption. We should look for a space where
# the agent goes to FINISHED, and then look for the next user message.
for action, obs in reversed(self.history):
if isinstance(action, MessageAction) and action.source == 'user':
return action.content
"""
Returns the latest user message that appears after a FinishAction, or the first (the task) if nothing was finished yet.
"""
last_user_message = None
for event in self.history.get_events(reverse=True):
if isinstance(event, MessageAction) and event.source == 'user':
last_user_message = event.content
elif isinstance(event, AgentFinishAction):
if last_user_message is not None:
return last_user_message
return last_user_message

237
opendevin/controller/stuck.py Normal file
View File

@@ -0,0 +1,237 @@
from typing import cast
from opendevin.controller.state.state import State
from opendevin.core.logger import opendevin_logger as logger
from opendevin.events.action.action import Action
from opendevin.events.action.empty import NullAction
from opendevin.events.action.message import MessageAction
from opendevin.events.event import Event, EventSource
from opendevin.events.observation.commands import (
CmdOutputObservation,
IPythonRunCellObservation,
)
from opendevin.events.observation.empty import NullObservation
from opendevin.events.observation.error import ErrorObservation
from opendevin.events.observation.observation import Observation
class StuckDetector:
def __init__(self, state: State):
self.state = state
def is_stuck(self):
# filter out MessageAction with source='user' from history
filtered_history = [
event
for event in self.state.history.get_events()
if not (
(isinstance(event, MessageAction) and event.source == EventSource.USER)
or
# there might be some NullAction or NullObservation in the history at least for now
isinstance(event, NullAction)
or isinstance(event, NullObservation)
)
]
# it takes 3 actions minimum to detect a loop, otherwise nothing to do here
if len(filtered_history) < 3:
return False
# the first few scenarios detect 3 or 4 repeated steps
# prepare the last 4 actions and observations, to check them out
last_actions: list[Event] = []
last_observations: list[Event] = []
# retrieve the last four actions and observations starting from the end of history, wherever they are
for event in reversed(filtered_history):
if isinstance(event, Action) and len(last_actions) < 4:
last_actions.append(event)
elif isinstance(event, Observation) and len(last_observations) < 4:
last_observations.append(event)
if len(last_actions) == 4 and len(last_observations) == 4:
break
# scenario 1: same action, same observation
if self._is_stuck_repeating_action_observation(last_actions, last_observations):
return True
# scenario 2: same action, errors
if self._is_stuck_repeating_action_error(last_actions, last_observations):
return True
# scenario 3: monologue
if self._is_stuck_monologue(filtered_history):
return True
# scenario 4: action, observation pattern on the last six steps
if len(filtered_history) < 6:
return False
if self._is_stuck_action_observation_pattern(filtered_history):
return True
return False
def _is_stuck_repeating_action_observation(self, last_actions, last_observations):
# scenario 1: same action, same observation
# it takes 4 actions and 4 observations to detect a loop
# assert len(last_actions) == 4 and len(last_observations) == 4
# reset almost_stuck reminder
self.state.almost_stuck = 0
# almost stuck? if two actions, obs are the same, we're almost stuck
if len(last_actions) >= 2 and len(last_observations) >= 2:
actions_equal = all(
self._eq_no_pid(last_actions[0], action) for action in last_actions[:2]
)
observations_equal = all(
self._eq_no_pid(last_observations[0], observation)
for observation in last_observations[:2]
)
# the last two actions and obs are the same?
if actions_equal and observations_equal:
self.state.almost_stuck = 2
# the last three actions and observations are the same?
if len(last_actions) >= 3 and len(last_observations) >= 3:
if (
actions_equal
and observations_equal
and self._eq_no_pid(last_actions[0], last_actions[2])
and self._eq_no_pid(last_observations[0], last_observations[2])
):
self.state.almost_stuck = 1
if len(last_actions) == 4 and len(last_observations) == 4:
if (
actions_equal
and observations_equal
and self._eq_no_pid(last_actions[0], last_actions[3])
and self._eq_no_pid(last_observations[0], last_observations[3])
):
logger.warning('Action, Observation loop detected')
self.state.almost_stuck = 0
return True
return False
def _is_stuck_repeating_action_error(self, last_actions, last_observations):
# scenario 2: same action, errors
# it takes 4 actions and 4 observations to detect a loop
# check if the last four actions are the same and result in errors
# are the last four actions the same?
if len(last_actions) == 4 and all(
self._eq_no_pid(last_actions[0], action) for action in last_actions
):
# and the last four observations all errors?
if all(isinstance(obs, ErrorObservation) for obs in last_observations):
logger.warning('Action, ErrorObservation loop detected')
return True
# or, are the last four observations all IPythonRunCellObservation with SyntaxError?
elif all(
isinstance(obs, IPythonRunCellObservation) for obs in last_observations
) and all(
cast(IPythonRunCellObservation, obs)
.content[-100:]
.find('SyntaxError: unterminated string literal (detected at line')
!= -1
and len(
cast(IPythonRunCellObservation, obs).content.split(
'SyntaxError: unterminated string literal (detected at line'
)[-1]
)
< 10
for obs in last_observations
):
logger.warning('Action, IPythonRunCellObservation loop detected')
return True
return False
def _is_stuck_monologue(self, filtered_history):
# scenario 3: monologue
# check for repeated MessageActions with source=AGENT
# see if the agent is engaged in a good old monologue, telling itself the same thing over and over
agent_message_actions = [
(i, event)
for i, event in enumerate(filtered_history)
if isinstance(event, MessageAction) and event.source == EventSource.AGENT
]
# last three message actions will do for this check
if len(agent_message_actions) >= 3:
last_agent_message_actions = agent_message_actions[-3:]
if all(
(last_agent_message_actions[0][1] == action[1])
for action in last_agent_message_actions
):
# check if there are any observations between the repeated MessageActions
# then it's not yet a loop, maybe it can recover
start_index = last_agent_message_actions[0][0]
end_index = last_agent_message_actions[-1][0]
has_observation_between = False
for event in filtered_history[start_index + 1 : end_index]:
if isinstance(event, Observation):
has_observation_between = True
break
if not has_observation_between:
logger.warning('Repeated MessageAction with source=AGENT detected')
return True
return False
def _is_stuck_action_observation_pattern(self, filtered_history):
# scenario 4: action, observation pattern on the last six steps
# check if the agent repeats the same (Action, Observation)
# every other step in the last six steps
last_six_actions: list[Event] = []
last_six_observations: list[Event] = []
# the end of history is most interesting
for event in reversed(filtered_history):
if isinstance(event, Action) and len(last_six_actions) < 6:
last_six_actions.append(event)
elif isinstance(event, Observation) and len(last_six_observations) < 6:
last_six_observations.append(event)
if len(last_six_actions) == 6 and len(last_six_observations) == 6:
break
# this pattern is every other step, like:
# (action_1, obs_1), (action_2, obs_2), (action_1, obs_1), (action_2, obs_2),...
if len(last_six_actions) == 6 and len(last_six_observations) == 6:
actions_equal = (
# action_0 == action_2 == action_4
self._eq_no_pid(last_six_actions[0], last_six_actions[2])
and self._eq_no_pid(last_six_actions[0], last_six_actions[4])
# action_1 == action_3 == action_5
and self._eq_no_pid(last_six_actions[1], last_six_actions[3])
and self._eq_no_pid(last_six_actions[1], last_six_actions[5])
)
observations_equal = (
# obs_0 == obs_2 == obs_4
self._eq_no_pid(last_six_observations[0], last_six_observations[2])
and self._eq_no_pid(last_six_observations[0], last_six_observations[4])
# obs_1 == obs_3 == obs_5
and self._eq_no_pid(last_six_observations[1], last_six_observations[3])
and self._eq_no_pid(last_six_observations[1], last_six_observations[5])
)
if actions_equal and observations_equal:
logger.warning('Action, Observation pattern detected')
return True
return False
def _eq_no_pid(self, obj1, obj2):
if isinstance(obj1, CmdOutputObservation) and isinstance(
obj2, CmdOutputObservation
):
# for loop detection, ignore command_id, which is the pid
return obj1.command == obj2.command and obj1.exit_code == obj2.exit_code
else:
# this is the default comparison
return obj1 == obj2

3
opendevin/core/logger.py
View File

@@ -164,10 +164,9 @@ def log_uncaught_exceptions(ex_cls, ex, tb):
sys.excepthook = log_uncaught_exceptions
opendevin_logger = logging.getLogger('opendevin')
opendevin_logger.setLevel(logging.INFO)
if config.debug:
opendevin_logger.setLevel(logging.DEBUG)
else:
opendevin_logger.setLevel(logging.INFO)
opendevin_logger.addHandler(get_file_handler())
opendevin_logger.addHandler(get_console_handler())
opendevin_logger.addFilter(SensitiveDataFilter(opendevin_logger.name))

2
opendevin/events/event.py
View File

@@ -17,7 +17,7 @@ class Event:
return ''
@property
def id(self) -> int | None:
def id(self) -> int:
if hasattr(self, '_id'):
return self._id # type: ignore[attr-defined]
return -1

72
opendevin/events/stream.py
View File

@@ -41,8 +41,11 @@ class EventStream:
try:
events = self._file_store.list(f'sessions/{self.sid}/events')
except FileNotFoundError:
logger.warning(f'No events found for session {self.sid}')
logger.debug(f'No events found for session {self.sid}')
self._cur_id = 0
return
# if we have events, we need to find the highest id to prepare for new events
for event_str in events:
id = self._get_id_from_filename(event_str)
if id >= self._cur_id:
@@ -59,17 +62,41 @@ class EventStream:
logger.warning(f'get id from filename ({filename}) failed.')
return -1
def get_events(self, start_id=0, end_id=None) -> Iterable[Event]:
event_id = start_id
while True:
if end_id is not None and event_id > end_id:
break
try:
event = self.get_event(event_id)
except FileNotFoundError:
break
yield event
event_id += 1
def get_events(
self,
start_id=0,
end_id=None,
reverse=False,
filter_out_type: tuple[type[Event], ...] | None = None,
) -> Iterable[Event]:
if reverse:
if end_id is None:
end_id = self._cur_id - 1
event_id = end_id
while event_id >= start_id:
try:
event = self.get_event(event_id)
if filter_out_type is None or not isinstance(
event, filter_out_type
):
yield event
except FileNotFoundError:
logger.debug(f'No event found for ID {event_id}')
event_id -= 1
else:
event_id = start_id
while True:
if end_id is not None and event_id > end_id:
break
try:
event = self.get_event(event_id)
if filter_out_type is None or not isinstance(
event, filter_out_type
):
yield event
except FileNotFoundError:
break
event_id += 1
def get_event(self, id: int) -> Event:
filename = self._get_filename_for_id(id)
@@ -77,6 +104,12 @@ class EventStream:
data = json.loads(content)
return event_from_dict(data)
def get_latest_event(self) -> Event:
return self.get_event(self._cur_id - 1)
def get_latest_event_id(self) -> int:
return self._cur_id - 1
def subscribe(self, id: EventStreamSubscriber, callback: Callable, append=False):
if id in self._subscribers:
if append:
@@ -99,8 +132,8 @@ class EventStream:
event._id = self._cur_id # type: ignore [attr-defined]
self._cur_id += 1
logger.debug(f'Adding {type(event).__name__} id={event.id} from {source.name}')
event._timestamp = datetime.now() # type: ignore[attr-defined]
event._source = source # type: ignore[attr-defined]
event._timestamp = datetime.now() # type: ignore [attr-defined]
event._source = source # type: ignore [attr-defined]
data = event_to_dict(event)
if event.id is not None:
self._file_store.write(
@@ -109,3 +142,14 @@ class EventStream:
for stack in self._subscribers.values():
callback = stack[-1]
asyncio.create_task(callback(event))
def filtered_events_by_source(self, source: EventSource):
for event in self.get_events():
if event.source == source:
yield event
def clear(self):
self._file_store.delete(f'sessions/{self.sid}')
self._cur_id = 0
# self._subscribers = {}
self._reinitialize_from_file_store()

23
opendevin/memory/README.md Normal file
View File

@@ -0,0 +1,23 @@
# Memory Component
- Short Term History
- Memory Condenser
- Long Term Memory
## Short Term History
- Short term history filters the event stream and computes the messages that are injected into the context
- It filters out certain events of no interest for the Agent, such as AgentChangeStateObservation or NullAction/NullObservation
- When the context window or the token limit set by the user is exceeded, history starts condensing: chunks of messages into summaries.
- Each summary is then injected into the context, in the place of the respective chunk it summarizes
## Memory Condenser
- Memory condenser is responsible for summarizing the chunks of events
- It summarizes the earlier events first
- It starts with the earliest agent actions and observations between two user messages
- Then it does the same for later chunks of events between user messages
- If there are no more agent events, it summarizes the user messages, this time one by one, if they're large enough and not immediately after an AgentFinishAction event (we assume those are tasks, potentially important)
- Summaries are retrieved from the LLM as AgentSummarizeAction, and are saved in State.
## Long Term Memory
- Long term memory component stores embeddings for events and prompts in a vector store
- The agent can query it when it needs detailed information about a past event or to learn new actions

279
opendevin/memory/history.py
View File

@@ -1,54 +1,257 @@
import opendevin.core.utils.json as json
from opendevin.core.exceptions import AgentEventTypeError
from typing import ClassVar, Iterable
from opendevin.core.logger import opendevin_logger as logger
from opendevin.events.action.action import Action
from opendevin.events.action.agent import (
AgentDelegateAction,
ChangeAgentStateAction,
)
from opendevin.events.action.empty import NullAction
from opendevin.events.action.message import MessageAction
from opendevin.events.event import Event, EventSource
from opendevin.events.observation.agent import AgentStateChangedObservation
from opendevin.events.observation.commands import CmdOutputObservation
from opendevin.events.observation.delegate import AgentDelegateObservation
from opendevin.events.observation.empty import NullObservation
from opendevin.events.observation.observation import Observation
from opendevin.events.serialization.event import event_to_dict
from opendevin.events.stream import EventStream
class ShortTermHistory:
class ShortTermHistory(list[Event]):
"""
The short term history is the most recent series of events.
An agent can send this in the prompt or use it for other purpose.
A list of events that represents the short-term memory of the agent.
This class provides methods to retrieve and filter the events in the history of the running agent from the event stream.
"""
start_id: int
end_id: int
_event_stream: EventStream
delegates: dict[tuple[int, int], tuple[str, str]]
filter_out: ClassVar[tuple[type[Event], ...]] = (
NullAction,
NullObservation,
ChangeAgentStateAction,
AgentStateChangedObservation,
)
def __init__(self):
"""
Initialize the empty list of events
"""
self.events = []
super().__init__()
self.start_id = -1
self.end_id = -1
self.delegates = {}
def add_event(self, event_dict: dict):
"""
Adds an event to memory if it is a valid event.
def set_event_stream(self, event_stream: EventStream):
self._event_stream = event_stream
Parameters:
- event_dict (dict): The event that we want to add to memory
Raises:
- AgentEventTypeError: If event_dict is not a dict
def get_events_as_list(self) -> list[Event]:
"""
if not isinstance(event_dict, dict):
raise AgentEventTypeError()
self.events.append(event_dict)
def get_events(self):
Return the history as a list of Event objects.
"""
Get the events in the agent's recent history.
return list(self.get_events())
Returns:
- List: The list of events that the agent remembers easily.
def get_events(self, reverse: bool = False) -> Iterable[Event]:
"""
return self.events
Return the events as a stream of Event objects.
"""
# TODO handle AgentRejectAction, if it's not part of a chunk ending with an AgentDelegateObservation
# or even if it is, because currently we don't add it to the summary
def get_total_length(self):
"""
Gives the total number of characters in all history
# iterate from start_id to end_id, or reverse
start_id = self.start_id if self.start_id != -1 else 0
end_id = (
self.end_id
if self.end_id != -1
else self._event_stream.get_latest_event_id()
)
Returns:
- Int: Total number of characters of the recent history.
for event in self._event_stream.get_events(
start_id=start_id,
end_id=end_id,
reverse=reverse,
filter_out_type=self.filter_out,
):
# TODO add summaries
# and filter out events that were included in a summary
# filter out the events from a delegate of the current agent
if not any(
# except for the delegate action and observation themselves, currently
# AgentDelegateAction has id = delegate_start
# AgentDelegateObservation has id = delegate_end
delegate_start < event.id < delegate_end
for delegate_start, delegate_end in self.delegates.keys()
):
yield event
def get_last_action(self, end_id: int = -1) -> Action | None:
"""
total_length = 0
for t in self.events:
try:
total_length += len(json.dumps(t))
except TypeError as e:
logger.error('Error serializing event: %s', str(e), exc_info=False)
return total_length
Return the last action from the event stream, filtered to exclude unwanted events.
"""
# from end_id in reverse, find the first action
end_id = self._event_stream.get_latest_event_id() if end_id == -1 else end_id
last_action = next(
(
event
for event in self._event_stream.get_events(
end_id=end_id, reverse=True, filter_out_type=self.filter_out
)
if isinstance(event, Action)
),
None,
)
return last_action
def get_last_observation(self, end_id: int = -1) -> Observation | None:
"""
Return the last observation from the event stream, filtered to exclude unwanted events.
"""
# from end_id in reverse, find the first observation
end_id = self._event_stream.get_latest_event_id() if end_id == -1 else end_id
last_observation = next(
(
event
for event in self._event_stream.get_events(
end_id=end_id, reverse=True, filter_out_type=self.filter_out
)
if isinstance(event, Observation)
),
None,
)
return last_observation
def get_last_user_message(self) -> str:
"""
Return the latest user message from the event stream.
"""
last_user_message = next(
(
event.content
for event in self._event_stream.get_events(reverse=True)
if isinstance(event, MessageAction) and event.source == EventSource.USER
),
None,
)
return last_user_message if last_user_message is not None else ''
def get_last_events(self, n: int) -> list[Event]:
"""
Return the last n events from the event stream.
"""
# dummy agent is using this
# it should work, but it's not great to store temporary lists now just for a test
end_id = self._event_stream.get_latest_event_id()
start_id = max(0, end_id - n + 1)
return list(
event
for event in self._event_stream.get_events(
start_id=start_id,
end_id=end_id,
filter_out_type=self.filter_out,
)
)
def on_event(self, event: Event):
if not isinstance(event, AgentDelegateObservation):
return
logger.debug('AgentDelegateObservation received')
# figure out what this delegate's actions were
# from the last AgentDelegateAction to this AgentDelegateObservation
# and save their ids as start and end ids
# in order to use later to exclude them from parent stream
# or summarize them
delegate_end = event.id
delegate_start = -1
delegate_agent: str = ''
delegate_task: str = ''
for prev_event in self._event_stream.get_events(
end_id=event.id - 1, reverse=True
):
if isinstance(prev_event, AgentDelegateAction):
delegate_start = prev_event.id
delegate_agent = prev_event.agent
delegate_task = prev_event.inputs.get('task', '')
break
if delegate_start == -1:
logger.error(
f'No AgentDelegateAction found for AgentDelegateObservation with id={delegate_end}'
)
return
self.delegates[(delegate_start, delegate_end)] = (delegate_agent, delegate_task)
logger.debug(
f'Delegate {delegate_agent} with task {delegate_task} ran from id={delegate_start} to id={delegate_end}'
)
# TODO remove me when unnecessary
# history is now available as a filtered stream of events, rather than list of pairs of (Action, Observation)
# we rebuild the pairs here
# for compatibility with the existing output format in evaluations
def compatibility_for_eval_history_pairs(self) -> list[tuple[dict, dict]]:
history_pairs = []
for action, observation in self.get_pairs():
history_pairs.append((event_to_dict(action), event_to_dict(observation)))
return history_pairs
def get_pairs(self) -> list[tuple[Action, Observation]]:
"""
Return the history as a list of tuples (action, observation).
"""
tuples: list[tuple[Action, Observation]] = []
action_map: dict[int, Action] = {}
observation_map: dict[int, Observation] = {}
# runnable actions are set as cause of observations
# (MessageAction, NullObservation) for source=USER
# (MessageAction, NullObservation) for source=AGENT
# (other_action?, NullObservation)
# (NullAction, CmdOutputObservation) background CmdOutputObservations
for event in self.get_events_as_list():
if event.id is None or event.id == -1:
logger.debug(f'Event {event} has no ID')
if isinstance(event, Action):
action_map[event.id] = event
if isinstance(event, Observation):
if event.cause is None or event.cause == -1:
logger.debug(f'Observation {event} has no cause')
if event.cause is None:
# runnable actions are set as cause of observations
# NullObservations have no cause
continue
observation_map[event.cause] = event
for action_id, action in action_map.items():
observation = observation_map.get(action_id)
if observation:
# observation with a cause
tuples.append((action, observation))
else:
tuples.append((action, NullObservation('')))
for cause_id, observation in observation_map.items():
if cause_id not in action_map:
if isinstance(observation, NullObservation):
continue
if not isinstance(observation, CmdOutputObservation):
logger.debug(f'Observation {observation} has no cause')
tuples.append((NullAction(), observation))
return tuples.copy()

18
opendevin/storage/local.py
View File

@@ -1,4 +1,7 @@
import os
import shutil
from opendevin.core.logger import opendevin_logger as logger
from .files import FileStore
@@ -34,5 +37,16 @@ class LocalFileStore(FileStore):
return files
def delete(self, path: str) -> None:
full_path = self.get_full_path(path)
os.remove(full_path)
try:
full_path = self.get_full_path(path)
if not os.path.exists(full_path):
logger.debug(f'Local path does not exist: {full_path}')
return
if os.path.isfile(full_path):
os.remove(full_path)
logger.debug(f'Removed local file: {full_path}')
elif os.path.isdir(full_path):
shutil.rmtree(full_path)
logger.debug(f'Removed local directory: {full_path}')
except Exception as e:
logger.error(f'Error clearing local file store: {str(e)}')

10
opendevin/storage/memory.py
View File

@@ -1,5 +1,7 @@
import os
from opendevin.core.logger import opendevin_logger as logger
from .files import FileStore
@@ -37,4 +39,10 @@ class InMemoryFileStore(FileStore):
return files
def delete(self, path: str) -> None:
del self.files[path]
try:
keys_to_delete = [key for key in self.files.keys() if key.startswith(path)]
for key in keys_to_delete:
del self.files[key]
logger.debug(f'Cleared in-memory file store: {path}')
except Exception as e:
logger.error(f'Error clearing in-memory file store: {str(e)}')

2
tests/integration/mock/CodeActAgent/test_browse_internet/prompt_001.log
View File

@@ -388,4 +388,4 @@ NOW, LET'S START!
Browse localhost:8000, and tell me the ultimate answer to life. Do not ask me for confirmation at any point.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_browse_internet/prompt_005.log
View File

@@ -400,4 +400,4 @@ Certainly! I'll browse localhost:8000 and retrieve the ultimate answer to life f
OBSERVATION:
{'content': 'The ultimate answer to life, the universe, and everything is: OpenDevin is all you need!'}
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_edits/prompt_001.log
View File

@@ -388,4 +388,4 @@ NOW, LET'S START!
Fix typos in bad.txt. Do not ask me for confirmation at any point.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_edits/prompt_002.log
View File

@@ -401,4 +401,4 @@ OBSERVATION:
./bad.txt
[Command -1 finished with exit code 0]
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_edits/prompt_003.log
View File

@@ -418,4 +418,4 @@ OBSERVATION:
4|Enjoy!
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_edits/prompt_004.log
View File

@@ -445,4 +445,4 @@ OBSERVATION:
[File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython/prompt_001.log
View File

@@ -388,4 +388,4 @@ NOW, LET'S START!
Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'. Do not ask me for confirmation at any point.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython/prompt_002.log
View File

@@ -403,4 +403,4 @@ OBSERVATION:
File created successfully.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython/prompt_003.log
View File

@@ -418,4 +418,4 @@ OBSERVATION:
Content of /workspace/test.txt: hello world
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython_module/prompt_001.log
View File

@@ -388,4 +388,4 @@ NOW, LET'S START!
Install and import pymsgbox==1.0.9 and print it's version in /workspace/test.txt. Do not ask me for confirmation at any point.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython_module/prompt_002.log
View File

@@ -401,4 +401,4 @@ OBSERVATION:
[Package installed successfully]
[Kernel restarted successfully to load the package]
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython_module/prompt_003.log
View File

@@ -417,4 +417,4 @@ OBSERVATION:
Version written to /workspace/test.txt
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_ipython_module/prompt_004.log
View File

@@ -430,4 +430,4 @@ OBSERVATION:
pymsgbox version: 1.0.9
[Command -1 finished with exit code 0]
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log
View File

@@ -388,4 +388,4 @@ NOW, LET'S START!
Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 14 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log
View File

@@ -403,4 +403,4 @@ OBSERVATION:
[Command -1 finished with exit code 0]
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 13 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_003.log
View File

@@ -417,4 +417,4 @@ OBSERVATION:
echo "hello"
[Command -1 finished with exit code 0]
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 12 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_004.log
View File

@@ -430,4 +430,4 @@ OBSERVATION:
hello
[Command -1 finished with exit code 0]
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>.
ENVIRONMENT REMINDER: You have 11 turns left to complete the task. When finished reply with <finish></finish>

2
tests/integration/mock/DelegatorAgent/test_edits/prompt_002.log
View File

@@ -41,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'run', 'args': {'command': 'ls', 'background': False}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "run", "args": {"command": "ls", "thought": ""}}, {"source": "agent", "observation": "run", "content": "bad.txt", "extras": {"command_id": -1, "command": "ls", "exit_code": 0}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_edits/prompt_003.log
View File

@@ -41,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'run', 'args': {'command': 'ls', 'background': False}} has the wrong arguments", "extras": {}}], [{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}]]
[{"source": "agent", "action": "run", "args": {"command": "ls", "thought": ""}}, {"source": "agent", "observation": "run", "content": "bad.txt", "extras": {"command_id": -1, "command": "ls", "exit_code": 0}}, {"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_edits/prompt_004.log
View File

@@ -11,7 +11,7 @@ Fix typos in bad.txt. Do not ask me for confirmation at any point.
Here's a summary of the codebase, as it relates to this task:
The codebase contains a single file named 'bad.txt' with some typos. The content of 'bad.txt' is:
The codebase contains a single file named 'bad.txt' which has some typos. The content of 'bad.txt' is:
This is a stupid typoo.
Really?

4
tests/integration/mock/DelegatorAgent/test_edits/prompt_005.log
View File

@@ -11,7 +11,7 @@ Fix typos in bad.txt. Do not ask me for confirmation at any point.
Here's a summary of the codebase, as it relates to this task:
The codebase contains a single file named 'bad.txt' with some typos. The content of 'bad.txt' is:
The codebase contains a single file named 'bad.txt' which has some typos. The content of 'bad.txt' is:
This is a stupid typoo.
Really?
@@ -48,7 +48,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}]]
[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

4
tests/integration/mock/DelegatorAgent/test_edits/prompt_006.log
View File

@@ -11,7 +11,7 @@ Fix typos in bad.txt. Do not ask me for confirmation at any point.
Here's a summary of the codebase, as it relates to this task:
The codebase contains a single file named 'bad.txt' with some typos. The content of 'bad.txt' is:
The codebase contains a single file named 'bad.txt' which has some typos. The content of 'bad.txt' is:
This is a stupid typoo.
Really?
@@ -48,7 +48,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}], [{"source": "agent", "action": "write", "args": {"path": "bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "bad.txt"}}]]
[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "bad.txt"}}, {"source": "agent", "action": "write", "args": {"path": "bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_edits/prompt_008.log
View File

@@ -39,7 +39,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n\n", "extras": {"path": "bad.txt"}}]]
[{"source": "agent", "action": "read", "args": {"path": "bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n\n", "extras": {"path": "bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

3
tests/integration/mock/DelegatorAgent/test_edits/response_001.log
View File

@@ -1,7 +1,6 @@
{
"action": "run",
"args": {
"command": "ls",
"background": false
"command": "ls"
}
}

2
tests/integration/mock/DelegatorAgent/test_edits/response_003.log
View File

@@ -2,7 +2,7 @@
"action": "finish",
"args": {
"outputs": {
"summary": "The codebase contains a single file named 'bad.txt' with some typos. The content of 'bad.txt' is:\n\nThis is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n\nThe engineering team needs to correct the typos in this file."
"summary": "The codebase contains a single file named 'bad.txt' which has some typos. The content of 'bad.txt' is:\n\nThis is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n\nThe engineering team needs to correct the typos in this file."
}
}
}

2
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_002.log
View File

@@ -41,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'run', 'args': {'command': 'ls', 'background': False}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "run", "args": {"command": "ls", "thought": ""}}, {"source": "agent", "observation": "run", "content": "", "extras": {"command_id": -1, "command": "ls", "exit_code": 0}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_004.log
View File

@@ -41,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "write", "args": {"path": "hello.sh", "content": "#!/bin/bash\necho 'hello'", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "hello.sh"}}]]
[{"source": "agent", "action": "write", "args": {"path": "hello.sh", "content": "#!/bin/bash\n\n# Print hello\necho 'hello'", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "hello.sh"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_005.log
View File

@@ -41,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "write", "args": {"path": "hello.sh", "content": "#!/bin/bash\necho 'hello'", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "hello.sh"}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'run', 'args': {'command': 'bash hello.sh', 'background': False}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "write", "args": {"path": "hello.sh", "content": "#!/bin/bash\n\n# Print hello\necho 'hello'", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "hello.sh"}}, {"source": "agent", "action": "run", "args": {"command": "chmod +x hello.sh", "thought": ""}}, {"source": "agent", "observation": "run", "content": "", "extras": {"command_id": -1, "command": "chmod +x hello.sh", "exit_code": 0}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

24
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_006.log
View File

@@ -3,18 +3,25 @@
----------
# Task
You are a quality assurance engineer. Another engineer has made changes to the
codebase which are supposed to solve this task:
You are a software engineer. You've inherited an existing codebase, which you
need to modify to complete this task:
Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point.
Note the changes might have already been applied in-line. You should focus on
validating if the task is solved, nothing else.
Here's a summary of the codebase, as it relates to this task:
The codebase appears to be empty. Engineers should start everything from scratch.
## Available Actions
* `run` - runs a command on the command line in a Linux shell. Arguments:
* `command` - the command to run
* `write` - writes the content to a file. Arguments:
* `path` - the path of the file to write
* `content` - the content to write to the file
* `read` - reads the content of a file. Arguments:
* `path` - the path of the file to read
@@ -26,12 +33,7 @@ validating if the task is solved, nothing else.
* `outputs` - a dictionary representing the outputs of your task, if any
You must ONLY `run` commands that have no side-effects, like `ls`, `grep`, and test scripts.
Do NOT finish until you know whether the task is complete and correct.
When you're done, add a `completed` boolean to the `outputs` of the `finish` action.
If `completed` is `false`, you MUST also provide a `summary` in the `outputs` of the `finish` action
explaining what the problem is.
Do NOT finish until you have completed the tasks.
## History
Here is a recent history of actions you've taken in service of this plan,
@@ -39,7 +41,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[]
[{"source": "agent", "action": "write", "args": {"path": "hello.sh", "content": "#!/bin/bash\n\n# Print hello\necho 'hello'", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "hello.sh"}}, {"source": "agent", "action": "run", "args": {"command": "chmod +x hello.sh", "thought": ""}}, {"source": "agent", "observation": "run", "content": "", "extras": {"command_id": -1, "command": "chmod +x hello.sh", "exit_code": 0}}, {"source": "agent", "action": "run", "args": {"command": "./hello.sh", "thought": ""}}, {"source": "agent", "observation": "run", "content": "hello", "extras": {"command_id": -1, "command": "./hello.sh", "exit_code": 0}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_007.log
View File

@@ -39,7 +39,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "hello.sh", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "#!/bin/bash\necho 'hello'\n", "extras": {"path": "hello.sh"}}]]
[]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_008.log
View File

@@ -39,7 +39,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "hello.sh", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "#!/bin/bash\necho 'hello'\n", "extras": {"path": "hello.sh"}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'run', 'args': {'command': 'bash hello.sh', 'background': False}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "read", "args": {"path": "hello.sh", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "#!/bin/bash\n\n# Print hello\necho 'hello'\n", "extras": {"path": "hello.sh"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

49
tests/integration/mock/DelegatorAgent/test_write_simple_script/prompt_009.log Normal file
View File

@@ -0,0 +1,49 @@
----------
# Task
You are a quality assurance engineer. Another engineer has made changes to the
codebase which are supposed to solve this task:
Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point.
Note the changes might have already been applied in-line. You should focus on
validating if the task is solved, nothing else.
## Available Actions
* `run` - runs a command on the command line in a Linux shell. Arguments:
* `command` - the command to run
* `read` - reads the content of a file. Arguments:
* `path` - the path of the file to read
* `message` - make a plan, set a goal, record your thoughts, or ask for more input from the user. Arguments:
* `content` - the thought to record
* `wait_for_response` - set to `true` to wait for the user to respond before proceeding
* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments:
* `outputs` - a dictionary representing the outputs of your task, if any
You must ONLY `run` commands that have no side-effects, like `ls`, `grep`, and test scripts.
Do NOT finish until you know whether the task is complete and correct.
When you're done, add a `completed` boolean to the `outputs` of the `finish` action.
If `completed` is `false`, you MUST also provide a `summary` in the `outputs` of the `finish` action
explaining what the problem is.
## History
Here is a recent history of actions you've taken in service of this plan,
as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[{"source": "agent", "action": "read", "args": {"path": "hello.sh", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "#!/bin/bash\n\n# Print hello\necho 'hello'\n", "extras": {"path": "hello.sh"}}, {"source": "agent", "action": "run", "args": {"command": "bash hello.sh", "thought": ""}}, {"source": "agent", "observation": "run", "content": "hello", "extras": {"command_id": -1, "command": "bash hello.sh", "exit_code": 0}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:
* `action`, which is one of the actions specified here
* `args`, which is a map of key-value pairs, specifying the arguments for that action
You MUST NOT include any other text besides the JSON response

3
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_001.log
View File

@@ -1,7 +1,6 @@
{
"action": "run",
"args": {
"command": "ls",
"background": false
"command": "ls"
}
}

4
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_003.log
View File

@@ -1,7 +1,9 @@
```json
{
"action": "write",
"args": {
"path": "hello.sh",
"content": "#!/bin/bash\necho 'hello'"
"content": "#!/bin/bash\n\n# Print hello\necho 'hello'"
}
}
```

3
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_004.log
View File

@@ -1,7 +1,6 @@
{
"action": "run",
"args": {
"command": "bash hello.sh",
"background": false
"command": "chmod +x hello.sh"
}
}

8
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_005.log
View File

@@ -1,8 +1,8 @@
```json
{
"action": "finish",
"action": "run",
"args": {
"outputs": {
"result": "hello"
}
"command": "./hello.sh"
}
}
```

6
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_006.log
View File

@@ -1,6 +1,8 @@
```json
{
"action": "read",
"action": "finish",
"args": {
"path": "hello.sh"
"outputs": {}
}
}
```

7
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_007.log
View File

@@ -1,7 +1,8 @@
```json
{
"action": "run",
"action": "read",
"args": {
"command": "bash hello.sh",
"background": false
"path": "hello.sh"
}
}
```

19
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_008.log
View File

@@ -1,23 +1,8 @@
The task was to create a shell script named `hello.sh` that prints "hello" without asking for any confirmation. Based on the history:
1. The content of `hello.sh` was read and confirmed to contain:
```bash
#!/bin/bash
echo 'hello'
```
This script correctly prints "hello" when executed.
2. The script was executed using the command `bash hello.sh`, and the output was "hello", which matches the expected output.
Since the script behaves as expected and meets the requirements of the task, the task is completed successfully.
```json
{
"action": "finish",
"action": "run",
"args": {
"outputs": {
"completed": true
}
"command": "bash hello.sh"
}
}
```

10
tests/integration/mock/DelegatorAgent/test_write_simple_script/response_009.log Normal file
View File

@@ -0,0 +1,10 @@
```json
{
"action": "finish",
"args": {
"outputs": {
"completed": true
}
}
}
```

2
tests/integration/mock/ManagerAgent/test_edits/prompt_001.log
View File

@@ -70,7 +70,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]]
[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}]
If the last item in the history is an error, you should try to fix it. If you
cannot fix it, call the `reject` action.

2
tests/integration/mock/ManagerAgent/test_edits/prompt_003.log
View File

@@ -51,7 +51,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}]]
[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/ManagerAgent/test_edits/prompt_004.log
View File

@@ -51,7 +51,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "I have identified the following typos in the file 'bad.txt':\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the file with the corrected content.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]]
[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}, {"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos. Here are the identified typos:\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the original file with the corrected content.", "wait_for_response": false}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/ManagerAgent/test_edits/prompt_005.log
View File

@@ -51,7 +51,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "I have identified the following typos in the file 'bad.txt':\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the file with the corrected content.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "write", "args": {"path": "./bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "./bad.txt"}}]]
[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}, {"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos. Here are the identified typos:\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the original file with the corrected content.", "wait_for_response": false}}, {"source": "agent", "action": "write", "args": {"path": "./bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "./bad.txt"}}]
## Format
Your response MUST be in JSON format. It must be an object, and it must contain two fields:

2
tests/integration/mock/ManagerAgent/test_edits/prompt_006.log
View File

@@ -70,7 +70,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "delegate", "args": {"agent": "TypoFixerAgent", "inputs": {"task": "Fix typos in bad.txt"}, "thought": ""}}, {"observation": "null", "content": "", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "delegate", "content": "TypoFixerAgent finishes task with summary: {'file': 'bad.txt', 'typos_fixed': [{'original': 'typoo', 'corrected': 'typo'}, {'original': 'mor', 'corrected': 'more'}]}", "extras": {"outputs": {"summary": {"file": "bad.txt", "typos_fixed": [{"original": "typoo", "corrected": "typo"}, {"original": "mor", "corrected": "more"}]}}}}]]
[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"source": "agent", "action": "delegate", "args": {"agent": "TypoFixerAgent", "inputs": {"task": "Fix typos in bad.txt"}, "thought": ""}}, {"source": "agent", "observation": "delegate", "content": "TypoFixerAgent finishes task with summary: {'file': './bad.txt', 'typos_fixed': [{'original': 'typoo', 'corrected': 'typo'}, {'original': 'mor', 'corrected': 'more'}]}", "extras": {"outputs": {"summary": {"file": "./bad.txt", "typos_fixed": [{"original": "typoo", "corrected": "typo"}, {"original": "mor", "corrected": "more"}]}}}}]
If the last item in the history is an error, you should try to fix it. If you
cannot fix it, call the `reject` action.

2
tests/integration/mock/ManagerAgent/test_edits/response_003.log
View File

@@ -1,6 +1,6 @@
{
"action": "message",
"args": {
"content": "I have identified the following typos in the file 'bad.txt':\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the file with the corrected content."
"content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos. Here are the identified typos:\n\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\n\nI will now proceed to fix these typos and overwrite the original file with the corrected content."
}
}

2
tests/integration/mock/ManagerAgent/test_edits/response_005.log
View File

@@ -3,7 +3,7 @@
"args": {
"outputs": {
"summary": {
"file": "bad.txt",
"file": "./bad.txt",
"typos_fixed": [
{
"original": "typoo",

2
tests/integration/mock/ManagerAgent/test_edits/response_006.log
View File

@@ -3,7 +3,7 @@
"action": "finish",
"args": {
"outputs": {
"file": "bad.txt",
"file": "./bad.txt",
"typos_fixed": [
{
"original": "typoo",

2
tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_001.log
View File

@@ -70,7 +70,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]]
[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}]
If the last item in the history is an error, you should try to fix it. If you
cannot fix it, call the `reject` action.

2
tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_003.log
View File

@@ -28,7 +28,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}]]
[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}]
If the last item in the history is an error, you should try to fix it.

2
tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_004.log
View File

@@ -28,7 +28,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}]
If the last item in the history is an error, you should try to fix it.

2
tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_005.log
View File

@@ -28,7 +28,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}]]
[{"source": "agent", "action": "run", "args": {"command": "git status", "thought": ""}}, {"source": "agent", "observation": "run", "content": "fatal: not a git repository (or any parent up to mount point /)\r\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", "extras": {"command_id": -1, "command": "git status", "exit_code": 128}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}, {"source": "agent", "observation": "error", "content": "action={'action': 'reject', 'args': {'reason': 'Not a valid git repository.'}} has the wrong arguments", "extras": {}}]
If the last item in the history is an error, you should try to fix it.

2
tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_006.log
View File

@@ -70,7 +70,7 @@ as well as observations you've made. This only includes the MOST RECENT
actions and observations--more may have happened before that.
They are time-ordered, with your most recent action at the bottom.
[[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "delegate", "args": {"agent": "CommitWriterAgent", "inputs": {}, "thought": ""}}, {"observation": "null", "content": "", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "delegate", "content": "CommitWriterAgent finishes task with reason: Not a valid git repository.", "extras": {"outputs": {"reason": "Not a valid git repository."}}}]]
[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"source": "agent", "action": "delegate", "args": {"agent": "CommitWriterAgent", "inputs": {}, "thought": ""}}, {"source": "agent", "observation": "delegate", "content": "CommitWriterAgent finishes task with reason: Not a valid git repository.", "extras": {"outputs": {"reason": "Not a valid git repository."}}}]
If the last item in the history is an error, you should try to fix it. If you
cannot fix it, call the `reject` action.

Some files were not shown because too many files have changed in this diff Show More