mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2026-01-09 14:57:59 -05:00
Co-authored-by: Engel Nyst <enyst@users.noreply.github.com>
This commit is contained in:
OpenHands
38
evaluation/benchmarks/agent_bench/README.md
Normal file
38
evaluation/benchmarks/agent_bench/README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# AgentBench Evaluation
|
||||
|
||||
This folder contains evaluation harness for evaluating agents on the [AgentBench: Evaluating LLMs as Agents](https://arxiv.org/abs/2308.03688). We currently only support running on the `osbench` subset.
|
||||
|
||||
## Setup Environment and LLM Configuration
|
||||
|
||||
Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
|
||||
|
||||
## Start the evaluation
|
||||
|
||||
```bash
|
||||
./evaluation/benchmarks/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
|
||||
```
|
||||
|
||||
- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
|
||||
LLM settings, as defined in your `config.toml`.
|
||||
- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
|
||||
like to evaluate. It could also be a release tag like `0.6.2`.
|
||||
- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
|
||||
to `CodeActAgent`.
|
||||
- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
|
||||
default, the script evaluates the entire SWE-bench_Lite test set (300 issues). Note:
|
||||
in order to use `eval_limit`, you must also set `agent`.
|
||||
|
||||
|
||||
Following is the basic command to start the evaluation.
|
||||
|
||||
You can update the arguments in the script `evaluation/benchmarks/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
|
||||
|
||||
- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
|
||||
- `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
|
||||
- `--max-iterations`: the number of iterations to run the evaluation. For example, `30`.
|
||||
- `--eval-num-workers`: the number of workers to use for evaluation. For example, `5`.
|
||||
- `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
|
||||
|
||||
```bash
|
||||
./evaluation/benchmarks/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
|
||||
```
|
||||
0
evaluation/benchmarks/agent_bench/__init__.py
Normal file
0
evaluation/benchmarks/agent_bench/__init__.py
Normal file
77
evaluation/benchmarks/agent_bench/helper.py
Normal file
77
evaluation/benchmarks/agent_bench/helper.py
Normal file
@@ -0,0 +1,77 @@
|
||||
import os
|
||||
import re
|
||||
from functools import partial
|
||||
|
||||
from evaluation.utils.shared import codeact_user_response
|
||||
from openhands.events.action import CmdRunAction, MessageAction
|
||||
|
||||
|
||||
def try_parse_answer(act) -> str | None:
|
||||
raw_ans = ''
|
||||
if isinstance(act, MessageAction) and act.source == 'agent':
|
||||
raw_ans = act.content
|
||||
elif isinstance(act, CmdRunAction) and act.source == 'agent':
|
||||
raw_ans = act.thought
|
||||
else:
|
||||
return None
|
||||
agent_answer = re.findall(r'<solution>(.*?)</solution>', raw_ans, re.DOTALL)
|
||||
if not agent_answer:
|
||||
return None
|
||||
return agent_answer[0].strip()
|
||||
|
||||
|
||||
FAKE_RESPONSES = {
|
||||
'CodeActAgent': partial(
|
||||
codeact_user_response, encapsulate_solution=True, try_parse=try_parse_answer
|
||||
),
|
||||
}
|
||||
|
||||
INST_SUFFIXES: dict[str, str] = {
|
||||
'CodeActAgent': (
|
||||
'When you think you have solved the question, '
|
||||
'please first send your answer to user through message and then exit.\n'
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def analysis_size(size_str):
|
||||
size_str = size_str.strip()
|
||||
avails = {
|
||||
'B': 1,
|
||||
'Byte': 1,
|
||||
'K': 1024,
|
||||
'KB': 1024,
|
||||
'M': 1024 * 1024,
|
||||
'MB': 1024 * 1024,
|
||||
'G': 1024 * 1024 * 1024,
|
||||
'GB': 1024 * 1024 * 1024,
|
||||
'T': 1024 * 1024 * 1024 * 1024,
|
||||
'TB': 1024 * 1024 * 1024 * 1024,
|
||||
'P': 1024 * 1024 * 1024 * 1024 * 1024,
|
||||
'PB': 1024 * 1024 * 1024 * 1024 * 1024,
|
||||
}
|
||||
for size_unit in avails:
|
||||
if size_str.endswith(size_unit):
|
||||
return int(size_str[: -len(size_unit)]) * avails[size_unit]
|
||||
return int(size_str)
|
||||
|
||||
|
||||
def compare_results(check_method: str, model_answer: str, final_ans: str) -> bool:
|
||||
try:
|
||||
match check_method:
|
||||
case 'check/integer-match.py':
|
||||
return int(model_answer) == int(final_ans)
|
||||
case 'check/size-match.py':
|
||||
return analysis_size(model_answer) == analysis_size(final_ans)
|
||||
return (
|
||||
model_answer.replace('\r\n', '\n').replace('\r', '\n').strip()
|
||||
== final_ans.replace('\r\n', '\n').replace('\r', '\n').strip()
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def create_sh_file(filename: str, cmds: str) -> None:
|
||||
with open(filename, 'w', encoding='utf-8') as file:
|
||||
file.write(cmds.replace('\r\n', '\n'))
|
||||
os.chmod(filename, 0o755)
|
||||
323
evaluation/benchmarks/agent_bench/run_infer.py
Normal file
323
evaluation/benchmarks/agent_bench/run_infer.py
Normal file
@@ -0,0 +1,323 @@
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from datasets import load_dataset
|
||||
|
||||
from evaluation.benchmarks.agent_bench.helper import (
|
||||
FAKE_RESPONSES,
|
||||
INST_SUFFIXES,
|
||||
compare_results,
|
||||
create_sh_file,
|
||||
)
|
||||
from evaluation.utils.shared import (
|
||||
EvalMetadata,
|
||||
EvalOutput,
|
||||
compatibility_for_eval_history_pairs,
|
||||
make_metadata,
|
||||
prepare_dataset,
|
||||
reset_logger_for_multiprocessing,
|
||||
run_evaluation,
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
SandboxConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime, run_controller
|
||||
from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
|
||||
from openhands.events.observation import CmdOutputObservation
|
||||
from openhands.runtime.base import Runtime
|
||||
from openhands.utils.async_utils import call_async_from_sync
|
||||
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
config = AppConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='eventstream',
|
||||
max_iterations=metadata.max_iterations,
|
||||
sandbox=SandboxConfig(
|
||||
base_container_image='python:3.12-bookworm',
|
||||
enable_auto_lint=True,
|
||||
use_host_network=False,
|
||||
),
|
||||
# do not mount workspace
|
||||
workspace_base=None,
|
||||
workspace_mount_path=None,
|
||||
)
|
||||
config.set_llm_config(metadata.llm_config)
|
||||
return config
|
||||
|
||||
|
||||
def initialize_runtime(
|
||||
runtime: Runtime,
|
||||
instance: pd.Series, # this argument is not required
|
||||
):
|
||||
"""Initialize the runtime for the agent.
|
||||
|
||||
This function is called before the runtime is used to run the agent.
|
||||
"""
|
||||
logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
|
||||
obs: CmdOutputObservation
|
||||
|
||||
# Set instance id
|
||||
action = CmdRunAction(command='mkdir -p /workspace')
|
||||
logger.info(action, extra={'msg_type': 'ACTION'})
|
||||
obs = runtime.run_action(action)
|
||||
assert obs.exit_code == 0
|
||||
|
||||
action = CmdRunAction(command='cd /workspace')
|
||||
logger.info(action, extra={'msg_type': 'ACTION'})
|
||||
obs = runtime.run_action(action)
|
||||
assert obs.exit_code == 0
|
||||
|
||||
init_cmd = instance.init
|
||||
if init_cmd is not None:
|
||||
script_name = f'{instance.instance_id}_init.sh'
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
host_script_path = os.path.join(tmpdir, script_name)
|
||||
create_sh_file(host_script_path, init_cmd)
|
||||
runtime.copy_to(
|
||||
host_script_path,
|
||||
'/workspace',
|
||||
)
|
||||
|
||||
logger.info(f'Running init script: {script_name}')
|
||||
action = CmdRunAction(command=f'chmod +x ./{script_name} && ./{script_name}')
|
||||
logger.info(action, extra={'msg_type': 'ACTION'})
|
||||
obs = runtime.run_action(action)
|
||||
logger.info(obs, extra={'msg_type': 'OBSERVATION'})
|
||||
assert obs.exit_code == 0
|
||||
|
||||
logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
|
||||
|
||||
|
||||
def complete_runtime(
|
||||
runtime: Runtime,
|
||||
instance: pd.Series, # this argument is not required, but it is used to get the workspace_dir_name
|
||||
) -> dict[str, Any]:
|
||||
"""Complete the runtime for the agent.
|
||||
|
||||
This function is called before the runtime is used to run the agent.
|
||||
If you need to do something in the sandbox to get the correctness metric after
|
||||
the agent has run, modify this function.
|
||||
"""
|
||||
logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
|
||||
obs: CmdOutputObservation
|
||||
|
||||
agent_answer = None
|
||||
get_agent_result_cmd = instance.get_agent_result
|
||||
if get_agent_result_cmd is not None:
|
||||
script_name = 'get_agent_result.sh'
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
host_script_path = os.path.join(tmpdir, script_name)
|
||||
create_sh_file(host_script_path, get_agent_result_cmd)
|
||||
runtime.copy_to(
|
||||
host_script_path,
|
||||
'/workspace',
|
||||
)
|
||||
logger.info(f'Running get agent result cmd: {script_name}')
|
||||
|
||||
action = CmdRunAction(
|
||||
command=f'chmod +x ./{script_name} && ./{script_name}',
|
||||
keep_prompt=False,
|
||||
)
|
||||
logger.info(action, extra={'msg_type': 'ACTION'})
|
||||
obs = runtime.run_action(action)
|
||||
logger.info(obs, extra={'msg_type': 'OBSERVATION'})
|
||||
assert obs.exit_code == 0
|
||||
agent_answer = obs.content
|
||||
# IF the agent answer is not found, retrieve it from the history
|
||||
# We wait until the controller finishes
|
||||
|
||||
final_ans = None
|
||||
if instance.ground_truth is not None:
|
||||
final_ans = instance.ground_truth
|
||||
else:
|
||||
get_ground_truth_cmd = instance.get_ground_truth
|
||||
if get_ground_truth_cmd is not None:
|
||||
script_name = 'get_ground_truth.sh'
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
host_script_path = os.path.join(tmpdir, script_name)
|
||||
create_sh_file(host_script_path, get_ground_truth_cmd)
|
||||
runtime.copy_to(
|
||||
host_script_path,
|
||||
'/workspace',
|
||||
)
|
||||
logger.info(f'Running get ground truth cmd: {script_name}')
|
||||
|
||||
action = CmdRunAction(
|
||||
command=f'chmod +x ./{script_name} && ./{script_name}',
|
||||
keep_prompt=False,
|
||||
)
|
||||
logger.info(action, extra={'msg_type': 'ACTION'})
|
||||
obs = runtime.run_action(action)
|
||||
logger.info(obs, extra={'msg_type': 'OBSERVATION'})
|
||||
final_ans = obs.content
|
||||
|
||||
logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
|
||||
return {
|
||||
'final_ans': final_ans,
|
||||
'agent_answer': agent_answer,
|
||||
}
|
||||
|
||||
|
||||
def process_instance(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
reset_logger: bool = True,
|
||||
) -> EvalOutput:
|
||||
config = get_config(metadata)
|
||||
|
||||
# Setup the logger properly, so you can run multi-processing to parallelize the evaluation
|
||||
if reset_logger:
|
||||
log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
|
||||
reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
|
||||
else:
|
||||
logger.info(f'Starting evaluation for instance {instance.instance_id}.')
|
||||
|
||||
# =============================================
|
||||
# build instruction
|
||||
# =============================================
|
||||
|
||||
# Prepare instruction
|
||||
instruction = (
|
||||
f'Please fix the following issue.\n'
|
||||
'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
|
||||
'Please encapsulate your final answer (answer ONLY) within <solution> and </solution>.\n'
|
||||
'For example: The answer to the question is <solution> 42 </solution>.\n'
|
||||
'# Problem \n'
|
||||
f'{instance.description}\n\n'
|
||||
)
|
||||
instruction += (
|
||||
'IMPORTANT: You should ONLY interact with the environment provided '
|
||||
'to you AND NEVER ASK FOR HUMAN HELP.\n'
|
||||
)
|
||||
# NOTE: You can actually set slightly different instruction for different agents
|
||||
instruction += INST_SUFFIXES[metadata.agent_class]
|
||||
|
||||
# =============================================
|
||||
# create sandbox and run the agent
|
||||
# =============================================
|
||||
|
||||
runtime: Runtime = create_runtime(config)
|
||||
call_async_from_sync(runtime.connect)
|
||||
|
||||
initialize_runtime(runtime, instance=instance)
|
||||
|
||||
# Here's how you can run the agent (similar to the `main` function) and get the final task state
|
||||
state: State | None = asyncio.run(
|
||||
run_controller(
|
||||
config=config,
|
||||
initial_user_action=MessageAction(content=instruction),
|
||||
runtime=runtime,
|
||||
fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
|
||||
)
|
||||
)
|
||||
if state is None:
|
||||
raise ValueError('State should not be None.')
|
||||
|
||||
# =============================================
|
||||
# result evaluation
|
||||
# =============================================
|
||||
|
||||
return_val = complete_runtime(runtime, instance)
|
||||
agent_answer = return_val['agent_answer']
|
||||
final_ans = return_val['final_ans']
|
||||
|
||||
# If the agent answer is not found, retrieve it from the history
|
||||
if agent_answer is None:
|
||||
agent_answer = ''
|
||||
logger.info('Retrieving agent answer from history.')
|
||||
raw_ans = ''
|
||||
|
||||
# retrieve the last agent message or thought
|
||||
for event in reversed(state.history):
|
||||
if event.source == 'agent':
|
||||
if isinstance(event, AgentFinishAction):
|
||||
raw_ans = event.thought
|
||||
break
|
||||
elif isinstance(event, MessageAction):
|
||||
raw_ans = event.content
|
||||
break
|
||||
elif isinstance(event, CmdRunAction):
|
||||
raw_ans = event.thought
|
||||
break
|
||||
|
||||
# parse the answer for a solution tag
|
||||
agent_answer = re.findall(r'<solution>(.*?)</solution>', raw_ans, re.DOTALL)
|
||||
if len(agent_answer) == 0:
|
||||
logger.warning(f'Failed to parse model answer: {raw_ans}')
|
||||
agent_answer = raw_ans
|
||||
else:
|
||||
agent_answer = agent_answer[0]
|
||||
|
||||
comparison_method = instance.comparison_method
|
||||
logger.info(
|
||||
f'Final message: {agent_answer} | Ground truth: {final_ans} | Comparison method: {comparison_method}'
|
||||
)
|
||||
test_result = compare_results(comparison_method, agent_answer, final_ans)
|
||||
|
||||
# history is now available as a stream of events, rather than list of pairs of (Action, Observation)
|
||||
# for compatibility with the existing output format, we can remake the pairs here
|
||||
# remove when it becomes unnecessary
|
||||
histories = compatibility_for_eval_history_pairs(state.history)
|
||||
|
||||
metrics = state.metrics.get() if state.metrics else None
|
||||
|
||||
# Save the output
|
||||
output = EvalOutput(
|
||||
instance_id=instance.instance_id,
|
||||
instance=instance.to_dict(),
|
||||
instruction=instruction,
|
||||
metadata=metadata,
|
||||
history=histories,
|
||||
metrics=metrics,
|
||||
error=state.last_error if state and state.last_error else None,
|
||||
test_result={
|
||||
'agent_answer': agent_answer,
|
||||
'final_answer': final_ans,
|
||||
'check_method': comparison_method,
|
||||
'result': test_result,
|
||||
},
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_arguments()
|
||||
dataset = load_dataset('iFurySt/AgentBench')
|
||||
agent_bench_tests = dataset['osbench'].to_pandas()
|
||||
|
||||
llm_config = None
|
||||
if args.llm_config:
|
||||
llm_config = get_llm_config_arg(args.llm_config)
|
||||
|
||||
if llm_config is None:
|
||||
raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
|
||||
|
||||
metadata = make_metadata(
|
||||
llm_config,
|
||||
'AgentBench-OS',
|
||||
args.agent_cls,
|
||||
args.max_iterations,
|
||||
args.eval_note,
|
||||
args.eval_output_dir,
|
||||
)
|
||||
output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
|
||||
instances = prepare_dataset(agent_bench_tests, output_file, args.eval_n_limit)
|
||||
|
||||
run_evaluation(
|
||||
instances, metadata, output_file, args.eval_num_workers, process_instance
|
||||
)
|
||||
42
evaluation/benchmarks/agent_bench/scripts/run_infer.sh
Executable file
42
evaluation/benchmarks/agent_bench/scripts/run_infer.sh
Executable file
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
source "evaluation/utils/version_control.sh"
|
||||
|
||||
MODEL_CONFIG=$1
|
||||
COMMIT_HASH=$2
|
||||
AGENT=$3
|
||||
EVAL_LIMIT=$4
|
||||
NUM_WORKERS=$5
|
||||
|
||||
if [ -z "$NUM_WORKERS" ]; then
|
||||
NUM_WORKERS=1
|
||||
echo "Number of workers not specified, use default $NUM_WORKERS"
|
||||
fi
|
||||
checkout_eval_branch
|
||||
|
||||
if [ -z "$AGENT" ]; then
|
||||
echo "Agent not specified, use default CodeActAgent"
|
||||
AGENT="CodeActAgent"
|
||||
fi
|
||||
|
||||
get_agent_version
|
||||
|
||||
echo "AGENT: $AGENT"
|
||||
echo "AGENT_VERSION: $AGENT_VERSION"
|
||||
echo "MODEL_CONFIG: $MODEL_CONFIG"
|
||||
|
||||
COMMAND="export PYTHONPATH=evaluation/benchmarks/agent_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/agent_bench/run_infer.py \
|
||||
--agent-cls $AGENT \
|
||||
--llm-config $MODEL_CONFIG \
|
||||
--max-iterations 30 \
|
||||
--eval-num-workers $NUM_WORKERS \
|
||||
--eval-note $AGENT_VERSION"
|
||||
|
||||
if [ -n "$EVAL_LIMIT" ]; then
|
||||
echo "EVAL_LIMIT: $EVAL_LIMIT"
|
||||
COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
|
||||
fi
|
||||
|
||||
# Run the command
|
||||
eval $COMMAND
|
||||
@@ -0,0 +1,37 @@
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:
|
||||
passed = []
|
||||
failed = []
|
||||
with open(res_file_path, 'r') as file:
|
||||
for line in file:
|
||||
data = json.loads(line.strip())
|
||||
instance_id = data['instance_id']
|
||||
resolved = False
|
||||
if 'test_result' in data and 'result' in data['test_result']:
|
||||
resolved = data['test_result']['result']
|
||||
if resolved:
|
||||
passed.append(instance_id)
|
||||
else:
|
||||
failed.append(instance_id)
|
||||
return passed, failed
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print(
|
||||
'Usage: poetry run python summarise_results.py <path_to_output_jsonl_file>'
|
||||
)
|
||||
sys.exit(1)
|
||||
json_file_path = sys.argv[1]
|
||||
passed_tests, failed_tests = extract_test_results(json_file_path)
|
||||
succ_rate = len(passed_tests) / (len(passed_tests) + len(failed_tests))
|
||||
print(
|
||||
f'\nPassed {len(passed_tests)} tests, failed {len(failed_tests)} tests, resolve rate = {succ_rate}'
|
||||
)
|
||||
print('PASSED TESTS:')
|
||||
print(passed_tests)
|
||||
print('FAILED TESTS:')
|
||||
print(failed_tests)
|
||||
Reference in New Issue
Block a user