github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-08 22:38:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(eval): Support evaluation on SWE-rebench (#10251)

Browse Source
This commit is contained in:
Ibragim Badertdinov
2025-08-12 17:05:43 +03:00
committed by GitHub
parent 2b7e44819f
commit 19a6b6b618
5 changed files with 141 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
evaluation/benchmarks/swe_bench/run_infer.py
View File

@@ -80,6 +80,8 @@ def set_dataset_type(dataset_name: str) -> str:
DATASET_TYPE = 'SWE-Gym'
elif 'swe-bench-live' in name_lower:
DATASET_TYPE = 'SWE-bench-Live'
elif 'swe-rebench' in name_lower:
DATASET_TYPE = 'SWE-rebench'
elif 'multimodal' in name_lower:
DATASET_TYPE = 'Multimodal'
else:
@@ -178,6 +180,8 @@ def get_instance_docker_image(
docker_image_prefix = 'docker.io/starryzhang/'
elif DATASET_TYPE == 'SWE-bench':
docker_image_prefix = 'docker.io/swebench/'
elif DATASET_TYPE == 'SWE-rebench':
docker_image_prefix = 'docker.io/swerebench/'
repo, name = instance_id.split('__')
image_name = f'{docker_image_prefix.rstrip("/")}/sweb.eval.x86_64.{repo}_1776_{name}:latest'.lower()
logger.debug(f'Using official SWE-Bench image: {image_name}')
@@ -318,6 +322,8 @@ def initialize_runtime(
# inject the instance swe entry
if DATASET_TYPE == 'SWE-bench-Live':
entry_script_path = 'instance_swe_entry_live.sh'
elif DATASET_TYPE == 'SWE-rebench':
entry_script_path = 'instance_swe_entry_rebench.sh'
else:
entry_script_path = 'instance_swe_entry.sh'
runtime.copy_to(