github/OpenHands
1
1
Fork 0
mirror of https://github.com/All-Hands-AI/OpenHands.git synced 2026-01-10 07:18:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

logname fix; improve test calling instruction (#3666)

Browse Source
This commit is contained in:
tobitege
2024-08-30 17:15:31 +02:00
committed by GitHub
parent 99139e9c72
commit dbb671a8a5
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
evaluation/aider_bench/run_infer.py
View File

@@ -177,9 +177,10 @@ def process_instance(
signature_file=f'{instance.instance_name}.py',
)
if USE_UNIT_TESTS:
print(f'\nInstruction to run test_file: {instance.instance_name}_test.py\n')
instruction += (
f'Use the test_file: {instance.instance_name}_test.py, to verify '
'the correctness of your solution. DO NOT EDIT the test file.\n\n'
f'Use `python -m unittest {instance.instance_name}_test.py` to run the test_file '
'and verify the correctness of your solution. DO NOT EDIT the test file.\n\n'
)
instruction += (

3
evaluation/utils/shared.py
View File

@@ -139,13 +139,14 @@ def make_metadata(
details: dict[str, Any] | None = None,
) -> EvalMetadata:
model_name = llm_config.model.split('/')[-1]
model_path = model_name.replace(':', '_')
eval_note = f'_N_{eval_note}' if eval_note else ''
eval_output_path = os.path.join(
eval_output_dir,
dataset_name,
agent_class,
f'{model_name}_maxiter_{max_iterations}{eval_note}',
f'{model_path}_maxiter_{max_iterations}{eval_note}',
)
pathlib.Path(eval_output_path).mkdir(parents=True, exist_ok=True)