mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2026-01-09 14:57:59 -05:00
Rename Conversation to ServerConversation and AppConfig to OpenHandsConfig (#8754)
Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
@@ -17,7 +17,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -59,10 +59,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -39,11 +39,11 @@ from openhands.utils.async_utils import call_async_from_sync
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-slim'
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
|
||||
@@ -24,7 +24,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
load_from_toml,
|
||||
parse_arguments,
|
||||
@@ -46,10 +46,10 @@ SKIP_NUM = (
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.11-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
|
||||
@@ -22,7 +22,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -55,12 +55,12 @@ FILE_EXT_MAP = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
BIOCODER_BENCH_CONTAINER_IMAGE = 'public.ecr.aws/i5g0m1f6/eval_biocoder:v1.0'
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = BIOCODER_BENCH_CONTAINER_IMAGE
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -70,11 +70,11 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -18,7 +18,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -33,13 +33,13 @@ SUPPORTED_AGENT_CLS = {'CodeActAgent'}
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
assert metadata.max_iterations == 1, (
|
||||
'max_iterations must be 1 for browsing delegation evaluation.'
|
||||
)
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -101,7 +101,7 @@ def get_instance_docker_image(repo_name: str) -> str:
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
repo_name = instance['repo'].split('/')[1]
|
||||
base_container_image = get_instance_docker_image(repo_name)
|
||||
logger.info(
|
||||
@@ -113,7 +113,7 @@ def get_config(
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = base_container_image
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
|
||||
@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -61,10 +61,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -47,10 +47,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -19,7 +19,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -39,10 +39,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -37,7 +37,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -60,10 +60,10 @@ ACTION_FORMAT = """
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -81,10 +81,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -19,10 +19,10 @@ from evaluation.utils.shared import (
|
||||
make_metadata,
|
||||
)
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
LLMConfig,
|
||||
OpenHandsConfig,
|
||||
get_parser,
|
||||
load_app_config,
|
||||
load_openhands_config,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime
|
||||
@@ -34,10 +34,10 @@ from openhands.utils.async_utils import call_async_from_sync
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
@@ -53,7 +53,7 @@ def get_config(
|
||||
return config
|
||||
|
||||
|
||||
config = load_app_config()
|
||||
config = load_openhands_config()
|
||||
|
||||
|
||||
def load_bench_config():
|
||||
|
||||
@@ -29,10 +29,10 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
load_app_config,
|
||||
load_openhands_config,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime, run_controller
|
||||
@@ -44,10 +44,10 @@ from openhands.utils.async_utils import call_async_from_sync
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
@@ -63,7 +63,7 @@ def get_config(
|
||||
return config
|
||||
|
||||
|
||||
config = load_app_config()
|
||||
config = load_openhands_config()
|
||||
|
||||
|
||||
def load_bench_config():
|
||||
|
||||
@@ -17,7 +17,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -44,14 +44,14 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'xingyaoww/od-eval-logic-reasoning:v1.0'
|
||||
sandbox_config.runtime_extra_deps = (
|
||||
'$OH_INTERPRETER_PATH -m pip install scitools-pyke'
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -54,10 +54,10 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
env_id: str,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'xingyaoww/od-eval-miniwob:v1.0'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
|
||||
@@ -22,7 +22,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -102,14 +102,14 @@ def load_incontext_example(task_name: str, with_tool: bool = True):
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'xingyaoww/od-eval-mint:v1.0'
|
||||
sandbox_config.runtime_extra_deps = (
|
||||
f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}'
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -4,11 +4,11 @@ import pprint
|
||||
|
||||
import tqdm
|
||||
|
||||
from openhands.core.config import get_llm_config_arg, get_parser, load_app_config
|
||||
from openhands.core.config import get_llm_config_arg, get_parser, load_openhands_config
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.llm.llm import LLM
|
||||
|
||||
config = load_app_config()
|
||||
config = load_openhands_config()
|
||||
|
||||
|
||||
def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:
|
||||
|
||||
@@ -33,10 +33,10 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
load_app_config,
|
||||
load_openhands_config,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime, run_controller
|
||||
@@ -45,7 +45,7 @@ from openhands.events.observation import CmdOutputObservation
|
||||
from openhands.runtime.base import Runtime
|
||||
from openhands.utils.async_utils import call_async_from_sync
|
||||
|
||||
config = load_app_config()
|
||||
config = load_openhands_config()
|
||||
|
||||
AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
|
||||
'CodeActAgent': codeact_user_response,
|
||||
@@ -76,10 +76,10 @@ ID2CONDA = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'public.ecr.aws/i5g0m1f6/ml-bench'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -28,8 +28,8 @@ from evaluation.utils.shared import (
|
||||
run_evaluation,
|
||||
)
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
LLMConfig,
|
||||
OpenHandsConfig,
|
||||
get_parser,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
@@ -73,7 +73,7 @@ def process_git_patch(patch):
|
||||
return patch
|
||||
|
||||
|
||||
def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
|
||||
def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
base_container_image = get_instance_docker_image(instance['instance_id'])
|
||||
logger.info(
|
||||
@@ -87,7 +87,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
|
||||
dataset_name=metadata.dataset,
|
||||
instance_id=instance['instance_id'],
|
||||
)
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
sandbox=sandbox_config,
|
||||
|
||||
@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -314,7 +314,7 @@ def get_instance_docker_image(instance: pd.Series):
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.2.1'
|
||||
if USE_INSTANCE_IMAGE:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
@@ -340,7 +340,7 @@ def get_config(
|
||||
instance_id=instance['instance_id'],
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
|
||||
@@ -20,7 +20,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -58,12 +58,12 @@ def format_task_dict(example, use_knowledge):
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
instance_id: str,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = (
|
||||
'docker.io/xingyaoww/openhands-eval-scienceagentbench'
|
||||
)
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
|
||||
@@ -24,8 +24,8 @@ from evaluation.utils.shared import (
|
||||
run_evaluation,
|
||||
)
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
LLMConfig,
|
||||
OpenHandsConfig,
|
||||
get_parser,
|
||||
)
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
@@ -69,7 +69,7 @@ def process_git_patch(patch):
|
||||
return patch
|
||||
|
||||
|
||||
def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
|
||||
def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
base_container_image = get_instance_docker_image(instance['instance_id'])
|
||||
logger.info(
|
||||
@@ -83,7 +83,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
|
||||
dataset_name=metadata.dataset,
|
||||
instance_id=instance['instance_id'],
|
||||
)
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
sandbox=sandbox_config,
|
||||
|
||||
@@ -40,12 +40,12 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
from openhands.core.config.utils import get_condenser_config_arg
|
||||
from openhands.core.config.condenser_config import NoOpCondenserConfig
|
||||
from openhands.core.config.utils import get_condenser_config_arg
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime, run_controller
|
||||
from openhands.critic import AgentFinishedCritic
|
||||
@@ -220,7 +220,7 @@ def get_instance_docker_image(
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
use_swebench_official_image = 'swe-gym' not in metadata.dataset.lower()
|
||||
base_container_image = get_instance_docker_image(
|
||||
@@ -244,7 +244,7 @@ def get_config(
|
||||
instance_id=instance['instance_id'],
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
@@ -721,15 +721,16 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
|
||||
# repos for the swe-bench instances:
|
||||
# ['astropy/astropy', 'django/django', 'matplotlib/matplotlib', 'mwaskom/seaborn', 'pallets/flask', 'psf/requests', 'pydata/xarray', 'pylint-dev/pylint', 'pytest-dev/pytest', 'scikit-learn/scikit-learn', 'sphinx-doc/sphinx', 'sympy/sympy']
|
||||
selected_repos = data['selected_repos']
|
||||
if isinstance(selected_repos, str): selected_repos = [selected_repos]
|
||||
if isinstance(selected_repos, str):
|
||||
selected_repos = [selected_repos]
|
||||
assert isinstance(selected_repos, list)
|
||||
logger.info(
|
||||
f'Filtering {selected_repos} tasks from "selected_repos"...'
|
||||
)
|
||||
subset = dataset[dataset["repo"].isin(selected_repos)]
|
||||
subset = dataset[dataset['repo'].isin(selected_repos)]
|
||||
logger.info(f'Retained {subset.shape[0]} tasks after filtering')
|
||||
return subset
|
||||
|
||||
|
||||
skip_ids = os.environ.get('SKIP_IDS', '').split(',')
|
||||
if len(skip_ids) > 0:
|
||||
logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
|
||||
@@ -806,7 +807,9 @@ if __name__ == '__main__':
|
||||
else:
|
||||
# If no specific condenser config is provided via env var, default to NoOpCondenser
|
||||
condenser_config = NoOpCondenserConfig()
|
||||
logger.debug('No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.')
|
||||
logger.debug(
|
||||
'No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.'
|
||||
)
|
||||
|
||||
details = {'mode': args.mode}
|
||||
_agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
|
||||
|
||||
@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -58,7 +58,7 @@ def _get_swebench_workspace_dir_name(instance: pd.Series) -> str:
|
||||
|
||||
|
||||
def get_instruction(instance: pd.Series, metadata: EvalMetadata):
|
||||
workspace_dir_name = _get_swebench_workspace_dir_name(instance)
|
||||
_get_swebench_workspace_dir_name(instance)
|
||||
instruction = f"""
|
||||
Consider the following issue description:
|
||||
|
||||
@@ -168,7 +168,7 @@ def get_instance_docker_image(instance_id: str, official_image: bool = False) ->
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
use_official_image = bool(
|
||||
'verified' in metadata.dataset.lower() or 'lite' in metadata.dataset.lower()
|
||||
@@ -197,7 +197,7 @@ def get_config(
|
||||
'REPO_PATH': f'/workspace/{workspace_dir_name}/',
|
||||
}
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
@@ -348,13 +348,13 @@ def initialize_runtime(
|
||||
|
||||
# Check if an existing graph index file is available
|
||||
graph_index_file_path = os.path.join(
|
||||
INDEX_BASE_DIR, 'graph_index_v2.3', f"{instance['instance_id']}.pkl"
|
||||
INDEX_BASE_DIR, 'graph_index_v2.3', f'{instance["instance_id"]}.pkl'
|
||||
)
|
||||
if INDEX_BASE_DIR and os.path.exists(graph_index_file_path):
|
||||
logger.info(
|
||||
f"Copying graph index from {graph_index_file_path} to /workspace/{workspace_dir_name}/_index_data/graph_index_v2.3"
|
||||
f'Copying graph index from {graph_index_file_path} to /workspace/{workspace_dir_name}/_index_data/graph_index_v2.3'
|
||||
)
|
||||
|
||||
|
||||
runtime.copy_to(
|
||||
graph_index_file_path,
|
||||
f'/workspace/{workspace_dir_name}/_index_data/graph_index_v2.3',
|
||||
@@ -364,9 +364,13 @@ def initialize_runtime(
|
||||
)
|
||||
obs = runtime.run_action(action)
|
||||
|
||||
bm25_index_dir = os.path.join(INDEX_BASE_DIR, 'BM25_index', instance['instance_id'])
|
||||
bm25_index_dir = os.path.join(
|
||||
INDEX_BASE_DIR, 'BM25_index', instance['instance_id']
|
||||
)
|
||||
runtime.copy_to(
|
||||
bm25_index_dir, f'/workspace/{workspace_dir_name}/_index_data', recursive=True
|
||||
bm25_index_dir,
|
||||
f'/workspace/{workspace_dir_name}/_index_data',
|
||||
recursive=True,
|
||||
)
|
||||
action = CmdRunAction(
|
||||
command=f'mv _index_data/{instance["instance_id"]} _index_data/bm25_index'
|
||||
|
||||
@@ -41,7 +41,7 @@ from evaluation.utils.shared import (
|
||||
reset_logger_for_multiprocessing,
|
||||
run_evaluation,
|
||||
)
|
||||
from openhands.core.config import AppConfig, SandboxConfig, get_parser
|
||||
from openhands.core.config import OpenHandsConfig, SandboxConfig, get_parser
|
||||
from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.core.main import create_runtime
|
||||
from openhands.events.action import CmdRunAction
|
||||
@@ -52,13 +52,13 @@ DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/kdja
|
||||
logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
|
||||
|
||||
|
||||
def get_config(instance: pd.Series) -> AppConfig:
|
||||
def get_config(instance: pd.Series) -> OpenHandsConfig:
|
||||
base_container_image = get_instance_docker_image(instance['instance_id_swebench'])
|
||||
assert base_container_image, (
|
||||
f'Invalid container image for instance {instance["instance_id_swebench"]}.'
|
||||
)
|
||||
logger.info(f'Using instance container image: {base_container_image}.')
|
||||
return AppConfig(
|
||||
return OpenHandsConfig(
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'eventstream'),
|
||||
sandbox=SandboxConfig(
|
||||
|
||||
@@ -35,7 +35,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
SandboxConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
@@ -117,7 +117,7 @@ def get_instance_docker_image(instance_id: str) -> str:
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of TestGenEval
|
||||
base_container_image = get_instance_docker_image(instance['instance_id_swebench'])
|
||||
logger.info(
|
||||
@@ -126,7 +126,7 @@ def get_config(
|
||||
f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
|
||||
@@ -15,8 +15,8 @@ from browsing import pre_login
|
||||
from evaluation.utils.shared import get_default_sandbox_config_for_eval
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
LLMConfig,
|
||||
OpenHandsConfig,
|
||||
get_agent_config_arg,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
@@ -36,13 +36,13 @@ def get_config(
|
||||
mount_path_on_host: str,
|
||||
llm_config: LLMConfig,
|
||||
agent_config: AgentConfig | None,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = base_container_image
|
||||
sandbox_config.enable_auto_lint = True
|
||||
# If the web services are running on the host machine, this must be set to True
|
||||
sandbox_config.use_host_network = True
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
run_as_openhands=False,
|
||||
max_budget_per_task=4,
|
||||
max_iterations=100,
|
||||
@@ -126,7 +126,7 @@ def codeact_user_response(state: State) -> str:
|
||||
def run_solver(
|
||||
runtime: Runtime,
|
||||
task_name: str,
|
||||
config: AppConfig,
|
||||
config: OpenHandsConfig,
|
||||
dependencies: list[str],
|
||||
save_final_state: bool,
|
||||
state_dir: str,
|
||||
@@ -274,7 +274,7 @@ if __name__ == '__main__':
|
||||
temp_dir = os.path.abspath(os.getenv('TMPDIR'))
|
||||
else:
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
config: AppConfig = get_config(
|
||||
config: OpenHandsConfig = get_config(
|
||||
args.task_image_name, task_short_name, temp_dir, agent_llm_config, agent_config
|
||||
)
|
||||
runtime: Runtime = create_runtime(config)
|
||||
|
||||
@@ -18,7 +18,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -40,10 +40,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
|
||||
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.base_container_image = 'python:3.12-bookworm'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
get_parser,
|
||||
)
|
||||
@@ -135,7 +135,7 @@ def get_instance_docker_image(instance_id: str, official_image: bool = False) ->
|
||||
def get_config(
|
||||
instance: pd.Series,
|
||||
metadata: EvalMetadata,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
# We use a different instance image for the each instance of swe-bench eval
|
||||
use_official_image = bool(
|
||||
'verified' in metadata.dataset.lower() or 'lite' in metadata.dataset.lower()
|
||||
@@ -160,7 +160,7 @@ def get_config(
|
||||
instance_id=instance['instance_id'],
|
||||
)
|
||||
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
max_iterations=metadata.max_iterations,
|
||||
|
||||
@@ -20,7 +20,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -48,7 +48,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
env_id: str,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
base_url = os.environ.get('VISUALWEBARENA_BASE_URL', None)
|
||||
openai_api_key = os.environ.get('OPENAI_API_KEY', None)
|
||||
openai_base_url = os.environ.get('OPENAI_BASE_URL', None)
|
||||
@@ -72,7 +72,7 @@ def get_config(
|
||||
'VWA_WIKIPEDIA': f'{base_url}:8888',
|
||||
'VWA_HOMEPAGE': f'{base_url}:4399',
|
||||
}
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -19,7 +19,7 @@ from evaluation.utils.shared import (
|
||||
)
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -44,7 +44,7 @@ SUPPORTED_AGENT_CLS = {'BrowsingAgent'}
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
env_id: str,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
base_url = os.environ.get('WEBARENA_BASE_URL', None)
|
||||
openai_api_key = os.environ.get('OPENAI_API_KEY', None)
|
||||
assert base_url is not None, 'WEBARENA_BASE_URL must be set'
|
||||
@@ -64,7 +64,7 @@ def get_config(
|
||||
'MAP': f'{base_url}:3000',
|
||||
'HOMEPAGE': f'{base_url}:4399',
|
||||
}
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime='docker',
|
||||
|
||||
@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
|
||||
from openhands.controller.state.state import State
|
||||
from openhands.core.config import (
|
||||
AgentConfig,
|
||||
AppConfig,
|
||||
OpenHandsConfig,
|
||||
get_llm_config_arg,
|
||||
parse_arguments,
|
||||
)
|
||||
@@ -41,10 +41,10 @@ FAKE_RESPONSES = {
|
||||
def get_config(
|
||||
metadata: EvalMetadata,
|
||||
instance_id: str,
|
||||
) -> AppConfig:
|
||||
) -> OpenHandsConfig:
|
||||
sandbox_config = get_default_sandbox_config_for_eval()
|
||||
sandbox_config.platform = 'linux/amd64'
|
||||
config = AppConfig(
|
||||
config = OpenHandsConfig(
|
||||
default_agent=metadata.agent_class,
|
||||
run_as_openhands=False,
|
||||
runtime=os.environ.get('RUNTIME', 'docker'),
|
||||
|
||||
@@ -2,9 +2,9 @@ import argparse
|
||||
|
||||
import pytest
|
||||
|
||||
from openhands.config import load_app_config
|
||||
from openhands.config import load_openhands_config
|
||||
|
||||
config = load_app_config()
|
||||
config = load_openhands_config()
|
||||
|
||||
if __name__ == '__main__':
|
||||
"""Main entry point of the script.
|
||||
|
||||
Reference in New Issue
Block a user