github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-01-10 23:58:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(benchmark): Include WebArenaSiteInfo.additional_info (e.g. credentials) in task input

Browse Source 
Without the `additional_info`, it is impossible to get past the login page on challenges where that is necessary.
This commit is contained in:
Reinier van der Leer
2024-02-16 17:18:49 +01:00
parent 23d58a3cc0
commit 2a55efb322
1 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
benchmark/agbenchmark/challenges/webarena.py
View File

@@ -37,7 +37,8 @@ site_info_map: dict[WebArenaSite, WebArenaSiteInfo] = {
base_url="http://git.junglegym.ai",
available=bool(_git_user and _git_password),
additional_info=(
f"To log in, use the username '{_git_user}' and password '{_git_password}'."
f"To log in to {{url}}, use the username '{_git_user}' "
f"and password '{_git_password}'."
),
unavailable_reason=(
"WEBARENA_GIT_CREDENTIALS not set (correctly): "
@@ -52,16 +53,22 @@ site_info_map: dict[WebArenaSite, WebArenaSiteInfo] = {
"shopping": WebArenaSiteInfo(base_url="http://shop.junglegym.ai"),
"shopping_admin": WebArenaSiteInfo(
base_url="http://cms.junglegym.ai/admin",
additional_info="To log in, use the username 'admin' and password 'admin1234'.",
additional_info=(
"To log in to {url}, use the username 'admin' and password 'admin1234'."
),
),
"wikipedia": WebArenaSiteInfo(base_url="http://wiki.junglegym.ai"),
}
def get_site_url(site: WebArenaSite) -> str:
def get_site_info(site: WebArenaSite) -> WebArenaSiteInfo:
if site not in site_info_map:
raise ValueError(f"JungleGym site '{site}' unknown, cannot resolve URL")
return site_info_map[site].base_url
return site_info_map[site]
def get_site_url(site: WebArenaSite) -> str:
return get_site_info(site).base_url
def resolve_uri(uri: str) -> str:
@@ -245,10 +252,15 @@ class WebArenaChallengeSpec(BaseModel):
@property
def assignment_for_agent(self):
sites = [get_site_url(s) for s in self.sites]
sites = [get_site_info(s) for s in self.sites]
nav_constraint = (
f"You are ONLY allowed to access URLs in {' and '.join(sites)}."
)
"You are ONLY allowed to access URLs in "
f"{' and '.join(s.base_url for s in sites)}.\n\n"
+ "\n".join(
s.additional_info.format(url=s.base_url)
for s in sites if s.additional_info
)
).strip()
return (
f"First of all, go to {self.start_url}. "