Fix email update to use /api/settings endpoint

Co-authored-by: openhands <openhands@all-hands.dev>

.devcontainer/devcontainer.json

@@ -0,0 +1,15 @@
+// For format details, see: https://aka.ms/devcontainer.json
+{
+	"name": "Python 3",
+	// Documentation for this image:
+	// - https://github.com/devcontainers/templates/tree/main/src/python
+	// - https://github.com/microsoft/vscode-remote-try-python
+	// - https://hub.docker.com/r/microsoft/devcontainers-python
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+	"features": {
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+		"ghcr.io/devcontainers-extra/features/poetry:2": {},
+		"ghcr.io/devcontainers/features/node:1": {},
+	},
+	"postCreateCommand": ".devcontainer/setup.sh",
+}

.devcontainer/setup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Install `nc`
+sudo apt update && sudo apt install netcat -y
+
+# Do common setup tasks
+source .openhands/setup.sh

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+# force *nix line endings so files don't look modified in container run from Windows clone
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true

.gitattributes

@@ -1 +1,7 @@
 *.ipynb linguist-vendored
+
+# force *nix line endings so files don't look modified in container run from Windows clone
+* text eol=lf
+# Git incorrectly thinks some media is text
+*.png -text
+*.mp4 -text

.github/workflows/openhands-resolver.yml

@@ -24,7 +24,7 @@ on:
       LLM_MODEL:
         required: false
         type: string
-        default: "anthropic/claude-3-7-sonnet-20250219"
+        default: "anthropic/claude-sonnet-4-20250514"
       LLM_API_VERSION:
         required: false
         type: string

.gitignore

@@ -161,8 +161,28 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
-.vscode/
+
+# VS Code: Ignore all but certain files that specify repo-specific settings.
+# https://stackoverflow.com/questions/32964920/should-i-commit-the-vscode-folder-to-source-control
+.vscode/**/*
+!.vscode/extensions.json
+!.vscode/launch.json
+!.vscode/settings.json
+!.vscode/tasks.json
+
+# VS Code extensions/forks:
 .cursorignore
+.rooignore
+.clineignore
+.windsurfignore
+.cursorrules
+.roorules
+.clinerules
+.windsurfrules
+.cursor/rules
+.roo/rules
+.cline/rules
+.windsurf/rules
 
 # evaluation
 evaluation/evaluation_outputs

.openhands/setup.sh

@@ -9,4 +9,5 @@ python -m pip install pre-commit
 if [ -d ".git" ]; then
     echo "Installing pre-commit hooks..."
     pre-commit install
+    make install-pre-commit-hooks
 fi

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    // force *nix line endings so files don't look modified in container run from Windows clone
+    "files.eol": "\n",
+    "files.trimTrailingWhitespace": true,
+    "files.insertFinalNewline": true,
+}

Development.md

@@ -1,8 +1,10 @@
 # Development Guide
 
 This guide is for people working on OpenHands and editing the source code.
-If you wish to contribute your changes, check out the [CONTRIBUTING.md](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md) on how to clone and setup the project 
-initially before moving on. Otherwise, you can clone the OpenHands project directly.
+If you wish to contribute your changes, check out the
+[CONTRIBUTING.md](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md)
+on how to clone and setup the project initially before moving on. Otherwise,
+you can clone the OpenHands project directly.
 
 ## Start the Server for Development
 
@@ -19,9 +21,20 @@ initially before moving on. Otherwise, you can clone the OpenHands project direc
 
 Make sure you have all these dependencies installed before moving on to `make build`.
 
+#### Dev container
+
+There is a [dev container](https://containers.dev/) available which provides a
+pre-configured environment with all the necessary dependencies installed if you
+are using a [supported editor or tool](https://containers.dev/supporting). For
+example, if you are using Visual Studio Code (VS Code) with the
+[Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+extension installed, you can open the project in a dev container by using the
+_Dev Container: Reopen in Container_ command from the Command Palette
+(Ctrl+Shift+P).
+
 #### Develop without sudo access
 
-If you want to develop without system admin/sudo access to upgrade/install `Python` and/or `NodeJs`, you can use 
+If you want to develop without system admin/sudo access to upgrade/install `Python` and/or `NodeJs`, you can use
 `conda` or `mamba` to manage the packages for you:
 
 ```bash
@@ -37,7 +50,7 @@ mamba install conda-forge::poetry
 
 ### 2. Build and Setup The Environment
 
-Begin by building the project which includes setting up the environment and installing dependencies. This step ensures 
+Begin by building the project which includes setting up the environment and installing dependencies. This step ensures
 that OpenHands is ready to run on your system:
 
 ```bash
@@ -54,11 +67,11 @@ To configure the LM of your choice, run:
 make setup-config
 ```
 
-This command will prompt you to enter the LLM API key, model name, and other variables ensuring that OpenHands is 
-tailored to your specific needs. Note that the model name will apply only when you run headless. If you use the UI, 
+This command will prompt you to enter the LLM API key, model name, and other variables ensuring that OpenHands is
+tailored to your specific needs. Note that the model name will apply only when you run headless. If you use the UI,
 please set the model in the UI.
 
-Note: If you have previously run OpenHands using the docker command, you may have already set some environmental 
+Note: If you have previously run OpenHands using the docker command, you may have already set some environmental
 variables in your terminal. The final configurations are set from highest to lowest priority:
 Environment variables > config.toml variables > default variables
 
@@ -77,14 +90,14 @@ make run
 
 #### Option B: Individual Server Startup
 
-- **Start the Backend Server:** If you prefer, you can start the backend server independently to focus on 
+- **Start the Backend Server:** If you prefer, you can start the backend server independently to focus on
 backend-related tasks or configurations.
 
   ```bash
   make start-backend
   ```
 
-- **Start the Frontend Server:** Similarly, you can start the frontend server on its own to work on frontend-related 
+- **Start the Frontend Server:** Similarly, you can start the frontend server on its own to work on frontend-related
 components or interface enhancements.
   ```bash
   make start-frontend
@@ -120,7 +133,7 @@ poetry run pytest ./tests/unit/test_*.py
 
 ### 9. Use existing Docker image
 
-To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker 
+To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker
 container image by setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
 
 Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.39-nikolaik`

README.md

@@ -67,7 +67,7 @@ docker run -it --rm --pull=always \
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
 
 When you open the application, you'll be asked to choose an LLM provider and add an API key.
-[Anthropic's Claude 3.7 Sonnet](https://www.anthropic.com/api) (`anthropic/claude-3-7-sonnet-20250219`)
+[Anthropic's Claude Sonnet 4](https://www.anthropic.com/api) (`anthropic/claude-sonnet-4-20250514`)
 works best, but you have [many options](https://docs.all-hands.dev/modules/usage/llms).
 
 ## 💡 Other ways to run OpenHands

README_CN.md

@@ -0,0 +1,146 @@
+
+<a name="readme-top"></a>
+
+<div align="center">
+  <img src="./docs/static/img/logo.png" alt="Logo" width="200">
+  <h1 align="center">OpenHands: 少写代码，多做事</h1>
+</div>
+
+
+<div align="center">
+  <a href="https://github.com/All-Hands-AI/OpenHands/graphs/contributors"><img src="https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Contributors"></a>
+  <a href="https://github.com/All-Hands-AI/OpenHands/stargazers"><img src="https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Stargazers"></a>
+  <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
+  <br/>
+  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-34zm4j0gj-Qz5kRHoca8DFCbqXPS~f_A"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="加入我们的Slack社区"></a>
+  <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="加入我们的Discord社区"></a>
+  <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="致谢"></a>
+  <br/>
+  <a href="https://docs.all-hands.dev/modules/usage/getting-started"><img src="https://img.shields.io/badge/Documentation-000?logo=googledocs&logoColor=FFE165&style=for-the-badge" alt="查看文档"></a>
+  <a href="https://arxiv.org/abs/2407.16741"><img src="https://img.shields.io/badge/Paper%20on%20Arxiv-000?logoColor=FFE165&logo=arxiv&style=for-the-badge" alt="Arxiv论文"></a>
+  <a href="https://docs.google.com/spreadsheets/d/1wOUdFCMyY6Nt0AIqF705KN4JKOWgeI4wUGUP60krXXs/edit?gid=0#gid=0"><img src="https://img.shields.io/badge/Benchmark%20score-000?logoColor=FFE165&logo=huggingface&style=for-the-badge" alt="评估基准分数"></a>
+  <hr>
+</div>
+
+欢迎使用OpenHands（前身为OpenDevin），这是一个由AI驱动的软件开发代理平台。
+
+OpenHands代理可以完成人类开发者能做的任何事情：修改代码、运行命令、浏览网页、调用API，甚至从StackOverflow复制代码片段。
+
+在[docs.all-hands.dev](https://docs.all-hands.dev)了解更多信息，或[注册OpenHands Cloud](https://app.all-hands.dev)开始使用。
+
+> [!IMPORTANT]
+> 在工作中使用OpenHands？我们很想与您交流！填写
+> [这份简短表格](https://docs.google.com/forms/d/e/1FAIpQLSet3VbGaz8z32gW9Wm-Grl4jpt5WgMXPgJ4EDPVmCETCBpJtQ/viewform)
+> 加入我们的设计合作伙伴计划，您将获得商业功能的早期访问权限，并有机会对我们的产品路线图提供意见。
+
+![应用截图](./docs/static/img/screenshot.png)
+
+## ☁️ OpenHands Cloud
+开始使用OpenHands的最简单方式是在[OpenHands Cloud](https://app.all-hands.dev)上，
+新用户可获得$50的免费额度。
+
+## 💻 在本地运行OpenHands
+
+OpenHands也可以使用Docker在本地系统上运行。
+查看[运行OpenHands](https://docs.all-hands.dev/modules/usage/installation)指南了解
+系统要求和更多信息。
+
+> [!WARNING]
+> 在公共网络上？请参阅我们的[强化Docker安装指南](https://docs.all-hands.dev/modules/usage/runtimes/docker#hardened-docker-installation)
+> 通过限制网络绑定和实施其他安全措施来保护您的部署。
+
+
+```bash
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.39-nikolaik
+
+docker run -it --rm --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.39-nikolaik \
+    -e LOG_ALL_EVENTS=true \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v ~/.openhands-state:/.openhands-state \
+    -p 3000:3000 \
+    --add-host host.docker.internal:host-gateway \
+    --name openhands-app \
+    docker.all-hands.dev/all-hands-ai/openhands:0.39
+```
+
+您将在[http://localhost:3000](http://localhost:3000)找到运行中的OpenHands！
+
+打开应用程序时，您将被要求选择一个LLM提供商并添加API密钥。
+[Anthropic的Claude Sonnet 4](https://www.anthropic.com/api)（`anthropic/claude-sonnet-4-20250514`）
+效果最佳，但您还有[许多选择](https://docs.all-hands.dev/modules/usage/llms)。
+
+## 💡 运行OpenHands的其他方式
+
+> [!CAUTION]
+> OpenHands旨在由单个用户在其本地工作站上运行。
+> 它不适合多租户部署，即多个用户共享同一实例。没有内置的身份验证、隔离或可扩展性。
+>
+> 如果您有兴趣在多租户环境中运行OpenHands，请
+> [与我们联系](https://docs.google.com/forms/d/e/1FAIpQLSet3VbGaz8z32gW9Wm-Grl4jpt5WgMXPgJ4EDPVmCETCBpJtQ/viewform)
+> 了解高级部署选项。
+
+您还可以[将OpenHands连接到本地文件系统](https://docs.all-hands.dev/modules/usage/runtimes/docker#connecting-to-your-filesystem)，
+以可编程的[无头模式](https://docs.all-hands.dev/modules/usage/how-to/headless-mode)运行OpenHands，
+通过[友好的CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode)与其交互，
+或使用[GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action)在标记的问题上运行它。
+
+访问[运行OpenHands](https://docs.all-hands.dev/modules/usage/installation)获取更多信息和设置说明。
+
+如果您想修改OpenHands源代码，请查看[Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md)。
+
+遇到问题？[故障排除指南](https://docs.all-hands.dev/modules/usage/troubleshooting)可以提供帮助。
+
+## 📖 文档
+  <a href="https://deepwiki.com/All-Hands-AI/OpenHands"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki" title="DeepWiki自动生成文档"></a>
+
+要了解有关项目的更多信息，以及使用OpenHands的技巧，
+请查看我们的[文档](https://docs.all-hands.dev/modules/usage/getting-started)。
+
+在那里，您将找到有关如何使用不同LLM提供商、
+故障排除资源和高级配置选项的资源。
+
+## 🤝 如何加入社区
+
+OpenHands是一个社区驱动的项目，我们欢迎每个人的贡献。我们大部分沟通
+通过Slack进行，因此这是开始的最佳场所，但我们也很乐意您通过Discord或Github与我们联系：
+
+- [加入我们的Slack工作空间](https://join.slack.com/t/openhands-ai/shared_invite/zt-34zm4j0gj-Qz5kRHoca8DFCbqXPS~f_A) - 这里我们讨论研究、架构和未来发展。
+- [加入我们的Discord服务器](https://discord.gg/ESHStjSjD4) - 这是一个社区运营的服务器，用于一般讨论、问题和反馈。
+- [阅读或发布Github问题](https://github.com/All-Hands-AI/OpenHands/issues) - 查看我们正在处理的问题，或添加您自己的想法。
+
+在[COMMUNITY.md](./COMMUNITY.md)中了解更多关于社区的信息，或在[CONTRIBUTING.md](./CONTRIBUTING.md)中找到有关贡献的详细信息。
+
+## 📈 进展
+
+在[这里](https://github.com/orgs/All-Hands-AI/projects/1)查看OpenHands月度路线图（每月月底在维护者会议上更新）。
+
+<p align="center">
+  <a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
+    <img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">
+  </a>
+</p>
+
+## 📜 许可证
+
+根据MIT许可证分发。有关更多信息，请参阅[`LICENSE`](./LICENSE)。
+
+## 🙏 致谢
+
+OpenHands由大量贡献者构建，每一份贡献都备受感谢！我们还借鉴了其他开源项目，对他们的工作深表感谢。
+
+有关OpenHands中使用的开源项目和许可证列表，请参阅我们的[CREDITS.md](./CREDITS.md)文件。
+
+## 📚 引用
+
+```
+@misc{openhands,
+      title={{OpenHands: An Open Platform for AI Software Developers as Generalist Agents}},
+      author={Xingyao Wang and Boxuan Li and Yufan Song and Frank F. Xu and Xiangru Tang and Mingchen Zhuge and Jiayi Pan and Yueqi Song and Bowen Li and Jaskirat Singh and Hoang H. Tran and Fuqiang Li and Ren Ma and Mingzhang Zheng and Bill Qian and Yanjun Shao and Niklas Muennighoff and Yizhe Zhang and Binyuan Hui and Junyang Lin and Robert Brennan and Hao Peng and Heng Ji and Graham Neubig},
+      year={2024},
+      eprint={2407.16741},
+      archivePrefix={arXiv},
+      primaryClass={cs.SE},
+      url={https://arxiv.org/abs/2407.16741},
+}
+```

config.template.toml

@@ -325,6 +325,15 @@ classpath = "my_package.my_module.MyCustomAgent"
 # Useful when deploying OpenHands in a remote machine where you need to expose a specific port.
 #vscode_port = 41234
 
+# Volume mounts in the format 'host_path:container_path[:mode]'
+# e.g. '/my/host/dir:/workspace:rw'
+# Multiple mounts can be specified using commas
+# e.g. '/path1:/workspace/path1,/path2:/workspace/path2:ro'
+
+# Configure volumes under the [sandbox] section:
+# [sandbox]
+# volumes = "/my/host/dir:/workspace:rw,/path2:/workspace/path2:ro"
+
 #################################### Security ###################################
 # Configuration for security features
 ##############################################################################

docs/README.md

@@ -52,4 +52,4 @@ $ poetry run python docs/translation_updater.py
 # ...
 ```
 
-This process uses `claude-3-7-sonnet-20250219` as base model and each language consumes at least ~30k input tokens and ~35k output tokens.
+This process uses `claude-sonnet-4-20250514` as base model and each language consumes at least ~30k input tokens and ~35k output tokens.

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/cloud/cloud-issue-resolver.md

@@ -8,18 +8,30 @@ Le Résolveur de Problèmes Cloud est disponible automatiquement lorsque vous ac
 - [Accès au dépôt GitHub](./github-installation#adding-repository-access)
 - [Accès au dépôt GitLab](./gitlab-installation#adding-repository-access)
 
+![Ajout d'accès au dépôt à OpenHands](/img/cloud/add-repo.png)
+
 ## Utilisation
 
 Après avoir accordé l'accès au dépôt OpenHands Cloud, vous pouvez utiliser le Résolveur de Problèmes Cloud sur les problèmes et les pull/merge requests dans vos dépôts.
 
 ### Travailler avec les Problèmes
 
-Sur votre dépôt, étiquetez un problème avec `openhands`. OpenHands va :
+Sur votre dépôt, étiquetez un problème avec `openhands` ou ajoutez un message commençant par `@openhands`. OpenHands va :
 1. Commenter le problème pour vous faire savoir qu'il y travaille
    - Vous pouvez cliquer sur le lien pour suivre la progression sur OpenHands Cloud
 2. Ouvrir une pull request (GitHub) ou une merge request (GitLab) s'il détermine que le problème a été résolu avec succès
 3. Commenter le problème avec un résumé des tâches effectuées et un lien vers la PR/MR
 
+![Résolveur de problèmes OpenHands en action](/img/cloud/issue-resolver.png)
+
+#### Exemples de Commandes pour les Problèmes
+
+Voici quelques exemples de commandes que vous pouvez utiliser avec le résolveur de problèmes :
+
+```
+@openhands lisez la description du problème et corrigez-le
+```
+
 ### Travailler avec les Pull/Merge Requests
 
 Pour qu'OpenHands travaille sur les pull requests (GitHub) ou les merge requests (GitLab), mentionnez `@openhands` dans les commentaires pour :
@@ -30,3 +42,15 @@ Pour qu'OpenHands travaille sur les pull requests (GitHub) ou les merge requests
 OpenHands va :
 1. Commenter pour vous faire savoir qu'il y travaille
 2. Effectuer la tâche demandée
+
+#### Exemples de Commandes pour les Pull/Merge Requests
+
+Voici quelques exemples de commandes que vous pouvez utiliser avec les pull/merge requests :
+
+```
+@openhands reflétez les commentaires de la revue
+```
+
+```
+@openhands corrigez les conflits de fusion et assurez-vous que le CI passe
+```

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/cloud/github-installation.md

@@ -36,6 +36,8 @@ Vous pouvez accorder à OpenHands l'accès à des dépôts spécifiques :
      - Permissions GitHub de l'utilisateur (propriétaire/collaborateur)
 3. Cliquez sur `Installer & Autoriser`
 
+![Ajout de l'accès au dépôt à OpenHands](/img/cloud/add-repo.png)
+
 ## Modification de l'Accès au Dépôt
 
 Vous pouvez modifier l'accès au dépôt à tout moment :

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -13,7 +13,7 @@ recommandations pour la sélection de modèles. Nos derniers résultats d'évalu
 
 Sur la base de ces résultats et des retours de la communauté, les modèles suivants ont été vérifiés comme fonctionnant raisonnablement bien avec OpenHands :
 
-- [anthropic/claude-3-7-sonnet-20250219](https://www.anthropic.com/api) (recommandé)
+- [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api) (recommandé)
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)
 - [openai/o3-mini](https://openai.com/index/openai-o3-mini/)

docs/i18n/ja/docusaurus-plugin-content-docs/current/usage/cloud/cloud-issue-resolver.md

@@ -8,18 +8,30 @@
 - [GitHubリポジトリアクセス](./github-installation#adding-repository-access)
 - [GitLabリポジトリアクセス](./gitlab-installation#adding-repository-access)
 
+![OpenHandsにリポジトリアクセスを追加する](/img/cloud/add-repo.png)
+
 ## 使用方法
 
 OpenHands Cloudリポジトリアクセスを許可した後、リポジトリの課題やプルリクエスト/マージリクエストでクラウド課題リゾルバーを使用できます。
 
 ### 課題の操作
 
-リポジトリで、課題に`openhands`というラベルを付けます。OpenHandsは以下を行います：
+リポジトリで、課題に`openhands`というラベルを付けるか、`@openhands`で始まるメッセージを追加します。OpenHandsは以下を行います：
 1. 課題にコメントして、作業中であることを知らせます
    - リンクをクリックすると、OpenHands Cloudで進捗状況を追跡できます
 2. 課題が正常に解決されたと判断した場合、プルリクエスト（GitHub）またはマージリクエスト（GitLab）を開きます
 3. 実行されたタスクの概要とPR/MRへのリンクを含むコメントを課題に残します
 
+![OpenHands課題リゾルバーの動作](/img/cloud/issue-resolver.png)
+
+#### 課題用のコマンド例
+
+以下は、課題リゾルバーで使用できるコマンドの例です：
+
+```
+@openhands 課題の説明を読んで修正してください
+```
+
 ### プルリクエスト/マージリクエストの操作
 
 プルリクエスト（GitHub）またはマージリクエスト（GitLab）でOpenHandsを動作させるには、コメントで`@openhands`を言及して以下を行います：
@@ -30,3 +42,15 @@ OpenHands Cloudリポジトリアクセスを許可した後、リポジトリ
 OpenHandsは以下を行います：
 1. 作業中であることを知らせるコメントをします
 2. リクエストされたタスクを実行します
+
+#### プルリクエスト/マージリクエスト用のコマンド例
+
+以下は、プルリクエスト/マージリクエストで使用できるコマンドの例です：
+
+```
+@openhands レビューコメントを反映してください
+```
+
+```
+@openhands マージの競合を修正し、CIが通過することを確認してください
+```

docs/i18n/ja/docusaurus-plugin-content-docs/current/usage/cloud/github-installation.md

@@ -36,6 +36,8 @@
      - ユーザーのGitHub権限（所有者/コラボレーター）
 3. `インストール＆承認`をクリックします
 
+![OpenHandsへのリポジトリアクセスの追加](/img/cloud/add-repo.png)
+
 ## リポジトリアクセスの変更
 
 リポジトリアクセスはいつでも変更できます：

docs/i18n/ja/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -13,7 +13,7 @@ OpenHandsはLiteLLMでサポートされているあらゆるLLMに接続でき
 
 これらの調査結果とコミュニティからのフィードバックに基づき、以下のモデルはOpenHandsでうまく動作することが確認されています：
 
-- [anthropic/claude-3-7-sonnet-20250219](https://www.anthropic.com/api) (推奨)
+- [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api) (推奨)
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)
 - [openai/o3-mini](https://openai.com/index/openai-o3-mini/)

docs/i18n/pt-BR/docusaurus-plugin-content-docs/current/usage/cloud/cloud-issue-resolver.md

@@ -8,18 +8,30 @@ O Resolvedor de Problemas na Nuvem está disponível automaticamente quando voc
 - [Acesso ao repositório GitHub](./github-installation#adding-repository-access)
 - [Acesso ao repositório GitLab](./gitlab-installation#adding-repository-access)
 
+![Adicionando acesso ao repositório ao OpenHands](/img/cloud/add-repo.png)
+
 ## Uso
 
 Após conceder acesso ao repositório OpenHands Cloud, você pode usar o Resolvedor de Problemas na Nuvem em issues e pull/merge requests em seus repositórios.
 
 ### Trabalhando com Issues
 
-No seu repositório, rotule uma issue com `openhands`. O OpenHands irá:
+No seu repositório, rotule uma issue com `openhands` ou adicione uma mensagem começando com `@openhands`. O OpenHands irá:
 1. Comentar na issue para informar que está trabalhando nela
    - Você pode clicar no link para acompanhar o progresso no OpenHands Cloud
 2. Abrir um pull request (GitHub) ou merge request (GitLab) se determinar que o problema foi resolvido com sucesso
 3. Comentar na issue com um resumo das tarefas realizadas e um link para o PR/MR
 
+![OpenHands resolvedor de problemas em ação](/img/cloud/issue-resolver.png)
+
+#### Exemplos de Comandos para Issues
+
+Aqui estão alguns exemplos de comandos que você pode usar com o resolvedor de problemas:
+
+```
+@openhands leia a descrição do problema e corrija-o
+```
+
 ### Trabalhando com Pull/Merge Requests
 
 Para fazer o OpenHands trabalhar em pull requests (GitHub) ou merge requests (GitLab), mencione `@openhands` nos comentários para:
@@ -30,3 +42,15 @@ Para fazer o OpenHands trabalhar em pull requests (GitHub) ou merge requests (Gi
 O OpenHands irá:
 1. Comentar para informar que está trabalhando nisso
 2. Realizar a tarefa solicitada
+
+#### Exemplos de Comandos para Pull/Merge Requests
+
+Aqui estão alguns exemplos de comandos que você pode usar com pull/merge requests:
+
+```
+@openhands reflita os comentários da revisão
+```
+
+```
+@openhands corrija os conflitos de merge e certifique-se de que o CI passa
+```

docs/i18n/pt-BR/docusaurus-plugin-content-docs/current/usage/cloud/github-installation.md

@@ -36,6 +36,8 @@ Você pode conceder ao OpenHands acesso a repositórios específicos:
      - Permissões do GitHub do usuário (proprietário/colaborador)
 3. Clique em `Instalar e Autorizar`
 
+![Adicionando acesso ao repositório ao OpenHands](/img/cloud/add-repo.png)
+
 ## Modificando o Acesso ao Repositório
 
 Você pode modificar o acesso ao repositório a qualquer momento:

docs/i18n/pt-BR/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -13,7 +13,7 @@ recomendações para seleção de modelos. Nossos resultados de benchmarking mai
 
 Com base nessas descobertas e feedback da comunidade, os seguintes modelos foram verificados e funcionam razoavelmente bem com o OpenHands:
 
-- [anthropic/claude-3-7-sonnet-20250219](https://www.anthropic.com/api) (recomendado)
+- [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api) (recomendado)
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)
 - [openai/o3-mini](https://openai.com/index/openai-o3-mini/)

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/cloud/cloud-issue-resolver.md

@@ -8,18 +8,30 @@
 - [GitHub仓库访问](./github-installation#adding-repository-access)
 - [GitLab仓库访问](./gitlab-installation#adding-repository-access)
 
+![向OpenHands添加仓库访问权限](/img/cloud/add-repo.png)
+
 ## 使用方法
 
 授予OpenHands Cloud仓库访问权限后，您可以在仓库的问题和拉取/合并请求中使用云问题解决器。
 
 ### 处理问题
 
-在您的仓库中，给问题添加`openhands`标签。OpenHands将会：
+在您的仓库中，给问题添加`openhands`标签或添加以`@openhands`开头的消息。OpenHands将会：
 1. 在问题上发表评论，让您知道它正在处理
    - 您可以点击链接在OpenHands Cloud上跟踪进度
 2. 如果确定问题已成功解决，则打开拉取请求（GitHub）或合并请求（GitLab）
 3. 在问题上发表评论，总结已执行的任务并提供PR/MR的链接
 
+![OpenHands问题解决器运行中](/img/cloud/issue-resolver.png)
+
+#### 问题命令示例
+
+以下是您可以在问题解决器中使用的命令示例：
+
+```
+@openhands 阅读问题描述并修复它
+```
+
 ### 处理拉取/合并请求
 
 要让OpenHands处理拉取请求（GitHub）或合并请求（GitLab），在评论中提及`@openhands`以：
@@ -30,3 +42,15 @@
 OpenHands将会：
 1. 发表评论让您知道它正在处理
 2. 执行请求的任务
+
+#### 拉取/合并请求命令示例
+
+以下是您可以在拉取/合并请求中使用的命令示例：
+
+```
+@openhands 反映审查评论
+```
+
+```
+@openhands 修复合并冲突并确保CI通过
+```

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/cloud/github-installation.md

@@ -36,6 +36,8 @@
      - 用户的GitHub权限（所有者/协作者）
 3. 点击`安装并授权`
 
+![向OpenHands添加仓库访问权限](/img/cloud/add-repo.png)
+
 ## 修改仓库访问权限
 
 您可以随时修改仓库访问权限：

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -12,7 +12,7 @@ OpenHands 可以连接到任何 LiteLLM 支持的 LLM。但是，它需要一个
 
 基于这些发现和社区反馈，以下模型已被验证可以与 OpenHands 合理地配合使用：
 
-- [anthropic/claude-3-7-sonnet-20250219](https://www.anthropic.com/api)（推荐）
+- [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api)（推荐）
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)
 - [openai/o3-mini](https://openai.com/index/openai-o3-mini/)

docs/modules/usage/cloud/cloud-issue-resolver.md

@@ -7,6 +7,8 @@ The Cloud Issue Resolver automates code fixes and provides intelligent assistanc
 The Cloud Issue Resolver is available automatically when you grant OpenHands Cloud repository access:
 - [GitHub repository access](./github-installation#adding-repository-access)
 
+![Adding repository access to OpenHands](/img/cloud/add-repo.png)
+
 ## Usage
 
 After granting OpenHands Cloud repository access, you can use the Cloud Issue Resolver on issues and pull requests in your repositories.
@@ -20,6 +22,16 @@ On your repository, label an issue with `openhands` or add a message starting wi
 2. Open a pull request if it determines that the issue has been successfully resolved
 3. Comment on the issue with a summary of the performed tasks and a link to the PR
 
+![OpenHands issue resolver in action](/img/cloud/issue-resolver.png)
+
+#### Example Commands for Issues
+
+Here are some examples of commands you can use with the issue resolver:
+
+```
+@openhands read the issue description and fix it
+```
+
 ### Working with Pull Requests
 
 To get OpenHands to work on pull requests, mention `@openhands` in comments to:
@@ -30,3 +42,15 @@ To get OpenHands to work on pull requests, mention `@openhands` in comments to:
 OpenHands will:
 1. Comment to let you know it is working on it
 2. Perform the requested task
+
+#### Example Commands for Pull Requests
+
+Here are some examples of commands you can use with pull requests:
+
+```
+@openhands reflect the review comments
+```
+
+```
+@openhands fix the merge conflicts and make sure that CI passes
+```

docs/modules/usage/cloud/github-installation.md

@@ -36,6 +36,8 @@ You can grant OpenHands access to specific repositories:
      - User's GitHub permissions (owner/collaborator)
 3. Click `Install & Authorize`
 
+![Adding repository access to OpenHands](/img/cloud/add-repo.png)
+
 ## Modifying Repository Access
 
 You can modify repository access at any time by visiting the Settings page and selecting `Configure GitHub Repositories` under the `Git` tab.

docs/modules/usage/configuration-options.md

@@ -66,9 +66,19 @@ The core configuration options are defined in the `[core]` section of the `confi
 
 - `file_store`
   - Type: `str`
-  - Default: `"memory"`
+  - Default: `"local"`
   - Description: File store type
 
+- `file_store_web_hook_url`
+  - Type: `str`
+  - Default: `None`
+  - Description: Optional url for a webhook to invoke after file store writes / deletes
+
+- `file_store_web_hook_headers`
+  - Type: `str`
+  - Default: `None`
+  - Description: HTTP Headers to include in web hook requests.
+
 - `file_uploads_allowed_extensions`
   - Type: `list of str`
   - Default: `[".*"]`
@@ -331,6 +341,8 @@ The agent configuration options are defined in the `[agent]` and `[agent.<agent_
 
 The sandbox configuration options are defined in the `[sandbox]` section of the `config.toml` file.
 
+
+
 To use these with the docker command, pass in `-e SANDBOX_<option>`. Example: `-e SANDBOX_TIMEOUT`.
 
 ### Execution

docs/modules/usage/how-to/cli-mode.md

@@ -23,7 +23,7 @@ This command opens an interactive prompt where you can type tasks or commands an
 
 1. Set the following environment variables in your terminal:
    - `SANDBOX_VOLUMES` to specify the directory you want OpenHands to access ([See using SANDBOX_VOLUMES for more info](../runtimes/docker#using-sandbox_volumes))
-   - `LLM_MODEL` - the LLM model to use (e.g. `export LLM_MODEL="anthropic/claude-3-7-sonnet-20250219"`)
+   - `LLM_MODEL` - the LLM model to use (e.g. `export LLM_MODEL="anthropic/claude-sonnet-4-20250514"`)
    - `LLM_API_KEY` - your API key (e.g. `export LLM_API_KEY="sk_test_12345"`)
 
 2. Run the following command:

docs/modules/usage/how-to/debugging.md

@@ -14,7 +14,7 @@ The following `launch.json` will allow debugging the agent, controller and serve
             "name": "OpenHands CLI",
             "type": "debugpy",
             "request": "launch",
-            "module": "openhands.core.cli",
+            "module": "openhands.cli.main",
             "justMyCode": false
         },
         {

docs/modules/usage/how-to/headless-mode.md

@@ -23,7 +23,7 @@ To run OpenHands in Headless mode with Docker:
 
 1. Set the following environment variables in your terminal:
    - `SANDBOX_VOLUMES` to specify the directory you want OpenHands to access ([See using SANDBOX_VOLUMES for more info](../runtimes/docker#using-sandbox_volumes))
-   - `LLM_MODEL` - the LLM model to use (e.g. `export LLM_MODEL="anthropic/claude-3-7-sonnet-20250219"`)
+   - `LLM_MODEL` - the LLM model to use (e.g. `export LLM_MODEL="anthropic/claude-sonnet-4-20250514"`)
    - `LLM_API_KEY` - your API key (e.g. `export LLM_API_KEY="sk_test_12345"`)
 
 2. Run the following Docker command:

docs/modules/usage/llms/llms.md

@@ -13,7 +13,7 @@ recommendations for model selection. Our latest benchmarking results can be foun
 
 Based on these findings and community feedback, these are the latest models that have been verified to work reasonably well with OpenHands:
 
-- [anthropic/claude-3-7-sonnet-20250219](https://www.anthropic.com/api) (recommended)
+- [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api) (recommended)
 - [openai/o4-mini](https://openai.com/index/introducing-o3-and-o4-mini/)
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)

docs/package.json

@@ -17,10 +17,10 @@
   },
   "// Note": "The OpenAPI spec is stored in docs/static/openapi.json so it's accessible at /openapi.json in the deployed site",
   "dependencies": {
-    "@docusaurus/core": "^3.7.0",
-    "@docusaurus/plugin-content-pages": "^3.7.0",
-    "@docusaurus/preset-classic": "^3.7.0",
-    "@docusaurus/theme-mermaid": "^3.7.0",
+    "@docusaurus/core": "^3.8.0",
+    "@docusaurus/plugin-content-pages": "^3.8.0",
+    "@docusaurus/preset-classic": "^3.8.0",
+    "@docusaurus/theme-mermaid": "^3.8.0",
     "@mdx-js/react": "^3.1.0",
     "@node-rs/jieba": "^2.0.1",
     "clsx": "^2.0.0",
@@ -33,7 +33,7 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.1",
-    "@docusaurus/tsconfig": "^3.7.0",
+    "@docusaurus/tsconfig": "^3.8.0",
     "@docusaurus/types": "^3.5.1",
     "swagger-cli": "^4.0.4",
     "swagger-ui-dist": "^5.22.0",

docs/translation_updater.py

@@ -57,7 +57,7 @@ def translate_content(content, target_lang):
     system_prompt = f'You are a professional translator. Translate the following content into {target_lang}. Preserve all Markdown formatting, code blocks, and front matter. Keep any {{% jsx %}} tags and similar intact. Do not translate code examples, URLs, or technical terms.'
 
     message = client.messages.create(
-        model='claude-3-7-sonnet-20250219',
+        model='claude-sonnet-4-20250514',
         max_tokens=4096,
         temperature=0,
         system=system_prompt,

evaluation/benchmarks/EDA/run_infer.py

@@ -17,7 +17,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -59,10 +59,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/agent_bench/run_infer.py

@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -39,11 +39,11 @@ from openhands.utils.async_utils import call_async_from_sync
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-slim'
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),

evaluation/benchmarks/aider_bench/run_infer.py

@@ -24,7 +24,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     load_from_toml,
     parse_arguments,
@@ -46,10 +46,10 @@ SKIP_NUM = (
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.11-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),

evaluation/benchmarks/biocoder/run_infer.py

@@ -22,7 +22,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -55,12 +55,12 @@ FILE_EXT_MAP = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     BIOCODER_BENCH_CONTAINER_IMAGE = 'public.ecr.aws/i5g0m1f6/eval_biocoder:v1.0'
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = BIOCODER_BENCH_CONTAINER_IMAGE
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/bird/run_infer.py

@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -70,11 +70,11 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/browsing_delegation/run_infer.py

@@ -18,7 +18,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -33,13 +33,13 @@ SUPPORTED_AGENT_CLS = {'CodeActAgent'}
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     assert metadata.max_iterations == 1, (
         'max_iterations must be 1 for browsing delegation evaluation.'
     )
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/commit0/run_infer.py

@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -101,7 +101,7 @@ def get_instance_docker_image(repo_name: str) -> str:
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     repo_name = instance['repo'].split('/')[1]
     base_container_image = get_instance_docker_image(repo_name)
     logger.info(
@@ -113,7 +113,7 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = base_container_image
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         max_iterations=metadata.max_iterations,

evaluation/benchmarks/discoverybench/run_infer.py

@@ -25,7 +25,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -61,10 +61,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/gaia/run_infer.py

@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -47,10 +47,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/gorilla/run_infer.py

@@ -19,7 +19,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -39,10 +39,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/gpqa/run_infer.py

@@ -37,7 +37,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -60,10 +60,10 @@ ACTION_FORMAT = """
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/humanevalfix/run_infer.py

@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -81,10 +81,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/lca_ci_build_repair/eval_infer.py

@@ -19,10 +19,10 @@ from evaluation.utils.shared import (
     make_metadata,
 )
 from openhands.core.config import (
-    AppConfig,
     LLMConfig,
+    OpenHandsConfig,
     get_parser,
-    load_app_config,
+    load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime
@@ -34,10 +34,10 @@ from openhands.utils.async_utils import call_async_from_sync
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',
@@ -53,7 +53,7 @@ def get_config(
     return config
 
 
-config = load_app_config()
+config = load_openhands_config()
 
 
 def load_bench_config():

evaluation/benchmarks/lca_ci_build_repair/run_infer.py

@@ -29,10 +29,10 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
-    load_app_config,
+    load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -44,10 +44,10 @@ from openhands.utils.async_utils import call_async_from_sync
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',
@@ -63,7 +63,7 @@ def get_config(
     return config
 
 
-config = load_app_config()
+config = load_openhands_config()
 
 
 def load_bench_config():

evaluation/benchmarks/logic_reasoning/run_infer.py

@@ -17,7 +17,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -44,14 +44,14 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'xingyaoww/od-eval-logic-reasoning:v1.0'
     sandbox_config.runtime_extra_deps = (
         '$OH_INTERPRETER_PATH -m pip install scitools-pyke'
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/miniwob/run_infer.py

@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -54,10 +54,10 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 def get_config(
     metadata: EvalMetadata,
     env_id: str,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'xingyaoww/od-eval-miniwob:v1.0'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),

evaluation/benchmarks/mint/run_infer.py

@@ -22,7 +22,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -102,14 +102,14 @@ def load_incontext_example(task_name: str, with_tool: bool = True):
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'xingyaoww/od-eval-mint:v1.0'
     sandbox_config.runtime_extra_deps = (
         f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}'
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/ml_bench/run_analysis.py

@@ -4,11 +4,11 @@ import pprint
 
 import tqdm
 
-from openhands.core.config import get_llm_config_arg, get_parser, load_app_config
+from openhands.core.config import get_llm_config_arg, get_parser, load_openhands_config
 from openhands.core.logger import openhands_logger as logger
 from openhands.llm.llm import LLM
 
-config = load_app_config()
+config = load_openhands_config()
 
 
 def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:

evaluation/benchmarks/ml_bench/run_infer.py

@@ -33,10 +33,10 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
-    load_app_config,
+    load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -45,7 +45,7 @@ from openhands.events.observation import CmdOutputObservation
 from openhands.runtime.base import Runtime
 from openhands.utils.async_utils import call_async_from_sync
 
-config = load_app_config()
+config = load_openhands_config()
 
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
     'CodeActAgent': codeact_user_response,
@@ -76,10 +76,10 @@ ID2CONDA = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'public.ecr.aws/i5g0m1f6/ml-bench'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/multi_swe_bench/eval_infer.py

@@ -28,8 +28,8 @@ from evaluation.utils.shared import (
     run_evaluation,
 )
 from openhands.core.config import (
-    AppConfig,
     LLMConfig,
+    OpenHandsConfig,
     get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
@@ -73,7 +73,7 @@ def process_git_patch(patch):
     return patch
 
 
-def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
+def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
     # We use a different instance image for the each instance of swe-bench eval
     base_container_image = get_instance_docker_image(instance['instance_id'])
     logger.info(
@@ -87,7 +87,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
         dataset_name=metadata.dataset,
         instance_id=instance['instance_id'],
     )
-    config = AppConfig(
+    config = OpenHandsConfig(
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),
         sandbox=sandbox_config,

evaluation/benchmarks/multi_swe_bench/run_infer.py

@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -314,7 +314,7 @@ def get_instance_docker_image(instance: pd.Series):
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.2.1'
     if USE_INSTANCE_IMAGE:
         # We use a different instance image for the each instance of swe-bench eval
@@ -340,7 +340,7 @@ def get_config(
         instance_id=instance['instance_id'],
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         max_iterations=metadata.max_iterations,

evaluation/benchmarks/scienceagentbench/run_infer.py

@@ -20,7 +20,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -58,12 +58,12 @@ def format_task_dict(example, use_knowledge):
 def get_config(
     metadata: EvalMetadata,
     instance_id: str,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = (
         'docker.io/xingyaoww/openhands-eval-scienceagentbench'
     )
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),

evaluation/benchmarks/swe_bench/README.md

@@ -2,6 +2,8 @@
 
 This folder contains the evaluation harness that we built on top of the original [SWE-Bench benchmark](https://www.swebench.com/) ([paper](https://arxiv.org/abs/2310.06770)).
 
+**UPDATE (5/26/2025): We now support running interactive SWE-Bench evaluation (see the paper [here](https://arxiv.org/abs/2502.13069))! For how to run it, checkout [this README](./SWE-Interact.md).**
+
 **UPDATE (4/8/2025): We now support running SWT-Bench evaluation! For more details, checkout [the corresponding section](#SWT-Bench-Evaluation).**
 
 **UPDATE (03/27/2025): We now support SWE-Bench multimodal evaluation! Simply use "princeton-nlp/SWE-bench_Multimodal" as the dataset name in the `run_infer.sh` script to evaluate on multimodal instances.**

evaluation/benchmarks/swe_bench/SWE-Interact.md

@@ -0,0 +1,92 @@
+# SWE-Interact Benchmark
+
+This document explains how to use the [Interactive SWE-Bench](https://arxiv.org/abs/2502.13069) benchmark scripts for running and evaluating interactive software engineering tasks.
+
+## Setting things up
+After following the [README](./README.md) to set up the environment, you would need to additionally add LLM configurations for simulated human users. In the original [paper](https://arxiv.org/abs/2502.13069), we use gpt-4o as the simulated human user. You can add the following to your `config.toml` file:
+
+```toml
+[llm.fake_user]
+model="litellm_proxy/gpt-4o-2024-08-06"
+api_key="<your-api-key>"
+temperature = 0.0
+base_url = "https://llm-proxy.eval.all-hands.dev"
+```
+
+## Running the Benchmark
+
+The main script for running the benchmark is `run_infer_interact.sh`. Here's how to use it:
+
+```bash
+bash ./evaluation/benchmarks/swe_bench/scripts/run_infer_interact.sh <model_config> <commit_hash> <agent> <eval_limit> <max_iter> <num_workers> <split>
+```
+
+### Parameters:
+
+- `model_config`: Path to the LLM configuration file (e.g., `llm.claude-3-7-sonnet`)
+- `commit_hash`: Git commit hash to use (e.g., `HEAD`)
+- `agent`: The agent class to use (e.g., `CodeActAgent`)
+- `eval_limit`: Number of examples to evaluate (e.g., `500`)
+- `max_iter`: Maximum number of iterations per task (e.g., `100`)
+- `num_workers`: Number of parallel workers (e.g., `1`)
+- `split`: Dataset split to use (e.g., `test`)
+
+### Example:
+
+```bash
+bash ./evaluation/benchmarks/swe_bench/scripts/run_infer_interact.sh llm.claude-3-7-sonnet HEAD CodeActAgent 500 100 1 test
+```
+
+### Additional Environment Variables:
+
+You can customize the behavior using these environment variables:
+
+- `RUN_WITH_BROWSING`: Enable/disable web browsing (default: false)
+- `USE_HINT_TEXT`: Enable/disable hint text (default: false)
+- `EVAL_CONDENSER`: Specify a condenser configuration
+- `EXP_NAME`: Add a custom experiment name to the output
+- `N_RUNS`: Number of runs to perform (default: 1)
+- `SKIP_RUNS`: Comma-separated list of run numbers to skip
+
+## Evaluating Results
+
+After running the benchmark, you can evaluate the results using `eval_infer.sh`:
+
+```bash
+./evaluation/benchmarks/swe_bench/scripts/eval_infer.sh <output_file> <instance_id> <dataset> <split>
+```
+
+### Parameters:
+
+- `output_file`: Path to the output JSONL file
+- `instance_id`: The specific instance ID to evaluate
+- `dataset`: Dataset name (e.g., `cmu-lti/interactive-swe`)
+- `split`: Dataset split (e.g., `test`)
+
+### Example:
+
+```bash
+./evaluation/benchmarks/swe_bench/scripts/eval_infer.sh evaluation/evaluation_outputs/outputs/cmu-lti__interactive-swe-test/CodeActAgent/claude-3-7-sonnet-20250219_maxiter_100_N_v0.39.0-no-hint-run_1/output.jsonl sphinx-doc__sphinx-8721 cmu-lti/interactive-swe test
+```
+
+## Output Structure
+
+The benchmark outputs are stored in the `evaluation/evaluation_outputs/outputs/` directory with the following structure:
+
+```
+evaluation/evaluation_outputs/outputs/
+└── cmu-lti__interactive-swe-{split}/
+    └── {agent}/
+        └── {model}-{date}_maxiter_{max_iter}_N_{version}-{options}-run_{run_number}/
+            └── output.jsonl
+```
+
+Where:
+- `{split}` is the dataset split (e.g., test)
+- `{agent}` is the agent class name
+- `{model}` is the model name
+- `{date}` is the run date
+- `{max_iter}` is the maximum iterations
+- `{version}` is the OpenHands version
+- `{options}` includes any additional options (e.g., no-hint, with-browsing)
+- `{run_number}` is the run number

evaluation/benchmarks/swe_bench/eval_infer.py

@@ -24,8 +24,8 @@ from evaluation.utils.shared import (
     run_evaluation,
 )
 from openhands.core.config import (
-    AppConfig,
     LLMConfig,
+    OpenHandsConfig,
     get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
@@ -69,7 +69,7 @@ def process_git_patch(patch):
     return patch
 
 
-def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
+def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
     # We use a different instance image for the each instance of swe-bench eval
     base_container_image = get_instance_docker_image(instance['instance_id'])
     logger.info(
@@ -83,7 +83,7 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
         dataset_name=metadata.dataset,
         instance_id=instance['instance_id'],
     )
-    config = AppConfig(
+    config = OpenHandsConfig(
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),
         sandbox=sandbox_config,

evaluation/benchmarks/swe_bench/loc_prompt.py

@@ -0,0 +1,69 @@
+TASK_INSTRUECTION="""
+Given the following GitHub problem description, your objective is to localize the specific files, classes or functions, and lines of code that need modification or contain key information to resolve the issue.
+
+Follow these steps to localize the issue:
+## Step 1: Categorize and Extract Key Problem Information
+ - Classify the problem statement into the following categories:
+    Problem description, error trace, code to reproduce the bug, and additional context.
+ - Identify modules in the '{package_name}' package mentioned in each category.
+ - Use extracted keywords and line numbers to search for relevant code references for additional context.
+
+## Step 2: Locate Referenced Modules
+- Accurately determine specific modules
+    - Explore the repo to familiarize yourself with its structure.
+    - Analyze the described execution flow to identify specific modules or components being referenced.
+- Pay special attention to distinguishing between modules with similar names using context and described execution flow.
+- Output Format for collected relevant modules:
+    - Use the format: 'file_path:QualifiedName'
+    - E.g., for a function `calculate_sum` in the `MathUtils` class located in `src/helpers/math_helpers.py`, represent it as: 'src/helpers/math_helpers.py:MathUtils.calculate_sum'.
+
+## Step 3: Analyze and Reproducing the Problem
+- Clarify the Purpose of the Issue
+    - If expanding capabilities: Identify where and how to incorporate new behavior, fields, or modules.
+    - If addressing unexpected behavior: Focus on localizing modules containing potential bugs.
+- Reconstruct the execution flow
+    - Identify main entry points triggering the issue.
+    - Trace function calls, class interactions, and sequences of events.
+    - Identify potential breakpoints causing the issue.
+    Important: Keep the reconstructed flow focused on the problem, avoiding irrelevant details.
+
+## Step 4: Locate Areas for Modification
+- Locate specific files, functions, or lines of code requiring changes or containing critical information for resolving the issue.
+- Consider upstream and downstream dependencies that may affect or be affected by the issue.
+- If applicable, identify where to introduce new fields, functions, or variables.
+- Think Thoroughly: List multiple potential solutions and consider edge cases that could impact the resolution.
+
+## Output Format for Final Results:
+Your final output should list the locations requiring modification, wrapped with triple backticks ```
+Each location should include the file path, class name (if applicable), function name, or line numbers, ordered by importance.
+Your answer would better include about 5 files.
+
+### Examples:
+```
+full_path1/file1.py
+line: 10
+class: MyClass1
+function: my_function1
+
+full_path2/file2.py
+line: 76
+function: MyClass2.my_function2
+
+full_path3/file3.py
+line: 24
+line: 156
+function: my_function3
+```
+
+Return just the location(s)
+
+Note: Your thinking should be thorough and so it's fine if it's very long.
+"""
+
+FAKE_USER_MSG_FOR_LOC = (
+    'Verify if the found locations contain all the necessary information to address the issue, and check for any relevant references in other parts of the codebase that may not have appeared in the search results. '
+    'If not, continue searching for additional locations related to the issue.\n'
+    'Verify that you have carefully analyzed the impact of the found locations on the repository, especially their dependencies. '
+    'If you think you have solved the task, please send your final answer (including the former answer and reranking) to user through message and then call `finish` to finish.\n'
+    'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
+)

evaluation/benchmarks/swe_bench/run_infer.py

@@ -40,12 +40,12 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
-from openhands.core.config.utils import get_condenser_config_arg
 from openhands.core.config.condenser_config import NoOpCondenserConfig
+from openhands.core.config.utils import get_condenser_config_arg
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
 from openhands.critic import AgentFinishedCritic
@@ -220,7 +220,7 @@ def get_instance_docker_image(
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     # We use a different instance image for the each instance of swe-bench eval
     use_swebench_official_image = 'swe-gym' not in metadata.dataset.lower()
     base_container_image = get_instance_docker_image(
@@ -244,7 +244,7 @@ def get_config(
         instance_id=instance['instance_id'],
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         max_iterations=metadata.max_iterations,
@@ -721,15 +721,16 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
                 # repos for the swe-bench instances:
                 # ['astropy/astropy', 'django/django', 'matplotlib/matplotlib', 'mwaskom/seaborn', 'pallets/flask', 'psf/requests', 'pydata/xarray', 'pylint-dev/pylint', 'pytest-dev/pytest', 'scikit-learn/scikit-learn', 'sphinx-doc/sphinx', 'sympy/sympy']
                 selected_repos = data['selected_repos']
-                if isinstance(selected_repos, str): selected_repos = [selected_repos]
+                if isinstance(selected_repos, str):
+                    selected_repos = [selected_repos]
                 assert isinstance(selected_repos, list)
                 logger.info(
                     f'Filtering {selected_repos} tasks from "selected_repos"...'
                 )
-                subset = dataset[dataset["repo"].isin(selected_repos)]
+                subset = dataset[dataset['repo'].isin(selected_repos)]
                 logger.info(f'Retained {subset.shape[0]} tasks after filtering')
                 return subset
-                
+
     skip_ids = os.environ.get('SKIP_IDS', '').split(',')
     if len(skip_ids) > 0:
         logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
@@ -806,7 +807,9 @@ if __name__ == '__main__':
     else:
         # If no specific condenser config is provided via env var, default to NoOpCondenser
         condenser_config = NoOpCondenserConfig()
-        logger.debug('No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.')
+        logger.debug(
+            'No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.'
+        )
 
     details = {'mode': args.mode}
     _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)

evaluation/benchmarks/swe_bench/run_infer_interact.py

@@ -0,0 +1,411 @@
+import asyncio
+import json
+import os
+
+import pandas as pd
+from datasets import load_dataset
+from litellm import completion as litellm_completion
+
+import openhands.agenthub
+from evaluation.benchmarks.swe_bench.run_infer import (
+    AgentFinishedCritic,
+    complete_runtime,
+    filter_dataset,
+    get_config,
+    initialize_runtime,
+)
+from evaluation.benchmarks.swe_bench.run_infer import (
+    get_instruction as base_get_instruction,
+)
+from evaluation.utils.shared import (
+    EvalException,
+    EvalMetadata,
+    EvalOutput,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    get_llm_config_arg,
+    get_parser,
+)
+from openhands.core.config.condenser_config import NoOpCondenserConfig
+from openhands.core.config.utils import get_condenser_config_arg
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import MessageAction
+from openhands.events.serialization.event import event_from_dict, event_to_dict
+from openhands.utils.async_utils import call_async_from_sync
+
+USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
+USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'false'
+
+
+class FakeUser:
+    def __init__(self, issue, hints, files):
+        self.system_message = f"""
+        You are a GitHub user reporting an issue. Here are the details of your issue and environment:
+
+        Issue: {issue}
+
+        Hints: {hints}
+
+        Files relative to your current directory: {files}
+
+        Your task is to respond to questions from a coder who is trying to solve your issue. The coder has a summarized version of the issue you have. Follow these rules:
+        1. If the coder asks a question that is directly related to the information in the issue you have, provide that information.
+        2. Always stay in character as a user reporting an issue, not as an AI assistant.
+        3. Keep your responses concise and to the point.
+        4. The coder has limited turns to solve the issue. Do not interact with the coder beyond 3 turns.
+
+        Respond with "I don't have that information" if the question is unrelated or you're unsure.
+        """
+        self.chat_history = [{'role': 'system', 'content': self.system_message}]
+        self.turns = 0
+        # Get LLM config from config.toml
+        self.llm_config = get_llm_config_arg(
+            'llm.fake_user'
+        )  # You can change 'fake_user' to any config name you want
+
+    def generate_reply(self, question):
+        if self.turns > 3:
+            return 'Please continue working on the task. Do NOT ask for more help.'
+        self.chat_history.append({'role': 'user', 'content': question.content})
+
+        response = litellm_completion(
+            model=self.llm_config.model,
+            messages=self.chat_history,
+            api_key=self.llm_config.api_key.get_secret_value(),
+            temperature=self.llm_config.temperature,
+            base_url=self.llm_config.base_url,
+        )
+
+        reply = response.choices[0].message.content
+        self.chat_history.append({'role': 'assistant', 'content': reply})
+        self.turns += 1
+        return reply
+
+
+# Global variable for fake user
+fake_user = None
+
+
+def get_fake_user_response(state: State) -> str:
+    global fake_user
+    if not fake_user:
+        return 'Please continue working on the task.'
+    last_agent_message = state.get_last_agent_message()
+    if last_agent_message:
+        return fake_user.generate_reply(last_agent_message)
+    return 'Please continue working on the task.'
+
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': get_fake_user_response,
+}
+
+
+def get_instruction(instance: pd.Series, metadata: EvalMetadata) -> MessageAction:
+    instance_copy = instance.copy()
+    instance_copy.problem_statement = f'{instance.problem_statement}\n\nHints:\nThe user has not provided all the necessary details about the issue, and there are some hidden details that are helpful. Please ask the user specific questions using non-code commands to gather the relevant information that the user has to help you solve the issue. Ensure you have all the details you require to solve the issue.'
+    return base_get_instruction(instance_copy, metadata)
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    config = get_config(instance, metadata)
+    global fake_user
+    original_issue = instance.original_issue
+    issue = str(original_issue)
+    fake_user = FakeUser(issue=issue, hints=instance.hints_text, files=instance.files)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    try:
+        initialize_runtime(runtime, instance, metadata)
+
+        message_action = get_instruction(instance, metadata)
+
+        # Here's how you can run the agent (similar to the `main` function) and get the final task state
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=message_action,
+                runtime=runtime,
+                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                    metadata.agent_class
+                ],
+            )
+        )
+
+        # if fatal error, throw EvalError to trigger re-run
+        if (
+            state
+            and state.last_error
+            and 'fatal error during agent execution' in state.last_error
+            and 'stuck in a loop' not in state.last_error
+        ):
+            raise EvalException('Fatal error detected: ' + state.last_error)
+
+        # Get git patch
+        return_val = complete_runtime(runtime, instance)
+        git_patch = return_val['git_patch']
+        logger.info(
+            f'Got git diff for instance {instance.instance_id}:\n--------\n{git_patch}\n--------'
+        )
+    finally:
+        runtime.close()
+
+    # Prepare test result
+    test_result = {
+        'git_patch': git_patch,
+    }
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    histories = [event_to_dict(event) for event in state.history]
+    metrics = state.metrics.get() if state.metrics else None
+
+    # Save the output
+    instruction = message_action.content
+    if message_action.image_urls:
+        instruction += (
+            '\n\n<image_urls>' + '\n'.join(message_action.image_urls) + '</image_urls>'
+        )
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        instance=instance.to_dict(),
+        test_result=test_result,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+    )
+    return output
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='cmu-lti/interactive-swe',
+        help='dataset to evaluate on',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+
+    args, _ = parser.parse_known_args()
+
+    # Load dataset from huggingface datasets
+    dataset = load_dataset(args.dataset, split=args.split)
+    swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
+    logger.info(
+        f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
+    )
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        llm_config.log_completions = True
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    # Get condenser config from environment variable
+    condenser_name = os.environ.get('EVAL_CONDENSER')
+    if condenser_name:
+        condenser_config = get_condenser_config_arg(condenser_name)
+        if condenser_config is None:
+            raise ValueError(
+                f'Could not find Condenser config: EVAL_CONDENSER={condenser_name}'
+            )
+    else:
+        # If no specific condenser config is provided via env var, default to NoOpCondenser
+        condenser_config = NoOpCondenserConfig()
+        logger.debug(
+            'No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.'
+        )
+
+    details = {'mode': 'interact'}
+    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
+
+    dataset_descrption = (
+        args.dataset.replace('/', '__') + '-' + args.split.replace('/', '__')
+    )
+    metadata = make_metadata(
+        llm_config,
+        dataset_descrption,
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        details=details,
+        condenser_config=condenser_config,
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    print(f'### OUTPUT FILE: {output_file} ###')
+
+    # Run evaluation in iterative mode:
+    # If a rollout fails to output AgentFinishAction, we will try again until it succeeds OR total 3 attempts have been made.
+    ITERATIVE_EVAL_MODE = (
+        os.environ.get('ITERATIVE_EVAL_MODE', 'false').lower() == 'true'
+    )
+    ITERATIVE_EVAL_MODE_MAX_ATTEMPTS = int(
+        os.environ.get('ITERATIVE_EVAL_MODE_MAX_ATTEMPTS', '3')
+    )
+
+    if not ITERATIVE_EVAL_MODE:
+        # load the dataset
+        instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
+        if len(instances) > 0 and not isinstance(
+            instances['PASS_TO_PASS'][instances['PASS_TO_PASS'].index[0]], str
+        ):
+            for col in ['PASS_TO_PASS', 'FAIL_TO_PASS']:
+                instances[col] = instances[col].apply(lambda x: str(x))
+        run_evaluation(
+            instances,
+            metadata,
+            output_file,
+            args.eval_num_workers,
+            process_instance,
+            timeout_seconds=8
+            * 60
+            * 60,  # 8 hour PER instance should be more than enough
+            max_retries=5,
+        )
+    else:
+        critic = AgentFinishedCritic()
+
+        def get_cur_output_file_path(attempt: int) -> str:
+            return (
+                f'{output_file.removesuffix(".jsonl")}.critic_attempt_{attempt}.jsonl'
+            )
+
+        eval_ids = None
+        for attempt in range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1):
+            cur_output_file = get_cur_output_file_path(attempt)
+            logger.info(
+                f'Running evaluation with critic {critic.__class__.__name__} for attempt {attempt} of {ITERATIVE_EVAL_MODE_MAX_ATTEMPTS}.'
+            )
+
+            # For deterministic eval, we set temperature to 0.1 for (>1) attempt
+            # so hopefully we get slightly different results
+            if attempt > 1 and metadata.llm_config.temperature == 0:
+                logger.info(
+                    f'Detected temperature is 0 for (>1) attempt {attempt}. Setting temperature to 0.1...'
+                )
+                metadata.llm_config.temperature = 0.1
+
+            # Load instances - at first attempt, we evaluate all instances
+            # On subsequent attempts, we only evaluate the instances that failed the previous attempt determined by critic
+            instances = prepare_dataset(
+                swe_bench_tests, cur_output_file, args.eval_n_limit, eval_ids=eval_ids
+            )
+            if len(instances) > 0 and not isinstance(
+                instances['PASS_TO_PASS'][instances['PASS_TO_PASS'].index[0]], str
+            ):
+                for col in ['PASS_TO_PASS', 'FAIL_TO_PASS']:
+                    instances[col] = instances[col].apply(lambda x: str(x))
+
+            # Run evaluation - but save them to cur_output_file
+            logger.info(
+                f'Evaluating {len(instances)} instances for attempt {attempt}...'
+            )
+            run_evaluation(
+                instances,
+                metadata,
+                cur_output_file,
+                args.eval_num_workers,
+                process_instance,
+                timeout_seconds=8
+                * 60
+                * 60,  # 8 hour PER instance should be more than enough
+                max_retries=5,
+            )
+
+            # When eval is done, we update eval_ids to the instances that failed the current attempt
+            instances_failed = []
+            logger.info(
+                f'Use critic {critic.__class__.__name__} to check {len(instances)} instances for attempt {attempt}...'
+            )
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    try:
+                        history = [
+                            event_from_dict(event) for event in instance['history']
+                        ]
+                        critic_result = critic.evaluate(
+                            history, instance['test_result'].get('git_patch', '')
+                        )
+                        if not critic_result.success:
+                            instances_failed.append(instance['instance_id'])
+                    except Exception as e:
+                        logger.error(
+                            f'Error loading history for instance {instance["instance_id"]}: {e}'
+                        )
+                        instances_failed.append(instance['instance_id'])
+            logger.info(
+                f'{len(instances_failed)} instances failed the current attempt {attempt}: {instances_failed}'
+            )
+            eval_ids = instances_failed
+
+            # If no instances failed, we break
+            if len(instances_failed) == 0:
+                break
+
+        # Then we should aggregate the results from all attempts into the original output file
+        # and remove the intermediate files
+        logger.info(
+            'Aggregating results from all attempts into the original output file...'
+        )
+        fout = open(output_file, 'w')
+        added_instance_ids = set()
+        for attempt in reversed(range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1)):
+            cur_output_file = get_cur_output_file_path(attempt)
+            if not os.path.exists(cur_output_file):
+                logger.warning(
+                    f'Intermediate output file {cur_output_file} does not exist. Skipping...'
+                )
+                continue
+
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    # Also make sure git_patch is not empty - otherwise we fall back to previous attempt (empty patch is worse than anything else)
+                    if (
+                        instance['instance_id'] not in added_instance_ids
+                        and instance['test_result'].get('git_patch', '').strip()
+                    ):
+                        fout.write(line)
+                        added_instance_ids.add(instance['instance_id'])
+            logger.info(
+                f'Aggregated instances from {cur_output_file}. Total instances added so far: {len(added_instance_ids)}'
+            )
+        fout.close()
+        logger.info(
+            f'Done! Total {len(added_instance_ids)} instances added to {output_file}'
+        )

evaluation/benchmarks/swe_bench/run_localize.py

@@ -0,0 +1,717 @@
+import asyncio
+import json
+import os
+import tempfile
+from typing import Any
+
+import pandas as pd
+import toml
+from datasets import load_dataset
+
+import openhands.agenthub
+from evaluation.benchmarks.swe_bench.resource.mapping import (
+    get_instance_resource_factor,
+)
+from evaluation.utils.shared import (
+    EvalException,
+    EvalMetadata,
+    EvalOutput,
+    assert_and_raise,
+    codeact_user_response,
+    get_default_sandbox_config_for_eval,
+    get_metrics,
+    is_fatal_evaluation_error,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    OpenHandsConfig,
+    get_llm_config_arg,
+    get_parser,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation, ErrorObservation
+from openhands.events.serialization.event import event_to_dict
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+from openhands.utils.shutdown_listener import sleep_if_should_continue
+
+USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
+INDEX_BASE_DIR = os.environ.get('INDEX_BASE_DIR', '')
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+    'LocAgent': codeact_user_response,
+}
+
+
+def _get_swebench_workspace_dir_name(instance: pd.Series) -> str:
+    return f'{instance.repo}__{instance.version}'.replace('/', '__')
+
+
+def get_instruction(instance: pd.Series, metadata: EvalMetadata):
+    _get_swebench_workspace_dir_name(instance)
+    instruction = f"""
+Consider the following issue description:
+
+<issue_description>
+{instance.problem_statement}
+</issue_description>
+
+Your objective is to localize the specific files, classes or functions, and lines of code that need modification or contain key information to resolve the issue.
+
+Follow these steps to localize the issue:
+## Step 1: Categorize and Extract Key Problem Information
+ - Classify the problem statement into the following categories:
+    Problem description, error trace, code to reproduce the bug, and additional context.
+ - Identify modules in the "{instance.instance_id.split('_')[0]}" package mentioned in each category.
+ - Use extracted keywords and line numbers to search for relevant code references for additional context.
+
+## Step 2: Locate Referenced Modules
+- Accurately determine specific modules
+    - Explore the repo to familiarize yourself with its structure.
+    - Analyze the described execution flow to identify specific modules or components being referenced.
+- Pay special attention to distinguishing between modules with similar names using context and described execution flow.
+- Output Format for collected relevant modules:
+    - Use the format: 'file_path:QualifiedName'
+    - E.g., for a function `calculate_sum` in the `MathUtils` class located in `src/helpers/math_helpers.py`, represent it as: 'src/helpers/math_helpers.py:MathUtils.calculate_sum'.
+
+## Step 3: Analyze and Reproducing the Problem
+- Clarify the Purpose of the Issue
+    - If expanding capabilities: Identify where and how to incorporate new behavior, fields, or modules.
+    - If addressing unexpected behavior: Focus on localizing modules containing potential bugs.
+- Reconstruct the execution flow
+    - Identify main entry points triggering the issue.
+    - Trace function calls, class interactions, and sequences of events.
+    - Identify potential breakpoints causing the issue.
+    Important: Keep the reconstructed flow focused on the problem, avoiding irrelevant details.
+
+## Step 4: Locate Areas for Modification
+- Locate specific files, functions, or lines of code requiring changes or containing critical information for resolving the issue.
+- Consider upstream and downstream dependencies that may affect or be affected by the issue.
+- If applicable, identify where to introduce new fields, functions, or variables.
+- Think Thoroughly: List multiple potential solutions and consider edge cases that could impact the resolution.
+
+## Output Format for Final Results:
+Your final output should list the locations requiring modification, wrapped with triple backticks ```
+Each location should include the file path, class name (if applicable), function name, or line numbers, ordered by importance.
+Your answer would better include about 5 files.
+
+### Examples:
+```
+full_path1/file1.py
+line: 10
+class: MyClass1
+function: my_function1
+
+full_path2/file2.py
+line: 76
+function: MyClass2.my_function2
+
+full_path3/file3.py
+line: 24
+line: 156
+function: my_function3
+```
+
+Return just the location(s)
+
+Note: Your thinking should be thorough and so it's fine if it's very long.
+"""
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
+        "Don't include any lambda functions!\n"
+        'You should NOT modify any files!\n'
+    )
+    if RUN_WITH_BROWSING:
+        instruction += """
+<IMPORTANT!>
+You SHOULD NEVER attempt to browse the web.
+</IMPORTANT!>
+"""
+    return instruction
+
+
+# TODO: migrate all swe-bench docker to ghcr.io/openhands
+DEFAULT_DOCKER_IMAGE_PREFIX = os.environ.get(
+    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/'
+)
+logger.info(f'Default docker image prefix: {DEFAULT_DOCKER_IMAGE_PREFIX}')
+
+
+def get_instance_docker_image(instance_id: str, official_image: bool = False) -> str:
+    if official_image:
+        # Official SWE-Bench image
+        # swebench/sweb.eval.x86_64.django_1776_django-11333:v1
+        docker_image_prefix = 'docker.io/swebench/'
+        repo, name = instance_id.split('__')
+        image_name = f'sweb.eval.x86_64.{repo}_1776_{name}:latest'
+        logger.warning(f'Using official SWE-Bench image: {image_name}')
+    else:
+        # OpenHands version of the image
+        docker_image_prefix = DEFAULT_DOCKER_IMAGE_PREFIX
+        image_name = 'sweb.eval.x86_64.' + instance_id
+        image_name = image_name.replace(
+            '__', '_s_'
+        )  # to comply with docker image naming convention
+    return (docker_image_prefix.rstrip('/') + '/' + image_name).lower()
+
+
+def get_config(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+) -> OpenHandsConfig:
+    # We use a different instance image for the each instance of swe-bench eval
+    use_official_image = bool(
+        'verified' in metadata.dataset.lower() or 'lite' in metadata.dataset.lower()
+    )
+    base_container_image = get_instance_docker_image(
+        instance['instance_id'], use_official_image
+    )
+    logger.info(
+        f'Using instance container image: {base_container_image}. '
+        f'Please make sure this image exists. '
+        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+    )
+
+    sandbox_config = get_default_sandbox_config_for_eval()
+    sandbox_config.base_container_image = base_container_image
+    sandbox_config.enable_auto_lint = True
+    sandbox_config.use_host_network = False
+    # Add platform to the sandbox config to solve issue 4401
+    sandbox_config.platform = 'linux/amd64'
+    sandbox_config.remote_runtime_resource_factor = get_instance_resource_factor(
+        dataset_name=metadata.dataset,
+        instance_id=instance['instance_id'],
+    )
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    sandbox_config.runtime_startup_env_vars = {
+        'REPO_PATH': f'/workspace/{workspace_dir_name}/',
+    }
+
+    config = OpenHandsConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        max_iterations=metadata.max_iterations,
+        runtime=os.environ.get('RUNTIME', 'docker'),
+        sandbox=sandbox_config,
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
+        )
+    )
+    agent_config = AgentConfig(
+        enable_jupyter=False,
+        enable_browsing=RUN_WITH_BROWSING,
+        enable_llm_editor=False,
+        condenser=metadata.condenser_config,
+        enable_prompt_extensions=False,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Initialization Fn')
+    logger.info('-' * 30)
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    obs: CmdOutputObservation
+
+    # Set instance id
+    action = CmdRunAction(
+        command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc"""
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0, f'Failed to export SWE_INSTANCE_ID: {str(obs)}'
+    )
+
+    action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """)
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to export USER: {str(obs)}')
+
+    # inject the init script
+    script_dir = os.path.dirname(__file__)
+
+    # inject the instance info
+    action = CmdRunAction(command='mkdir -p /swe_util/eval_data/instances')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to create /swe_util/eval_data/instances: {str(obs)}',
+    )
+
+    swe_instance_json_name = 'swe-bench-instance.json'
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Construct the full path for the desired file name within the temporary directory
+        temp_file_path = os.path.join(temp_dir, swe_instance_json_name)
+        # Write to the file with the desired name within the temporary directory
+        with open(temp_file_path, 'w') as f:
+            if not isinstance(instance, dict):
+                json.dump([instance.to_dict()], f)
+            else:
+                json.dump([instance], f)
+
+        # Copy the file to the desired location
+        runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/')
+
+        # inject the instance swe entry
+        runtime.copy_to(
+            str(os.path.join(script_dir, 'scripts/setup/instance_swe_entry.sh')),
+            '/swe_util/',
+        )
+
+    action = CmdRunAction(command='cat ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to cat ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command='source ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    if isinstance(obs, ErrorObservation):
+        logger.error(f'Failed to source ~/.bashrc: {str(obs)}')
+    assert_and_raise(obs.exit_code == 0, f'Failed to source ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command='source /swe_util/instance_swe_entry.sh')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to source /swe_util/instance_swe_entry.sh: {str(obs)}',
+    )
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git reset --hard')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to git reset --hard: {str(obs)}')
+
+    action = CmdRunAction(
+        command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done'
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to remove git remotes: {str(obs)}')
+
+    # Copy the processed indexes if available
+    action = CmdRunAction(command='mkdir _index_data/graph_index_v2.3')
+    obs = runtime.run_action(action)
+
+    # Check if an existing graph index file is available
+    graph_index_file_path = os.path.join(
+        INDEX_BASE_DIR, 'graph_index_v2.3', f'{instance["instance_id"]}.pkl'
+    )
+    if INDEX_BASE_DIR and os.path.exists(graph_index_file_path):
+        logger.info(
+            f'Copying graph index from {graph_index_file_path} to /workspace/{workspace_dir_name}/_index_data/graph_index_v2.3'
+        )
+
+        runtime.copy_to(
+            graph_index_file_path,
+            f'/workspace/{workspace_dir_name}/_index_data/graph_index_v2.3',
+        )
+        action = CmdRunAction(
+            command=f'mv _index_data/graph_index_v2.3/{instance["instance_id"]}.pkl _index_data/graph_index_v2.3/code_graph.pkl'
+        )
+        obs = runtime.run_action(action)
+
+        bm25_index_dir = os.path.join(
+            INDEX_BASE_DIR, 'BM25_index', instance['instance_id']
+        )
+        runtime.copy_to(
+            bm25_index_dir,
+            f'/workspace/{workspace_dir_name}/_index_data',
+            recursive=True,
+        )
+        action = CmdRunAction(
+            command=f'mv _index_data/{instance["instance_id"]} _index_data/bm25_index'
+        )
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        assert_and_raise(obs.exit_code == 0, f'Failed to mv file: {str(obs)}')
+
+    action = CmdRunAction(command='which python')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0 and 'testbed' in obs.content,
+        f'Expected to find python interpreter from testbed, but got: {str(obs)}',
+    )
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Initialization Fn')
+    logger.info('-' * 30)
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Completion Fn')
+    logger.info('-' * 30)
+    obs: CmdOutputObservation
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    if obs.exit_code == -1:
+        # The previous command is still running
+        # We need to kill previous command
+        logger.info('The previous command is still running, trying to kill it...')
+        action = CmdRunAction(command='C-c')
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+        # Then run the command again
+        action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git config --global core.pager ""')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git config --global core.pager "": {str(obs)}',
+    )
+
+    # First check for any git repositories in subdirectories
+    action = CmdRunAction(command='find . -type d -name .git -not -path "./.git"')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to find git repositories: {str(obs)}',
+    )
+
+    git_dirs = [p for p in obs.content.strip().split('\n') if p]
+    if git_dirs:
+        # Remove all .git directories in subdirectories
+        for git_dir in git_dirs:
+            action = CmdRunAction(command=f'rm -rf "{git_dir}"')
+            action.set_hard_timeout(600)
+            logger.info(action, extra={'msg_type': 'ACTION'})
+            obs = runtime.run_action(action)
+            logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+            assert_and_raise(
+                isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+                f'Failed to remove git directory {git_dir}: {str(obs)}',
+            )
+
+    # add all files
+    action = CmdRunAction(command='git add -A')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git add -A: {str(obs)}',
+    )
+
+    n_retries = 0
+    git_patch = None
+    while n_retries < 5:
+        action = CmdRunAction(
+            command=f'git diff --no-color --cached {instance["base_commit"]}'
+        )
+        action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        n_retries += 1
+        if isinstance(obs, CmdOutputObservation):
+            if obs.exit_code == 0:
+                git_patch = obs.content.strip()
+                break
+            else:
+                logger.info('Failed to get git diff, retrying...')
+                sleep_if_should_continue(10)
+        elif isinstance(obs, ErrorObservation):
+            logger.error(f'Error occurred: {obs.content}. Retrying...')
+            sleep_if_should_continue(10)
+        else:
+            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+
+    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Completion Fn')
+    logger.info('-' * 30)
+    return {'git_patch': git_patch}
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+    runtime_failure_count: int = 0,
+) -> EvalOutput:
+    config = get_config(instance, metadata)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    # Increase resource_factor with increasing attempt_id
+    if runtime_failure_count > 0:
+        config.sandbox.remote_runtime_resource_factor = min(
+            config.sandbox.remote_runtime_resource_factor * (2**runtime_failure_count),
+            8,
+        )
+        logger.warning(
+            f'This is the {runtime_failure_count + 1}th attempt for instance {instance.instance_id}, setting resource factor to {config.sandbox.remote_runtime_resource_factor}'
+        )
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    try:
+        initialize_runtime(runtime, instance)
+
+        instruction = get_instruction(instance, metadata)
+
+        # Here's how you can run the agent (similar to the `main` function) and get the final task state
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=MessageAction(content=instruction),
+                runtime=runtime,
+                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                    metadata.agent_class
+                ],
+            )
+        )
+
+        # if fatal error, throw EvalError to trigger re-run
+        if is_fatal_evaluation_error(state.last_error):
+            raise EvalException('Fatal error detected: ' + state.last_error)
+
+        # ======= THIS IS SWE-Bench specific =======
+        # Get git patch
+        return_val = complete_runtime(runtime, instance)
+        git_patch = return_val['git_patch']
+        logger.info(
+            f'Got git diff for instance {instance.instance_id}:\n--------\n{git_patch}\n--------'
+        )
+    finally:
+        runtime.close()
+    # ==========================================
+
+    # ======= Attempt to evaluate the agent's edits =======
+    # we use eval_infer.sh to evaluate the agent's edits, not here
+    # because the agent may alter the environment / testcases
+    test_result = {
+        'git_patch': git_patch,
+    }
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
+    histories = [event_to_dict(event) for event in state.history]
+    metrics = get_metrics(state)
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        instance=instance.to_dict(),  # SWE Bench specific
+        test_result=test_result,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+    )
+    return output
+
+
+def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
+    file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.toml')
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as file:
+            data = toml.load(file)
+            if 'selected_ids' in data:
+                selected_ids = data['selected_ids']
+                logger.info(
+                    f'Filtering {len(selected_ids)} tasks from "selected_ids"...'
+                )
+                subset = dataset[dataset[filter_column].isin(selected_ids)]
+                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
+                return subset
+    skip_ids = os.environ.get('SKIP_IDS', '').split(',')
+    if len(skip_ids) > 0:
+        logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
+        return dataset[~dataset[filter_column].isin(skip_ids)]
+    return dataset
+
+
+# A list of instances that are known to be tricky to infer
+# (will cause runtime failure even with resource factor = 8)
+SWEGYM_EXCLUDE_IDS = [
+    'dask__dask-10422',
+    'pandas-dev__pandas-50548',
+    'pandas-dev__pandas-53672',
+    'pandas-dev__pandas-54174',
+    'pandas-dev__pandas-55518',
+    'pandas-dev__pandas-58383',
+    'pydata__xarray-6721',
+    'pytest-dev__pytest-10081',
+    'pytest-dev__pytest-7236',
+]
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='princeton-nlp/SWE-bench',
+        help='data set to evaluate on, either full-test or lite-test',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+    args, _ = parser.parse_known_args()
+
+    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
+    # so we don't need to manage file uploading to OpenHands's repo
+    dataset = load_dataset(args.dataset, split=args.split)
+    swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
+    logger.info(
+        f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
+    )
+    if 'SWE-Gym' in args.dataset:
+        swe_bench_tests = swe_bench_tests[
+            ~swe_bench_tests['instance_id'].isin(SWEGYM_EXCLUDE_IDS)
+        ]
+        logger.info(
+            f'{len(swe_bench_tests)} tasks left after excluding SWE-Gym excluded tasks'
+        )
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        llm_config.log_completions = True
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    details = {}
+    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
+
+    dataset_descrption = (
+        args.dataset.replace('/', '__') + '-' + args.split.replace('/', '__')
+    )
+    metadata = make_metadata(
+        llm_config,
+        dataset_descrption,
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        details=details,
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    print(f'### OUTPUT FILE: {output_file} ###')
+    instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
+
+    if len(instances) > 0 and not isinstance(
+        instances['PASS_TO_PASS'][instances['PASS_TO_PASS'].index[0]], str
+    ):
+        for col in ['PASS_TO_PASS', 'FAIL_TO_PASS']:
+            instances[col] = instances[col].apply(lambda x: str(x))
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+        timeout_seconds=8 * 60 * 60,  # 8 hour PER instance should be more than enough
+        max_retries=5,
+    )

evaluation/benchmarks/swe_bench/scripts/run_infer_interact.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+MAX_ITER=$5
+NUM_WORKERS=$6
+SPLIT=$8
+N_RUNS=$9
+
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$MAX_ITER" ]; then
+  echo "MAX_ITER not specified, use default 100"
+  MAX_ITER=100
+fi
+
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
+
+if [ -z "$DATASET" ]; then
+  echo "DATASET not specified, use default cmu-lti/interactive-swe"
+  DATASET="cmu-lti/interactive-swe"
+fi
+
+if [ -z "$SPLIT" ]; then
+  echo "SPLIT not specified, use default test"
+  SPLIT="test"
+fi
+
+if [ -n "$EVAL_CONDENSER" ]; then
+  echo "Using Condenser Config: $EVAL_CONDENSER"
+else
+  echo "No Condenser Config provided via EVAL_CONDENSER, use default (NoOpCondenser)."
+fi
+
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+echo "SPLIT: $SPLIT"
+echo "MAX_ITER: $MAX_ITER"
+echo "NUM_WORKERS: $NUM_WORKERS"
+echo "COMMIT_HASH: $COMMIT_HASH"
+echo "EVAL_CONDENSER: $EVAL_CONDENSER"
+
+# Default to NOT use Hint
+if [ -z "$USE_HINT_TEXT" ]; then
+  export USE_HINT_TEXT=false
+fi
+echo "USE_HINT_TEXT: $USE_HINT_TEXT"
+EVAL_NOTE="$OPENHANDS_VERSION"
+# if not using Hint, add -no-hint to the eval note
+if [ "$USE_HINT_TEXT" = false ]; then
+  EVAL_NOTE="$EVAL_NOTE-no-hint"
+fi
+
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
+if [ -n "$EXP_NAME" ]; then
+  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+# Add condenser config to eval note if provided
+if [ -n "$EVAL_CONDENSER" ]; then
+  EVAL_NOTE="${EVAL_NOTE}-${EVAL_CONDENSER}"
+fi
+
+function run_eval() {
+  local eval_note="${1}"
+  COMMAND="poetry run python evaluation/benchmarks/swe_bench/run_infer_interact.py \
+    --agent-cls $AGENT \
+    --llm-config $MODEL_CONFIG \
+    --max-iterations $MAX_ITER \
+    --eval-num-workers $NUM_WORKERS \
+    --eval-note $eval_note \
+    --dataset $DATASET \
+    --split $SPLIT"
+
+  if [ -n "$EVAL_LIMIT" ]; then
+    echo "EVAL_LIMIT: $EVAL_LIMIT"
+    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+  fi
+
+  # Run the command
+  eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+if [ -z "$N_RUNS" ]; then
+  N_RUNS=1
+  echo "N_RUNS not specified, use default $N_RUNS"
+fi
+
+# Skip runs if the run number is in the SKIP_RUNS list
+# read from env variable SKIP_RUNS as a comma separated list of run numbers
+SKIP_RUNS=(${SKIP_RUNS//,/ })
+for i in $(seq 1 $N_RUNS); do
+  if [[ " ${SKIP_RUNS[@]} " =~ " $i " ]]; then
+    echo "Skipping run $i"
+    continue
+  fi
+  current_eval_note="$EVAL_NOTE-run_$i"
+  echo "EVAL_NOTE: $current_eval_note"
+  run_eval $current_eval_note
+done
+
+checkout_original_branch

evaluation/benchmarks/swe_bench/scripts/run_localize.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+MAX_ITER=$5
+NUM_WORKERS=$6
+DATASET=$7
+SPLIT=$8
+N_RUNS=$9
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$MAX_ITER" ]; then
+  echo "MAX_ITER not specified, use default 100"
+  MAX_ITER=100
+fi
+
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
+
+if [ -z "$DATASET" ]; then
+  echo "DATASET not specified, use default princeton-nlp/SWE-bench_Lite"
+  DATASET="princeton-nlp/SWE-bench_Lite"
+fi
+
+if [ -z "$SPLIT" ]; then
+  echo "SPLIT not specified, use default test"
+  SPLIT="test"
+fi
+
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+echo "SPLIT: $SPLIT"
+
+# Default to NOT use Hint
+if [ -z "$USE_HINT_TEXT" ]; then
+  export USE_HINT_TEXT=false
+fi
+echo "USE_HINT_TEXT: $USE_HINT_TEXT"
+EVAL_NOTE="$OPENHANDS_VERSION"
+# if not using Hint, add -no-hint to the eval note
+if [ "$USE_HINT_TEXT" = false ]; then
+  EVAL_NOTE="$EVAL_NOTE-no-hint"
+fi
+
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
+if [ -n "$EXP_NAME" ]; then
+  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+
+function run_eval() {
+  local eval_note=$1
+  COMMAND="poetry run python evaluation/benchmarks/swe_bench/run_localize.py \
+    --agent-cls $AGENT \
+    --llm-config $MODEL_CONFIG \
+    --max-iterations $MAX_ITER \
+    --eval-num-workers $NUM_WORKERS \
+    --eval-note $eval_note \
+    --dataset $DATASET \
+    --split $SPLIT"
+
+  if [ -n "$EVAL_LIMIT" ]; then
+    echo "EVAL_LIMIT: $EVAL_LIMIT"
+    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+  fi
+
+  # Run the command
+  eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+if [ -z "$N_RUNS" ]; then
+  N_RUNS=1
+  echo "N_RUNS not specified, use default $N_RUNS"
+fi
+
+# Skip runs if the run number is in the SKIP_RUNS list
+# read from env variable SKIP_RUNS as a comma separated list of run numbers
+SKIP_RUNS=(${SKIP_RUNS//,/ })
+for i in $(seq 1 $N_RUNS); do
+  if [[ " ${SKIP_RUNS[@]} " =~ " $i " ]]; then
+    echo "Skipping run $i"
+    continue
+  fi
+  current_eval_note="$EVAL_NOTE-run_$i"
+  echo "EVAL_NOTE: $current_eval_note"
+  run_eval $current_eval_note
+done
+
+checkout_original_branch

evaluation/benchmarks/testgeneval/eval_infer.py

@@ -41,7 +41,7 @@ from evaluation.utils.shared import (
     reset_logger_for_multiprocessing,
     run_evaluation,
 )
-from openhands.core.config import AppConfig, SandboxConfig, get_parser
+from openhands.core.config import OpenHandsConfig, SandboxConfig, get_parser
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime
 from openhands.events.action import CmdRunAction
@@ -52,13 +52,13 @@ DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/kdja
 logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
 
 
-def get_config(instance: pd.Series) -> AppConfig:
+def get_config(instance: pd.Series) -> OpenHandsConfig:
     base_container_image = get_instance_docker_image(instance['instance_id_swebench'])
     assert base_container_image, (
         f'Invalid container image for instance {instance["instance_id_swebench"]}.'
     )
     logger.info(f'Using instance container image: {base_container_image}.')
-    return AppConfig(
+    return OpenHandsConfig(
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'eventstream'),
         sandbox=SandboxConfig(

evaluation/benchmarks/testgeneval/run_infer.py

@@ -35,7 +35,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     SandboxConfig,
     get_llm_config_arg,
     get_parser,
@@ -117,7 +117,7 @@ def get_instance_docker_image(instance_id: str) -> str:
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     # We use a different instance image for the each instance of TestGenEval
     base_container_image = get_instance_docker_image(instance['instance_id_swebench'])
     logger.info(
@@ -126,7 +126,7 @@ def get_config(
         f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         max_iterations=metadata.max_iterations,

evaluation/benchmarks/the_agent_company/run_infer.py

@@ -15,8 +15,8 @@ from browsing import pre_login
 from evaluation.utils.shared import get_default_sandbox_config_for_eval
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
     LLMConfig,
+    OpenHandsConfig,
     get_agent_config_arg,
     get_llm_config_arg,
     get_parser,
@@ -36,13 +36,13 @@ def get_config(
     mount_path_on_host: str,
     llm_config: LLMConfig,
     agent_config: AgentConfig | None,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = base_container_image
     sandbox_config.enable_auto_lint = True
     # If the web services are running on the host machine, this must be set to True
     sandbox_config.use_host_network = True
-    config = AppConfig(
+    config = OpenHandsConfig(
         run_as_openhands=False,
         max_budget_per_task=4,
         max_iterations=100,
@@ -126,7 +126,7 @@ def codeact_user_response(state: State) -> str:
 def run_solver(
     runtime: Runtime,
     task_name: str,
-    config: AppConfig,
+    config: OpenHandsConfig,
     dependencies: list[str],
     save_final_state: bool,
     state_dir: str,
@@ -274,7 +274,7 @@ if __name__ == '__main__':
         temp_dir = os.path.abspath(os.getenv('TMPDIR'))
     else:
         temp_dir = tempfile.mkdtemp()
-    config: AppConfig = get_config(
+    config: OpenHandsConfig = get_config(
         args.task_image_name, task_short_name, temp_dir, agent_llm_config, agent_config
     )
     runtime: Runtime = create_runtime(config)

evaluation/benchmarks/toolqa/run_infer.py

@@ -18,7 +18,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -40,10 +40,10 @@ AGENT_CLS_TO_INST_SUFFIX = {
 
 def get_config(
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/visual_swe_bench/run_infer.py

@@ -30,7 +30,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     get_parser,
 )
@@ -135,7 +135,7 @@ def get_instance_docker_image(instance_id: str, official_image: bool = False) ->
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
-) -> AppConfig:
+) -> OpenHandsConfig:
     # We use a different instance image for the each instance of swe-bench eval
     use_official_image = bool(
         'verified' in metadata.dataset.lower() or 'lite' in metadata.dataset.lower()
@@ -160,7 +160,7 @@ def get_config(
         instance_id=instance['instance_id'],
     )
 
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         max_iterations=metadata.max_iterations,

evaluation/benchmarks/visualwebarena/run_infer.py

@@ -20,7 +20,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -48,7 +48,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 def get_config(
     metadata: EvalMetadata,
     env_id: str,
-) -> AppConfig:
+) -> OpenHandsConfig:
     base_url = os.environ.get('VISUALWEBARENA_BASE_URL', None)
     openai_api_key = os.environ.get('OPENAI_API_KEY', None)
     openai_base_url = os.environ.get('OPENAI_BASE_URL', None)
@@ -72,7 +72,7 @@ def get_config(
         'VWA_WIKIPEDIA': f'{base_url}:8888',
         'VWA_HOMEPAGE': f'{base_url}:4399',
     }
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/benchmarks/webarena/run_infer.py

@@ -19,7 +19,7 @@ from evaluation.utils.shared import (
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -44,7 +44,7 @@ SUPPORTED_AGENT_CLS = {'BrowsingAgent'}
 def get_config(
     metadata: EvalMetadata,
     env_id: str,
-) -> AppConfig:
+) -> OpenHandsConfig:
     base_url = os.environ.get('WEBARENA_BASE_URL', None)
     openai_api_key = os.environ.get('OPENAI_API_KEY', None)
     assert base_url is not None, 'WEBARENA_BASE_URL must be set'
@@ -64,7 +64,7 @@ def get_config(
         'MAP': f'{base_url}:3000',
         'HOMEPAGE': f'{base_url}:4399',
     }
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime='docker',

evaluation/integration_tests/run_infer.py

@@ -21,7 +21,7 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
-    AppConfig,
+    OpenHandsConfig,
     get_llm_config_arg,
     parse_arguments,
 )
@@ -41,10 +41,10 @@ FAKE_RESPONSES = {
 def get_config(
     metadata: EvalMetadata,
     instance_id: str,
-) -> AppConfig:
+) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.platform = 'linux/amd64'
-    config = AppConfig(
+    config = OpenHandsConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
         runtime=os.environ.get('RUNTIME', 'docker'),

evaluation/regression/run_tests.py

@@ -2,9 +2,9 @@ import argparse
 
 import pytest
 
-from openhands.config import load_app_config
+from openhands.config import load_openhands_config
 
-config = load_app_config()
+config = load_openhands_config()
 
 if __name__ == '__main__':
     """Main entry point of the script.

frontend/__tests__/components/chat/action-suggestions.test.tsx

@@ -4,7 +4,6 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
 import { ActionSuggestions } from "#/components/features/chat/action-suggestions";
 import OpenHands from "#/api/open-hands";
 import { MOCK_DEFAULT_USER_SETTINGS } from "#/mocks/handlers";
-import { ConversationProvider } from "#/context/conversation-context";
 
 // Mock dependencies
 vi.mock("posthog-js", () => ({
@@ -48,11 +47,9 @@ vi.mock("react-router", () => ({
 const renderActionSuggestions = () =>
   render(<ActionSuggestions onSuggestionsClick={() => {}} />, {
     wrapper: ({ children }) => (
-      <ConversationProvider>
-        <QueryClientProvider client={new QueryClient()}>
-          {children}
-        </QueryClientProvider>
-      </ConversationProvider>
+      <QueryClientProvider client={new QueryClient()}>
+        {children}
+      </QueryClientProvider>
     ),
   });
 

frontend/__tests__/components/features/conversation-panel/conversation-panel.test.tsx

@@ -7,6 +7,7 @@ import React from "react";
 import { renderWithProviders } from "test-utils";
 import { ConversationPanel } from "#/components/features/conversation-panel/conversation-panel";
 import OpenHands from "#/api/open-hands";
+import { Conversation } from "#/api/open-hands.types";
 
 describe("ConversationPanel", () => {
   const onCloseMock = vi.fn();
@@ -37,11 +38,13 @@ describe("ConversationPanel", () => {
     }));
   });
 
-  const mockConversations = [
+  const mockConversations: Conversation[] = [
     {
       conversation_id: "1",
       title: "Conversation 1",
       selected_repository: null,
+      git_provider: null,
+      selected_branch: null,
       last_updated_at: "2021-10-01T12:00:00Z",
       created_at: "2021-10-01T12:00:00Z",
       status: "STOPPED" as const,
@@ -52,6 +55,8 @@ describe("ConversationPanel", () => {
       conversation_id: "2",
       title: "Conversation 2",
       selected_repository: null,
+      git_provider: null,
+      selected_branch: null,
       last_updated_at: "2021-10-02T12:00:00Z",
       created_at: "2021-10-02T12:00:00Z",
       status: "STOPPED" as const,
@@ -62,6 +67,8 @@ describe("ConversationPanel", () => {
       conversation_id: "3",
       title: "Conversation 3",
       selected_repository: null,
+      git_provider: null,
+      selected_branch: null,
       last_updated_at: "2021-10-03T12:00:00Z",
       created_at: "2021-10-03T12:00:00Z",
       status: "STOPPED" as const,
@@ -141,11 +148,13 @@ describe("ConversationPanel", () => {
 
   it("should delete a conversation", async () => {
     const user = userEvent.setup();
-    const mockData = [
+    const mockData: Conversation[] = [
       {
         conversation_id: "1",
         title: "Conversation 1",
         selected_repository: null,
+        git_provider: null,
+        selected_branch: null,
         last_updated_at: "2021-10-01T12:00:00Z",
         created_at: "2021-10-01T12:00:00Z",
         status: "STOPPED" as const,
@@ -156,6 +165,8 @@ describe("ConversationPanel", () => {
         conversation_id: "2",
         title: "Conversation 2",
         selected_repository: null,
+        git_provider: null,
+        selected_branch: null,
         last_updated_at: "2021-10-02T12:00:00Z",
         created_at: "2021-10-02T12:00:00Z",
         status: "STOPPED" as const,
@@ -166,6 +177,8 @@ describe("ConversationPanel", () => {
         conversation_id: "3",
         title: "Conversation 3",
         selected_repository: null,
+        git_provider: null,
+        selected_branch: null,
         last_updated_at: "2021-10-03T12:00:00Z",
         created_at: "2021-10-03T12:00:00Z",
         status: "STOPPED" as const,

frontend/__tests__/components/features/home/home-header.test.tsx

@@ -8,6 +8,29 @@ import userEvent from "@testing-library/user-event";
 import { HomeHeader } from "#/components/features/home/home-header";
 import OpenHands from "#/api/open-hands";
 
+// Mock the translation function
+vi.mock("react-i18next", async () => {
+  const actual = await vi.importActual("react-i18next");
+  return {
+    ...actual,
+    useTranslation: () => ({
+      t: (key: string) => {
+        // Return a mock translation for the test
+        const translations: Record<string, string> = {
+          "HOME$LETS_START_BUILDING": "Let's start building",
+          "HOME$LAUNCH_FROM_SCRATCH": "Launch from Scratch",
+          "HOME$LOADING": "Loading...",
+          "HOME$OPENHANDS_DESCRIPTION": "OpenHands is an AI software engineer",
+          "HOME$NOT_SURE_HOW_TO_START": "Not sure how to start?",
+          "HOME$READ_THIS": "Read this"
+        };
+        return translations[key] || key;
+      },
+      i18n: { language: "en" },
+    }),
+  };
+});
+
 const renderHomeHeader = () => {
   const RouterStub = createRoutesStub([
     {
@@ -38,7 +61,7 @@ describe("HomeHeader", () => {
     renderHomeHeader();
 
     const launchButton = screen.getByRole("button", {
-      name: /launch from scratch/i,
+      name: /Launch from Scratch/i,
     });
     await userEvent.click(launchButton);
 
@@ -60,11 +83,11 @@ describe("HomeHeader", () => {
     renderHomeHeader();
 
     const launchButton = screen.getByRole("button", {
-      name: /launch from scratch/i,
+      name: /Launch from Scratch/i,
     });
     await userEvent.click(launchButton);
 
-    expect(launchButton).toHaveTextContent(/Loading/i);
+    expect(launchButton).toHaveTextContent(/Loading.../i);
     expect(launchButton).toBeDisabled();
   });
 });

frontend/__tests__/context/ws-client-provider.test.tsx

@@ -56,15 +56,15 @@ function TestComponent() {
 describe("WsClientProvider", () => {
   beforeEach(() => {
     vi.clearAllMocks();
-    vi.mock("#/hooks/query/use-user-conversation", () => ({
-      useUserConversation: () => {
+    vi.mock("#/hooks/query/use-active-conversation", () => ({
+      useActiveConversation: () => {
         return { data: {
         conversation_id: "1",
         title: "Conversation 1",
         selected_repository: null,
         last_updated_at: "2021-10-01T12:00:00Z",
         created_at: "2021-10-01T12:00:00Z",
-        status: "STOPPED" as const,
+        status: "RUNNING" as const,
         url: null,
         session_api_key: null,
       }}},

frontend/__tests__/routes/llm-settings.test.tsx

@@ -48,7 +48,7 @@ describe("Content", () => {
 
       await waitFor(() => {
         expect(provider).toHaveValue("Anthropic");
-        expect(model).toHaveValue("claude-3-7-sonnet-20250219");
+        expect(model).toHaveValue("claude-sonnet-4-20250514");
 
         expect(apiKey).toHaveValue("");
         expect(apiKey).toHaveProperty("placeholder", "");
@@ -105,7 +105,7 @@ describe("Content", () => {
       within(advancedForm).getByTestId("llm-custom-model-input");
       within(advancedForm).getByTestId("base-url-input");
       within(advancedForm).getByTestId("llm-api-key-input");
-      within(advancedForm).getByTestId("llm-api-key-help-anchor");
+      within(advancedForm).getByTestId("llm-api-key-help-anchor-advanced");
       within(advancedForm).getByTestId("agent-input");
       within(advancedForm).getByTestId("enable-confirmation-mode-switch");
       within(advancedForm).getByTestId("enable-memory-condenser-switch");
@@ -135,7 +135,7 @@ describe("Content", () => {
       );
       const condensor = screen.getByTestId("enable-memory-condenser-switch");
 
-      expect(model).toHaveValue("anthropic/claude-3-7-sonnet-20250219");
+      expect(model).toHaveValue("anthropic/claude-sonnet-4-20250514");
       expect(baseUrl).toHaveValue("");
       expect(apiKey).toHaveValue("");
       expect(apiKey).toHaveProperty("placeholder", "");
@@ -542,7 +542,7 @@ describe("Form submission", () => {
 
     // select model
     await userEvent.click(model);
-    const modelOption = screen.getByText("claude-3-7-sonnet-20250219");
+    const modelOption = screen.getByText("claude-sonnet-4-20250514");
     await userEvent.click(modelOption);
 
     const submitButton = screen.getByTestId("submit-button");
@@ -550,7 +550,7 @@ describe("Form submission", () => {
 
     expect(saveSettingsSpy).toHaveBeenCalledWith(
       expect.objectContaining({
-        llm_model: "anthropic/claude-3-7-sonnet-20250219",
+        llm_model: "anthropic/claude-sonnet-4-20250514",
         llm_base_url: "",
         confirmation_mode: false,
       }),

frontend/__tests__/utils/check-hardcoded-strings.test.tsx

@@ -19,7 +19,11 @@ describe("Check for hardcoded English strings", () => {
     const text = container.textContent;
 
     // List of English strings that should be translated
-    const hardcodedStrings = ["What do you want to build?"];
+    const hardcodedStrings = [
+      "What do you want to build?",
+      "Launch from Scratch",
+      "Read this"
+    ];
 
     // Check each string
     hardcodedStrings.forEach((str) => {

frontend/__tests__/utils/check-home-hardcoded-strings.test.tsx

@@ -0,0 +1,42 @@
+import { render } from "@testing-library/react";
+import { test, expect, describe, vi } from "vitest";
+import { HomeHeader } from "#/components/features/home/home-header";
+
+// Mock dependencies
+vi.mock("#/hooks/mutation/use-create-conversation", () => ({
+  useCreateConversation: () => ({
+    mutate: vi.fn(),
+    isPending: false,
+    isSuccess: false,
+  }),
+}));
+
+vi.mock("#/hooks/use-is-creating-conversation", () => ({
+  useIsCreatingConversation: () => false,
+}));
+
+vi.mock("react-i18next", () => ({
+  useTranslation: () => ({
+    t: (key: string) => key,
+  }),
+}));
+
+describe("Check for hardcoded English strings in Home components", () => {
+  test("HomeHeader should not have hardcoded English strings", () => {
+    const { container } = render(<HomeHeader />);
+
+    // Get all text content
+    const text = container.textContent;
+
+    // List of English strings that should be translated
+    const hardcodedStrings = [
+      "Launch from Scratch",
+      "Read this",
+    ];
+
+    // Check each string
+    hardcodedStrings.forEach((str) => {
+      expect(text).not.toContain(str);
+    });
+  });
+});

frontend/__tests__/utils/extract-model-and-provider.test.ts

@@ -71,6 +71,18 @@ describe("extractModelAndProvider", () => {
       separator: "/",
     });
 
+    expect(extractModelAndProvider("claude-sonnet-4-20250514")).toEqual({
+      provider: "anthropic",
+      model: "claude-sonnet-4-20250514",
+      separator: "/",
+    });
+
+    expect(extractModelAndProvider("claude-opus-4-20250514")).toEqual({
+      provider: "anthropic",
+      model: "claude-opus-4-20250514",
+      separator: "/",
+    });
+
     expect(extractModelAndProvider("claude-3-haiku-20240307")).toEqual({
       provider: "anthropic",
       model: "claude-3-haiku-20240307",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.39.1",
+  "version": "0.39.2",
   "private": true,
   "type": "module",
   "engines": {
@@ -10,28 +10,28 @@
     "@heroui/react": "2.7.8",
     "@microlink/react-json-view": "^1.26.2",
     "@monaco-editor/react": "^4.7.0-rc.0",
-    "@react-router/node": "^7.6.0",
-    "@react-router/serve": "^7.6.0",
+    "@react-router/node": "^7.6.1",
+    "@react-router/serve": "^7.6.1",
     "@react-types/shared": "^3.29.1",
     "@reduxjs/toolkit": "^2.8.2",
     "@stripe/react-stripe-js": "^3.7.0",
     "@stripe/stripe-js": "^7.3.0",
-    "@tanstack/react-query": "^5.76.1",
+    "@tanstack/react-query": "^5.77.2",
     "@vitejs/plugin-react": "^4.4.0",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.4.0",
     "axios": "^1.9.0",
     "clsx": "^2.1.1",
     "eslint-config-airbnb-typescript": "^18.0.0",
-    "framer-motion": "^12.12.1",
-    "i18next": "^25.1.3",
+    "framer-motion": "^12.14.0",
+    "i18next": "^25.2.1",
     "i18next-browser-languagedetector": "^8.1.0",
     "i18next-http-backend": "^3.0.2",
     "isbot": "^5.1.28",
     "jose": "^6.0.11",
     "lucide-react": "^0.511.0",
     "monaco-editor": "^0.52.2",
-    "posthog-js": "^1.245.1",
+    "posthog-js": "^1.245.2",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
     "react-highlight": "^0.15.0",
@@ -40,7 +40,7 @@
     "react-icons": "^5.5.0",
     "react-markdown": "^10.1.0",
     "react-redux": "^9.2.0",
-    "react-router": "^7.6.0",
+    "react-router": "^7.6.1",
     "react-syntax-highlighter": "^15.6.1",
     "react-textarea-autosize": "^8.5.9",
     "remark-gfm": "^4.0.1",
@@ -83,9 +83,9 @@
     "@babel/types": "^7.27.0",
     "@mswjs/socket.io-binding": "^0.1.1",
     "@playwright/test": "^1.52.0",
-    "@react-router/dev": "^7.6.0",
+    "@react-router/dev": "^7.6.1",
     "@tailwindcss/typography": "^0.5.16",
-    "@tanstack/eslint-plugin-query": "^5.74.7",
+    "@tanstack/eslint-plugin-query": "^5.78.0",
     "@testing-library/dom": "^10.4.0",
     "@testing-library/jest-dom": "^6.6.1",
     "@testing-library/react": "^16.3.0",

frontend/public/mockServiceWorker.js

@@ -8,7 +8,7 @@
  * - Please do NOT serve this file on production.
  */
 
-const PACKAGE_VERSION = '2.7.6'
+const PACKAGE_VERSION = '2.8.4'
 const INTEGRITY_CHECKSUM = '00729d72e3b82faf54ca8b9621dbb96f'
 const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
 const activeClientIds = new Set()

frontend/scripts/check-unlocalized-strings.cjs

@@ -113,6 +113,7 @@ const EXCLUDED_TECHNICAL_STRINGS = [
   "GitHub API", // Git provider specific terminology
   "add-secret-form", // Test ID for secret form
   "edit-secret-form", // Test ID for secret form
+  "search-api-key-input", // Input name for search API key
 ];
 
 function isExcludedTechnicalString(str) {

frontend/src/api/open-hands.types.ts

@@ -76,6 +76,8 @@ export interface Conversation {
   conversation_id: string;
   title: string;
   selected_repository: string | null;
+  selected_branch: string | null;
+  git_provider: string | null;
   last_updated_at: string;
   created_at: string;
   status: ProjectStatus;

frontend/src/components/features/browser/browser.tsx

@@ -3,7 +3,7 @@ import { useSelector, useDispatch } from "react-redux";
 import { RootState } from "#/store";
 import { BrowserSnapshot } from "./browser-snapshot";
 import { EmptyBrowserMessage } from "./empty-browser-message";
-import { useConversation } from "#/context/conversation-context";
+import { useConversationId } from "#/hooks/use-conversation-id";
 import {
   initialState as browserInitialState,
   setUrl,
@@ -14,7 +14,7 @@ export function BrowserPanel() {
   const { url, screenshotSrc } = useSelector(
     (state: RootState) => state.browser,
   );
-  const { conversationId } = useConversation();
+  const { conversationId } = useConversationId();
   const dispatch = useDispatch();
 
   useEffect(() => {

frontend/src/components/features/chat/action-suggestions.tsx

@@ -4,8 +4,7 @@ import { useTranslation } from "react-i18next";
 import { SuggestionItem } from "#/components/features/suggestions/suggestion-item";
 import { I18nKey } from "#/i18n/declaration";
 import { useUserProviders } from "#/hooks/use-user-providers";
-import { useConversation } from "#/context/conversation-context";
-import { useUserConversation } from "#/hooks/query/use-user-conversation";
+import { useActiveConversation } from "#/hooks/query/use-active-conversation";
 
 interface ActionSuggestionsProps {
   onSuggestionsClick: (value: string) => void;
@@ -16,9 +15,7 @@ export function ActionSuggestions({
 }: ActionSuggestionsProps) {
   const { t } = useTranslation();
   const { providers } = useUserProviders();
-  const { conversationId } = useConversation();
-  const { data: conversation } = useUserConversation(conversationId);
-
+  const { data: conversation } = useActiveConversation();
   const [hasPullRequest, setHasPullRequest] = React.useState(false);
 
   const providersAreSet = providers.length > 0;

frontend/src/components/features/chat/chat-interface.tsx

@@ -26,7 +26,6 @@ import { downloadTrajectory } from "#/utils/download-trajectory";
 import { displayErrorToast } from "#/utils/custom-toast-handlers";
 import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
 import { useWSErrorMessage } from "#/hooks/use-ws-error-message";
-import i18n from "#/i18n";
 import { ErrorMessageBanner } from "./error-message-banner";
 import { shouldRenderEvent } from "./event-content-helpers/should-render-event";
 
@@ -153,11 +152,13 @@ export function ChatInterface() {
           />
         )}
 
-        {isWaitingForUserInput && (
-          <ActionSuggestions
-            onSuggestionsClick={(value) => handleSendMessage(value, [])}
-          />
-        )}
+        {isWaitingForUserInput &&
+          events.length > 0 &&
+          !optimisticUserMessage && (
+            <ActionSuggestions
+              onSuggestionsClick={(value) => handleSendMessage(value, [])}
+            />
+          )}
       </div>
 
       <div className="flex flex-col gap-[6px] px-4 pb-4">
@@ -179,11 +180,7 @@ export function ChatInterface() {
           {!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
         </div>
 
-        {errorMessage && (
-          <ErrorMessageBanner
-            message={i18n.exists(errorMessage) ? t(errorMessage) : errorMessage}
-          />
-        )}
+        {errorMessage && <ErrorMessageBanner message={errorMessage} />}
 
         <InteractiveChatBox
           onSubmit={handleSendMessage}

frontend/src/components/features/chat/error-message-banner.tsx

@@ -1,3 +1,7 @@
+import { Trans } from "react-i18next";
+import { Link } from "react-router";
+import i18n from "#/i18n";
+
 interface ErrorMessageBannerProps {
   message: string;
 }
@@ -5,7 +9,23 @@ interface ErrorMessageBannerProps {
 export function ErrorMessageBanner({ message }: ErrorMessageBannerProps) {
   return (
     <div className="w-full rounded-lg p-2 text-black border border-red-800 bg-red-500">
-      {message}
+      {i18n.exists(message) ? (
+        <Trans
+          i18nKey={message}
+          components={{
+            a: (
+              <Link
+                className="underline font-bold cursor-pointer"
+                to="/settings/billing"
+              >
+                link
+              </Link>
+            ),
+          }}
+        />
+      ) : (
+        message
+      )}
     </div>
   );
 }

frontend/src/components/features/chat/event-content-helpers/get-event-content.tsx

@@ -28,41 +28,58 @@ export const getEventContent = (
   let details: string = "";
 
   if (isOpenHandsAction(event)) {
-    title = (
-      <Trans
-        i18nKey={`ACTION_MESSAGE$${event.action.toUpperCase()}`}
-        values={{
-          path: hasPathProperty(event.args) && event.args.path,
-          command:
-            hasCommandProperty(event.args) && trimText(event.args.command, 80),
-          mcp_tool_name: event.action === "call_tool_mcp" && event.args.name,
-        }}
-        components={{
-          path: <PathComponent />,
-          cmd: <MonoComponent />,
-        }}
-      />
-    );
+    const actionKey = `ACTION_MESSAGE$${event.action.toUpperCase()}`;
+
+    // If translation key exists, use Trans component
+    if (i18n.exists(actionKey)) {
+      title = (
+        <Trans
+          i18nKey={actionKey}
+          values={{
+            path: hasPathProperty(event.args) && event.args.path,
+            command:
+              hasCommandProperty(event.args) &&
+              trimText(event.args.command, 80),
+            mcp_tool_name: event.action === "call_tool_mcp" && event.args.name,
+          }}
+          components={{
+            path: <PathComponent />,
+            cmd: <MonoComponent />,
+          }}
+        />
+      );
+    } else {
+      // For generic actions, just use the uppercase type
+      title = event.action.toUpperCase();
+    }
     details = getActionContent(event);
   }
 
   if (isOpenHandsObservation(event)) {
-    title = (
-      <Trans
-        i18nKey={`OBSERVATION_MESSAGE$${event.observation.toUpperCase()}`}
-        values={{
-          path: hasPathProperty(event.extras) && event.extras.path,
-          command:
-            hasCommandProperty(event.extras) &&
-            trimText(event.extras.command, 80),
-          mcp_tool_name: event.observation === "mcp" && event.extras.name,
-        }}
-        components={{
-          path: <PathComponent />,
-          cmd: <MonoComponent />,
-        }}
-      />
-    );
+    const observationKey = `OBSERVATION_MESSAGE$${event.observation.toUpperCase()}`;
+
+    // If translation key exists, use Trans component
+    if (i18n.exists(observationKey)) {
+      title = (
+        <Trans
+          i18nKey={observationKey}
+          values={{
+            path: hasPathProperty(event.extras) && event.extras.path,
+            command:
+              hasCommandProperty(event.extras) &&
+              trimText(event.extras.command, 80),
+            mcp_tool_name: event.observation === "mcp" && event.extras.name,
+          }}
+          components={{
+            path: <PathComponent />,
+            cmd: <MonoComponent />,
+          }}
+        />
+      );
+    } else {
+      // For generic observations, just use the uppercase type
+      title = event.observation.toUpperCase();
+    }
     details = getObservationContent(event);
   }
 

frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts

@@ -46,14 +46,6 @@ const getBrowseObservationContent = (event: BrowseObservation) => {
   return contentDetails;
 };
 
-const getMcpObservationContent = (event: OpenHandsObservation): string => {
-  let { content } = event;
-  if (content.length > MAX_CONTENT_LENGTH) {
-    content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
-  }
-  return `**Output:**\n\`\`\`\n${content.trim() || i18n.t("OBSERVATION$MCP_NO_OUTPUT")}\n\`\`\``;
-};
-
 const getRecallObservationContent = (event: RecallObservation): string => {
   let content = "";
 
@@ -124,8 +116,6 @@ export const getObservationContent = (event: OpenHandsObservation): string => {
       return getCommandObservationContent(event);
     case "browse":
       return getBrowseObservationContent(event);
-    case "mcp":
-      return getMcpObservationContent(event);
     case "recall":
       return getRecallObservationContent(event);
     default:

frontend/src/components/features/chat/event-content-helpers/should-render-event.ts

@@ -1,6 +1,11 @@
 import { OpenHandsAction } from "#/types/core/actions";
 import { OpenHandsEventType } from "#/types/core/base";
-import { isOpenHandsAction, isOpenHandsObservation } from "#/types/core/guards";
+import {
+  isCommandAction,
+  isCommandObservation,
+  isOpenHandsAction,
+  isOpenHandsObservation,
+} from "#/types/core/guards";
 import { OpenHandsObservation } from "#/types/core/observations";
 
 const COMMON_NO_RENDER_LIST: OpenHandsEventType[] = [
@@ -15,11 +20,21 @@ export const shouldRenderEvent = (
   event: OpenHandsAction | OpenHandsObservation,
 ) => {
   if (isOpenHandsAction(event)) {
+    if (isCommandAction(event) && event.source === "user") {
+      // For user commands, we always hide them from the chat interface
+      return false;
+    }
+
     const noRenderList = COMMON_NO_RENDER_LIST.concat(ACTION_NO_RENDER_LIST);
     return !noRenderList.includes(event.action);
   }
 
   if (isOpenHandsObservation(event)) {
+    if (isCommandObservation(event) && event.source === "user") {
+      // For user commands, we always hide them from the chat interface
+      return false;
+    }
+
     return !COMMON_NO_RENDER_LIST.includes(event.observation);
   }
 

frontend/src/components/features/chat/event-message.tsx

@@ -8,11 +8,13 @@ import {
   isOpenHandsObservation,
   isFinishAction,
   isRejectObservation,
+  isMcpObservation,
 } from "#/types/core/guards";
 import { OpenHandsObservation } from "#/types/core/observations";
 import { ImageCarousel } from "../images/image-carousel";
 import { ChatMessage } from "./chat-message";
 import { ErrorMessage } from "./error-message";
+import { MCPObservationContent } from "./mcp-observation-content";
 import { getObservationResult } from "./event-content-helpers/get-observation-result";
 import { getEventContent } from "./event-content-helpers/get-event-content";
 import { GenericEventMessage } from "./generic-event-message";
@@ -46,12 +48,11 @@ export function EventMessage({
     );
   }
 
-  if (
-    hasObservationPair &&
-    isOpenHandsAction(event) &&
-    hasThoughtProperty(event.args)
-  ) {
-    return <ChatMessage type="agent" message={event.args.thought} />;
+  if (hasObservationPair && isOpenHandsAction(event)) {
+    if (hasThoughtProperty(event.args)) {
+      return <ChatMessage type="agent" message={event.args.thought} />;
+    }
+    return null;
   }
 
   if (isFinishAction(event)) {
@@ -78,6 +79,19 @@ export function EventMessage({
     return <ChatMessage type="agent" message={event.content} />;
   }
 
+  if (isMcpObservation(event)) {
+    return (
+      <div>
+        <GenericEventMessage
+          title={getEventContent(event).title}
+          details={<MCPObservationContent event={event} />}
+          success={getObservationResult(event)}
+        />
+        {shouldShowConfirmationButtons && <ConfirmationButtons />}
+      </div>
+    );
+  }
+
   return (
     <div>
       {isOpenHandsAction(event) && hasThoughtProperty(event.args) && (

frontend/src/components/features/chat/generic-event-message.tsx

@@ -10,7 +10,7 @@ import { ObservationResultStatus } from "./event-content-helpers/get-observation
 
 interface GenericEventMessageProps {
   title: React.ReactNode;
-  details: string;
+  details: string | React.ReactNode;
   success?: ObservationResultStatus;
 }
 
@@ -44,18 +44,21 @@ export function GenericEventMessage({
         {success && <SuccessIndicator status={success} />}
       </div>
 
-      {showDetails && (
-        <Markdown
-          components={{
-            code,
-            ul,
-            ol,
-          }}
-          remarkPlugins={[remarkGfm]}
-        >
-          {details}
-        </Markdown>
-      )}
+      {showDetails &&
+        (typeof details === "string" ? (
+          <Markdown
+            components={{
+              code,
+              ul,
+              ol,
+            }}
+            remarkPlugins={[remarkGfm]}
+          >
+            {details}
+          </Markdown>
+        ) : (
+          details
+        ))}
     </div>
   );
 }

frontend/src/components/features/chat/mcp-observation-content.tsx

@@ -0,0 +1,73 @@
+import React from "react";
+import ReactJsonView from "@microlink/react-json-view";
+import { useTranslation } from "react-i18next";
+import { MCPObservation } from "#/types/core/observations";
+import { JSON_VIEW_THEME } from "#/utils/constants";
+
+interface MCPObservationContentProps {
+  event: MCPObservation;
+}
+
+export function MCPObservationContent({ event }: MCPObservationContentProps) {
+  const { t } = useTranslation();
+
+  // Parse the content as JSON if possible
+  let outputData: unknown;
+  try {
+    outputData = JSON.parse(event.content);
+  } catch (e) {
+    // If parsing fails, use the raw content
+    outputData = event.content;
+  }
+
+  const hasArguments =
+    event.extras.arguments && Object.keys(event.extras.arguments).length > 0;
+
+  return (
+    <div className="flex flex-col gap-4">
+      {/* Arguments section */}
+      {hasArguments && (
+        <div className="flex flex-col gap-2">
+          <div className="flex items-center justify-between">
+            <h3 className="text-sm font-semibold text-gray-300">
+              {t("MCP_OBSERVATION$ARGUMENTS")}
+            </h3>
+          </div>
+          <div className="p-3 bg-gray-900 rounded-md overflow-auto text-gray-300 max-h-[200px] shadow-inner">
+            <ReactJsonView
+              name={false}
+              src={event.extras.arguments}
+              theme={JSON_VIEW_THEME}
+              collapsed={1}
+              displayDataTypes={false}
+            />
+          </div>
+        </div>
+      )}
+
+      {/* Output section */}
+      <div className="flex flex-col gap-2">
+        <div className="flex items-center justify-between">
+          <h3 className="text-sm font-semibold text-gray-300">
+            {t("MCP_OBSERVATION$OUTPUT")}
+          </h3>
+        </div>
+        <div className="p-3 bg-gray-900 rounded-md overflow-auto text-gray-300 max-h-[300px] shadow-inner">
+          {typeof outputData === "object" && outputData !== null ? (
+            <ReactJsonView
+              name={false}
+              src={outputData}
+              theme={JSON_VIEW_THEME}
+              collapsed={1}
+              displayDataTypes={false}
+            />
+          ) : (
+            <pre className="whitespace-pre-wrap">
+              {event.content.trim() || t("OBSERVATION$MCP_NO_OUTPUT")}
+            </pre>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}