migrate to use OH version

Co-authored-by: openhands <openhands@all-hands.dev>

.github/workflows/integration-runner.yml

@@ -160,7 +160,6 @@ jobs:
           echo "api_key = \"$LLM_API_KEY\"" >> config.toml
           echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
           echo "temperature = 0.0" >> config.toml
-
       - name: Run integration test evaluation for DelegatorAgent (DeepSeek)
         env:
           SANDBOX_FORCE_REBUILD_RUNTIME: True
@@ -174,12 +173,42 @@ jobs:
           cat $REPORT_FILE_DELEGATOR_DEEPSEEK >> $GITHUB_ENV
           echo >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
+      # -------------------------------------------------------------
+      # Run VisualBrowsingAgent tests for DeepSeek, limited to t05 and t06
+      - name: Wait a little bit (again)
+        run: sleep 5
+
+      - name: Configure config.toml for testing VisualBrowsingAgent (DeepSeek)
+        env:
+          LLM_MODEL: "litellm_proxy/deepseek-chat"
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          MAX_ITERATIONS: 15
+        run: |
+          echo "[llm.eval]" > config.toml
+          echo "model = \"$LLM_MODEL\"" >> config.toml
+          echo "api_key = \"$LLM_API_KEY\"" >> config.toml
+          echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
+          echo "temperature = 0.0" >> config.toml
+      - name: Run integration test evaluation for VisualBrowsingAgent (DeepSeek)
+        env:
+          SANDBOX_FORCE_REBUILD_RUNTIME: True
+        run: |
+          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD VisualBrowsingAgent '' 15 $N_PROCESSES "t05_simple_browsing,t06_github_pr_browsing.py" 'visualbrowsing_deepseek_run'
+
+          # Find and export the visual browsing agent test results
+          REPORT_FILE_VISUALBROWSING_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/VisualBrowsingAgent/deepseek*_maxiter_15_N* -name "report.md" -type f | head -n 1)
+          echo "REPORT_FILE_VISUALBROWSING_DEEPSEEK: $REPORT_FILE_VISUALBROWSING_DEEPSEEK"
+          echo "INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK<<EOF" >> $GITHUB_ENV
+          cat $REPORT_FILE_VISUALBROWSING_DEEPSEEK >> $GITHUB_ENV
+          echo >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
 
       - name: Create archive of evaluation outputs
         run: |
           TIMESTAMP=$(date +'%y-%m-%d-%H-%M')
           cd evaluation/evaluation_outputs/outputs  # Change to the outputs directory
-          tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/* integration_tests/DelegatorAgent/*  # Only include the actual result directories
+          tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/* integration_tests/DelegatorAgent/* integration_tests/VisualBrowsingAgent/* # Only include the actual result directories
 
       - name: Upload evaluation results as artifact
         uses: actions/upload-artifact@v4
@@ -227,4 +256,7 @@ jobs:
               **Integration Tests Report Delegator (DeepSeek)**
               ${{ env.INTEGRATION_TEST_REPORT_DELEGATOR_DEEPSEEK }}
               ---
+              **Integration Tests Report VisualBrowsing (DeepSeek)**
+              ${{ env.INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK }}
+              ---
               Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }})

.github/workflows/openhands-resolver.yml

@@ -84,6 +84,10 @@ jobs:
         run: |
           python -m pip index versions openhands-ai > openhands_versions.txt
           OPENHANDS_VERSION=$(head -n 1 openhands_versions.txt | awk '{print $2}' | tr -d '()')
+          # Ensure requirements.txt ends with newline before appending
+          if [ -f requirements.txt ] && [ -s requirements.txt ]; then
+            sed -i -e '$a\' requirements.txt
+          fi
           echo "openhands-ai==${OPENHANDS_VERSION}" >> requirements.txt
           cat requirements.txt
 

.gitignore

@@ -176,6 +176,7 @@ evaluation/gorilla/data
 evaluation/toolqa/data
 evaluation/scienceagentbench/benchmark
 evaluation/commit0_bench/repos
+evaluation/visualcodebench/
 
 # openhands resolver
 output/

Development.md

@@ -100,7 +100,7 @@ poetry run pytest ./tests/unit/test_*.py
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by
 setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
 
-Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.20-nikolaik`
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.21-nikolaik`
 
 ## Develop inside Docker container
 

README.md

@@ -39,21 +39,21 @@ Learn more at [docs.all-hands.dev](https://docs.all-hands.dev), or jump to the [
 ## ⚡ Quick Start
 
 The easiest way to run OpenHands is in Docker.
-See the [Installation](https://docs.all-hands.dev/modules/usage/installation) guide for
+See the [Running OpenHands](https://docs.all-hands.dev/modules/usage/installation) guide for
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands-state:/.openhands-state \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20
+    docker.all-hands.dev/all-hands-ai/openhands:0.21
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
@@ -69,7 +69,7 @@ run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules
 interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode),
 or run it on tagged issues with [a github action](https://docs.all-hands.dev/modules/usage/how-to/github-action).
 
-Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
+Visit [Running OpenHands](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
 > [!CAUTION]
 > OpenHands is meant to be run by a single user on their local workstation.

config.template.toml

@@ -75,7 +75,7 @@ workspace_base = "./workspace"
 #run_as_openhands = true
 
 # Runtime environment
-#runtime = "eventstream"
+#runtime = "docker"
 
 # Name of the default agent
 #default_agent = "CodeActAgent"

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.20-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.21-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik}
       #- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of openhands-state for this user
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/configuration-options.md

@@ -373,7 +373,7 @@ Les options de configuration de l'agent sont définies dans les sections `[agent
   - Description : Si l'éditeur LLM est activé dans l'espace d'action (fonctionne uniquement avec l'appel de fonction)
 
 **Utilisation du micro-agent**
-- `use_microagents`
+- `enable_prompt_extensions`
   - Type : `bool`
   - Valeur par défaut : `true`
   - Description : Indique si l'utilisation des micro-agents est activée ou non

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/cli-mode.md

@@ -52,7 +52,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -61,7 +61,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.cli
 ```
 

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -46,7 +46,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -56,6 +56,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.main -t "write a bash script that prints hi" --no-auto-continue
 ```

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/installation.mdx

@@ -13,16 +13,16 @@
 La façon la plus simple d'exécuter OpenHands est avec Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20
+    docker.all-hands.dev/all-hands-ai/openhands:0.21
 ```
 
 Vous pouvez également exécuter OpenHands en mode [headless scriptable](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), en tant que [CLI interactive](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), ou en utilisant l'[Action GitHub OpenHands](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/runtimes.md

@@ -13,7 +13,7 @@ C'est le Runtime par défaut qui est utilisé lorsque vous démarrez OpenHands.
 
 ```
 docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     # ...
 ```

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/cli-mode.md

@@ -50,7 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -59,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.cli
 ```
 

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -47,7 +47,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -57,6 +57,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.main -t "write a bash script that prints hi" --no-auto-continue
 ```

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/installation.mdx

@@ -11,16 +11,16 @@
 在 Docker 中运行 OpenHands 是最简单的方式。
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20
+    docker.all-hands.dev/all-hands-ai/openhands:0.21
 ```
 
 你也可以在可脚本化的[无头模式](https://docs.all-hands.dev/modules/usage/how-to/headless-mode)下运行 OpenHands，作为[交互式 CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode)，或使用 [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action)。

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/runtimes.md

@@ -11,7 +11,7 @@
 
 ```
 docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     # ...
 ```

docs/modules/usage/getting-started.mdx

@@ -1,6 +1,6 @@
 # Getting Started with OpenHands
 
-So you've [installed OpenHands](./installation) and have
+So you've [run OpenHands](./installation) and have
 [set up your LLM](./installation#setup). Now what?
 
 OpenHands can help you tackle a wide variety of engineering tasks. But the technology

docs/modules/usage/how-to/cli-mode.md

@@ -35,7 +35,7 @@ To run OpenHands in CLI mode with Docker:
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -45,7 +45,7 @@ docker run -it \
     -v ~/.openhands-state:/.openhands-state \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.cli
 ```
 

docs/modules/usage/how-to/headless-mode.md

@@ -32,7 +32,7 @@ To run OpenHands in Headless mode with Docker:
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -43,7 +43,7 @@ docker run -it \
     -v ~/.openhands-state:/.openhands-state \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.21 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
 

docs/modules/usage/installation.mdx

@@ -1,27 +1,66 @@
-# Installation
+# Running OpenHands
 
 ## System Requirements
 
-- Docker version 26.0.0+ or Docker Desktop 4.31.0+.
-- You must be using Linux or Mac OS.
-  - If you are on Windows, you must use [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+- MacOS with [Docker Desktop support](https://docs.docker.com/desktop/setup/install/mac-install/#system-requirements)
+- Linux
+- Windows with [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) and [Docker Desktop support](https://docs.docker.com/desktop/setup/install/windows-install/#system-requirements)
 
-## Start the app
+## Prerequisites
+
+<details>
+  <summary>MacOS</summary>
+  ### Docker Desktop
+
+  1. [Install Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install).
+  2. Open Docker Desktop, go to `Settings > Advanced` and ensure `Allow the default Docker socket to be used` is enabled.
+</details>
+
+<details>
+  <summary>Linux</summary>
+
+  :::note
+  Tested with Ubuntu 22.04.
+  :::
+
+  ### Docker Desktop
+
+  1. [Install Docker Desktop on Linux](https://docs.docker.com/desktop/setup/install/linux/).
+
+</details>
+
+<details>
+  <summary>Windows</summary>
+  ### WSL
+
+  1. [Install WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+  2. Run `wsl --version` in powershell and confirm `Default Version: 2`.
+
+  ### Docker Desktop
+
+  1. [Install Docker Desktop on Windows](https://docs.docker.com/desktop/setup/install/windows-install).
+  2. Open Docker Desktop, go to `Settings` and confirm the following:
+  - General: `Use the WSL 2 based engine` is enabled.
+  - Resources > WSL Integration: `Enable integration with my default WSL distro` is enabled.
+
+</details>
+
+## Start the App
 
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands-state:/.openhands-state \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.20
+    docker.all-hands.dev/all-hands-ai/openhands:0.21
 ```
 
 You'll find OpenHands running at http://localhost:3000!

docs/modules/usage/runtimes.md

@@ -16,7 +16,7 @@ some flags being passed to `docker run` that make this possible:
 
 ```
 docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.20-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.21-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     # ...
 ```

docs/sidebars.ts

@@ -5,7 +5,7 @@ const sidebars: SidebarsConfig = {
   docsSidebar: [
     {
       type: 'doc',
-      label: 'Installation',
+      label: 'Running OpenHands',
       id: 'usage/installation',
     },
     {

evaluation/benchmarks/miniwob/README.md

@@ -8,6 +8,9 @@ Please follow instruction [here](../../README.md#setup) to setup your local deve
 
 ## Test if your environment works
 
+Follow the instructions here https://miniwob.farama.org/content/getting_started/ & https://miniwob.farama.org/content/viewing/
+to set up MiniWoB server in your local environment at http://localhost:8080/miniwob/
+
 Access with browser the above MiniWoB URLs and see if they load correctly.
 
 ## Run Evaluation

evaluation/benchmarks/swe_bench/eval_infer.py

@@ -71,7 +71,7 @@ def process_git_patch(patch):
     return patch
 
 
-def get_config(instance: pd.Series) -> AppConfig:
+def get_config(metadata: EvalMetadata, instance: pd.Series) -> AppConfig:
     # We use a different instance image for the each instance of swe-bench eval
     base_container_image = get_instance_docker_image(instance['instance_id'])
     logger.info(
@@ -132,7 +132,7 @@ def process_instance(
     else:
         logger.info(f'Starting evaluation for instance {instance.instance_id}.')
 
-    config = get_config(instance)
+    config = get_config(metadata, instance)
     instance_id = instance.instance_id
     model_patch = instance['model_patch']
     test_spec: TestSpec = instance['test_spec']

evaluation/benchmarks/swe_bench/run_infer.py

@@ -158,6 +158,7 @@ def get_config(
         codeact_enable_browsing=RUN_WITH_BROWSING,
         codeact_enable_llm_editor=False,
         condenser=metadata.condenser_config,
+        enable_prompt_extensions=False,
     )
     config.set_agent_config(agent_config)
     return config

evaluation/benchmarks/visualcodebench/eval.py

@@ -0,0 +1,674 @@
+from collections import Counter
+from copy import deepcopy
+from difflib import SequenceMatcher
+from io import BytesIO
+
+from bs4 import BeautifulSoup, Comment, NavigableString, Tag
+import cv2
+import numpy as np
+import torch
+from colormath.color_conversions import convert_color
+from colormath.color_diff import delta_e_cie2000
+from colormath.color_objects import LabColor, sRGBColor
+from PIL import Image, ImageChops, ImageColor
+from scipy.optimize import linear_sum_assignment
+from transformers import CLIPModel, CLIPProcessor
+
+from openhands.core.logger import openhands_logger as logger
+
+
+def calculate_similarity(block1, block2):
+    """Calculate text similarity between two blocks using SequenceMatcher."""
+    text_similarity = SequenceMatcher(None, block1['text'], block2['text']).ratio()
+    return text_similarity
+
+
+def adjust_cost_for_context(cost_matrix, consecutive_bonus=1.0, window_size=20):
+    """Adjust cost matrix by considering context similarity."""
+    if window_size <= 0:
+        return cost_matrix
+
+    n, m = cost_matrix.shape
+    adjusted_cost_matrix = np.copy(cost_matrix)
+
+    for i in range(n):
+        for j in range(m):
+            if adjusted_cost_matrix[i][j] >= -0.5:
+                continue
+            nearby_matrix = cost_matrix[
+                max(0, i - window_size) : min(n, i + window_size + 1),
+                max(0, j - window_size) : min(m, j + window_size + 1),
+            ]
+            flattened_array = nearby_matrix.flatten()
+            sorted_array = np.sort(flattened_array)[::-1]
+            sorted_array = np.delete(
+                sorted_array, np.where(sorted_array == cost_matrix[i, j])[0][0]
+            )
+            top_k_elements = sorted_array[-window_size * 2 :]
+            bonus = consecutive_bonus * np.sum(top_k_elements)
+            adjusted_cost_matrix[i][j] += bonus
+    return adjusted_cost_matrix
+
+
+def create_cost_matrix(A, B):
+    """Create cost matrix for block matching."""
+    n = len(A)
+    m = len(B)
+    cost_matrix = np.zeros((n, m))
+    for i in range(n):
+        for j in range(m):
+            cost_matrix[i, j] = -calculate_similarity(A[i], B[j])
+    return cost_matrix
+
+
+def calculate_distance_max_1d(x1, y1, x2, y2):
+    """Calculate maximum 1D distance between points."""
+    return max(abs(x2 - x1), abs(y2 - y1))
+
+
+def calculate_ratio(h1, h2):
+    """Calculate ratio between two heights."""
+    return max(h1, h2) / min(h1, h2)
+
+
+def rgb_to_lab(rgb):
+    """Convert RGB color to Lab color space."""
+    rgb_color = sRGBColor(rgb[0], rgb[1], rgb[2], is_upscaled=True)
+    lab_color = convert_color(rgb_color, LabColor)
+    return lab_color
+
+
+def color_similarity_ciede2000(rgb1, rgb2):
+    """Calculate color similarity using CIEDE2000 formula."""
+    lab1 = rgb_to_lab(rgb1)
+    lab2 = rgb_to_lab(rgb2)
+    delta_e = delta_e_cie2000(lab1, lab2)
+    similarity = max(0, 1 - (delta_e / 100))
+    return similarity
+
+
+def merge_blocks_wo_check(block1, block2):
+    """Merge two blocks without additional checks."""
+    merged_text = block1['text'] + ' ' + block2['text']
+    x_min = min(block1['bbox'][0], block2['bbox'][0])
+    y_min = min(block1['bbox'][1], block2['bbox'][1])
+    x_max = max(
+        block1['bbox'][0] + block1['bbox'][2], block2['bbox'][0] + block2['bbox'][2]
+    )
+    y_max = max(
+        block1['bbox'][1] + block1['bbox'][3], block2['bbox'][1] + block2['bbox'][3]
+    )
+    merged_bbox = (x_min, y_min, x_max - x_min, y_max - y_min)
+    merged_color = tuple(
+        (color1 + color2) // 2
+        for color1, color2 in zip(block1['color'], block2['color'])
+    )
+    return {'text': merged_text, 'bbox': merged_bbox, 'color': merged_color}
+
+
+def find_maximum_matching(A, B, consecutive_bonus, window_size):
+    """Find maximum matching between two sets of blocks."""
+    cost_matrix = create_cost_matrix(A, B)
+    cost_matrix = adjust_cost_for_context(cost_matrix, consecutive_bonus, window_size)
+    row_ind, col_ind = linear_sum_assignment(cost_matrix)
+    current_cost = cost_matrix[row_ind, col_ind].tolist()
+    return list(zip(row_ind, col_ind)), current_cost, cost_matrix
+
+
+def remove_indices(lst, indices):
+    """Remove indices from list in reverse order."""
+    for index in sorted(indices, reverse=True):
+        if index < len(lst):
+            lst.pop(index)
+    return lst
+
+
+def merge_blocks_by_list(blocks, merge_list):
+    """Merge blocks according to merge list."""
+    pop_list = []
+    while merge_list:
+        i = merge_list[0][0]
+        j = merge_list[0][1]
+        blocks[i] = merge_blocks_wo_check(blocks[i], blocks[j])
+        pop_list.append(j)
+        merge_list.pop(0)
+        if merge_list:
+            new_merge_list = []
+            for k in range(len(merge_list)):
+                if (
+                    merge_list[k][0] != i
+                    and merge_list[k][1] != i
+                    and merge_list[k][0] != j
+                    and merge_list[k][1] != j
+                ):
+                    new_merge_list.append(merge_list[k])
+            merge_list = new_merge_list
+    remove_indices(blocks, pop_list)
+    return blocks
+
+
+def difference_of_means(list1, list2):
+    """Calculate difference of means between two lists."""
+    counter1 = Counter(list1)
+    counter2 = Counter(list2)
+
+    for element in set(list1) & set(list2):
+        common_count = min(counter1[element], counter2[element])
+        counter1[element] -= common_count
+        counter2[element] -= common_count
+
+    unique_list1 = [item for item in counter1.elements()]
+    unique_list2 = [item for item in counter2.elements()]
+
+    mean_list1 = sum(unique_list1) / len(unique_list1) if unique_list1 else 0
+    mean_list2 = sum(unique_list2) / len(unique_list2) if unique_list2 else 0
+
+    if mean_list1 - mean_list2 > 0:
+        if min(unique_list1) > min(unique_list2):
+            return mean_list1 - mean_list2
+        return 0.0
+    return mean_list1 - mean_list2
+
+
+def find_possible_merge(A, B, consecutive_bonus, window_size, debug=False):
+    """Find possible merges between blocks."""
+    merge_bonus = 0.0
+    merge_windows = 1
+
+    def sortFn(value):
+        return value[2]
+
+    while True:
+        A_changed = False
+        B_changed = False
+
+        matching, current_cost, cost_matrix = find_maximum_matching(
+            A, B, merge_bonus, merge_windows
+        )
+
+        if len(A) >= 2:
+            merge_list = []
+            for i in range(len(A) - 1):
+                new_A = deepcopy(A)
+                new_A[i] = merge_blocks_wo_check(new_A[i], new_A[i + 1])
+                new_A.pop(i + 1)
+                updated_matching, updated_cost, _ = find_maximum_matching(
+                    new_A, B, merge_bonus, merge_windows
+                )
+                diff = difference_of_means(current_cost, updated_cost)
+                if diff > 0.05:
+                    merge_list.append([i, i + 1, diff])
+
+            merge_list.sort(key=sortFn, reverse=True)
+            if merge_list:
+                A_changed = True
+                A = merge_blocks_by_list(A, merge_list)
+                matching, current_cost, cost_matrix = find_maximum_matching(
+                    A, B, merge_bonus, merge_windows
+                )
+
+        if len(B) >= 2:
+            merge_list = []
+            for i in range(len(B) - 1):
+                new_B = deepcopy(B)
+                new_B[i] = merge_blocks_wo_check(new_B[i], new_B[i + 1])
+                new_B.pop(i + 1)
+                updated_matching, updated_cost, _ = find_maximum_matching(
+                    A, new_B, merge_bonus, merge_windows
+                )
+                diff = difference_of_means(current_cost, updated_cost)
+                if diff > 0.05:
+                    merge_list.append([i, i + 1, diff])
+
+            merge_list.sort(key=sortFn, reverse=True)
+            if merge_list:
+                B_changed = True
+                B = merge_blocks_by_list(B, merge_list)
+                matching, current_cost, cost_matrix = find_maximum_matching(
+                    A, B, merge_bonus, merge_windows
+                )
+
+        if not A_changed and not B_changed:
+            break
+
+    matching, _, _ = find_maximum_matching(A, B, consecutive_bonus, window_size)
+    return A, B, matching
+
+
+def merge_blocks_by_bbox(blocks):
+    """Merge blocks with same bounding box."""
+    merged_blocks = {}
+    for block in blocks:
+        bbox = tuple(block['bbox'])
+        if bbox in merged_blocks:
+            existing_block = merged_blocks[bbox]
+            existing_block['text'] += ' ' + block['text']
+            existing_block['color'] = [
+                (ec + c) / 2 for ec, c in zip(existing_block['color'], block['color'])
+            ]
+        else:
+            merged_blocks[bbox] = block
+    return list(merged_blocks.values())
+
+
+def mask_bounding_boxes_with_inpainting(image, bounding_boxes):
+    """Mask bounding boxes in image using inpainting."""
+    image_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    mask = np.zeros(image_cv.shape[:2], dtype=np.uint8)
+    height, width = image_cv.shape[:2]
+
+    for bbox in bounding_boxes:
+        x_ratio, y_ratio, w_ratio, h_ratio = bbox
+        x = int(x_ratio * width)
+        y = int(y_ratio * height)
+        w = int(w_ratio * width)
+        h = int(h_ratio * height)
+        mask[y : y + h, x : x + w] = 255
+
+    inpainted_image = cv2.inpaint(image_cv, mask, 3, cv2.INPAINT_TELEA)
+    return Image.fromarray(cv2.cvtColor(inpainted_image, cv2.COLOR_BGR2RGB))
+
+
+def rescale_and_mask(image, blocks):
+    """Rescale image and mask blocks."""
+    if blocks:
+        image = mask_bounding_boxes_with_inpainting(image, blocks)
+
+    width, height = image.size
+    if width < height:
+        new_size = (width, width)
+    else:
+        new_size = (height, height)
+
+    return image.resize(new_size, Image.LANCZOS)
+
+
+def calculate_clip_similarity(image1, image2, blocks1, blocks2):
+    """Calculate CLIP similarity between two images."""
+    model = CLIPModel.from_pretrained('openai/clip-vit-base-patch32')
+    processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    model = model.to(device)
+
+    # Mask and preprocess images
+    image1_masked = rescale_and_mask(image1, [block['bbox'] for block in blocks1])
+    image2_masked = rescale_and_mask(image2, [block['bbox'] for block in blocks2])
+    inputs = processor(
+        images=[image1_masked, image2_masked], return_tensors='pt', padding=True
+    )
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+
+    # Calculate features and similarity
+    with torch.no_grad():
+        image_features = model.get_image_features(**inputs)
+        image_features1 = image_features[0].unsqueeze(0)
+        image_features2 = image_features[1].unsqueeze(0)
+        image_features1 /= image_features1.norm(dim=-1, keepdim=True)
+        image_features2 /= image_features2.norm(dim=-1, keepdim=True)
+        similarity = (image_features1 @ image_features2.T).item()
+
+    return similarity
+
+
+def rgb_to_hex(rgb):
+    """Convert an RGB tuple to hexadecimal format."""
+    return '{:02X}{:02X}{:02X}'.format(*rgb)
+
+
+class ColorPool:
+    def __init__(self, offset=0):
+        color_values = list(range(10, 251, 16))
+        color_list = [((r + offset) % 256, (g + offset) % 256, (b + offset) % 256) 
+                     for r in color_values for g in color_values for b in color_values]
+        self.color_pool = [rgb_to_hex(color) for color in color_list]
+
+    def pop_color(self):
+        if self.color_pool:
+            return self.color_pool.pop()
+        else:
+            raise NotImplementedError
+
+
+def process_html_str(html_str, offset=0):
+    """Process HTML string to assign unique colors to text elements."""
+    soup = BeautifulSoup(html_str, 'html.parser')
+
+    def update_style(element, property_name, value):
+        important_value = f"{value} !important"
+        styles = element.attrs.get('style', '').split(';')
+        updated_styles = [s for s in styles if not s.strip().startswith(property_name) and len(s.strip()) > 0]
+        updated_styles.append(f"{property_name}: {important_value}")
+        element['style'] = '; '.join(updated_styles).strip()
+
+    # Set background color of all elements to transparent white
+    for element in soup.find_all(True):
+        update_style(element, 'background-color', 'rgba(255, 255, 255, 0.0)')
+
+    color_pool = ColorPool(offset)
+    text_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'div', 'span', 'a', 'b', 'li', 
+                 'table', 'td', 'th', 'button', 'footer', 'header', 'figcaption']
+
+    for tag in soup.find_all(text_tags):
+        color = f"#{color_pool.pop_color()}"
+        update_style(tag, 'color', color)
+        update_style(tag, 'opacity', '1.0')
+
+    return str(soup)
+
+
+def similar(n1, n2):
+    """Check if two numbers are similar within a threshold."""
+    return abs(n1 - n2) <= 8
+
+
+def find_different_pixels(image1, image2):
+    """Find pixels that differ between two images."""
+    if image1.size != image2.size:
+        logger.warning("Images are not the same size")
+        return None
+
+    image1 = image1.convert('RGB')
+    image2 = image2.convert('RGB')
+    pixels1 = image1.load()
+    pixels2 = image2.load()
+    different_pixels = []
+
+    for x in range(image1.size[0]):
+        for y in range(image1.size[1]):
+            r1, g1, b1 = pixels1[x, y]
+            r2, g2, b2 = pixels2[x, y]
+            if similar((r1 + 50) % 256, r2) and similar((g1 + 50) % 256, g2) and similar((b1 + 50) % 256, b2):
+                different_pixels.append((y, x))
+
+    return np.stack(different_pixels) if different_pixels else None
+
+
+def extract_text_with_color(html_str):
+    """Extract text and color information from HTML string."""
+    def get_color(tag):
+        if 'style' in tag.attrs:
+            styles = tag['style'].split(';')
+            color_style = [s for s in styles if 'color' in s and 'background-color' not in s]
+            if color_style:
+                color = color_style[-1].split(':')[1].strip().replace(" !important", "")
+                if color[0] == "#":
+                    return color
+                else:
+                    try:
+                        if color.startswith('rgb'):
+                            color = tuple(map(int, color[4:-1].split(',')))
+                        else:
+                            color = ImageColor.getrgb(color)
+                        return '#{:02x}{:02x}{:02x}'.format(*color)
+                    except ValueError:
+                        logger.warning(f"Unable to identify or convert color: {color}")
+                        return None
+        return None
+
+    def extract_text_recursive(element, parent_color='#000000'):
+        if isinstance(element, Comment):
+            return None
+        elif isinstance(element, NavigableString):
+            text = element.strip()
+            return (text, parent_color) if text else None
+        elif isinstance(element, Tag):
+            current_color = get_color(element) or parent_color
+            children_texts = filter(None, [extract_text_recursive(child, current_color) 
+                                        for child in element.children])
+            return list(children_texts)
+
+    soup = BeautifulSoup(html_str, 'html.parser')
+    body = soup.body
+    return extract_text_recursive(body) if body else []
+
+
+def flatten_tree(tree):
+    """Flatten a nested tree structure into a list."""
+    flat_list = []
+    def flatten(node):
+        if isinstance(node, list):
+            for item in node:
+                flatten(item)
+        else:
+            flat_list.append(node)
+    flatten(tree)
+    return flat_list
+
+
+def get_blocks_from_image_diff_pixels(image, html_text_color_tree, different_pixels):
+    """Extract text blocks from image using color differences."""
+    image_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    x_w = image_cv.shape[0]
+    y_w = image_cv.shape[1]
+
+    def hex_to_bgr(hex_color):
+        hex_color = hex_color.lstrip('#')
+        rgb = tuple(int(hex_color[i:i+2], 16) for i in (0, 2, 4))
+        return rgb[::-1]
+
+    def get_intersect(arr1, arr2):
+        arr1_reshaped = arr1.view([('', arr1.dtype)] * arr1.shape[1])
+        arr2_reshaped = arr2.view([('', arr2.dtype)] * arr2.shape[1])
+        common_rows = np.intersect1d(arr1_reshaped, arr2_reshaped)
+        return common_rows.view(arr1.dtype).reshape(-1, arr1.shape[1])
+
+    blocks = []
+    for item in html_text_color_tree:
+        try:
+            color = np.array(hex_to_bgr(item[1]), dtype="uint8")
+        except:
+            continue
+
+        lower = color - 4
+        upper = color + 4
+        mask = cv2.inRange(image_cv, lower, upper)
+        coords = np.column_stack(np.where(mask > 0))
+        coords = get_intersect(coords, different_pixels)
+
+        if coords.size == 0:
+            continue
+
+        x_min, y_min = np.min(coords, axis=0)
+        x_max, y_max = np.max(coords, axis=0)
+        
+        # Get average color from original image
+        color_coords = coords.copy()
+        color_coords = color_coords[color_coords[:, 0] <= x_max]
+        color_coords = color_coords[color_coords[:, 1] <= y_max]
+        colors = [image_cv[x, y] for x, y in color_coords]
+        avg_color = tuple(map(int, np.mean(colors, axis=0)))[::-1]  # Convert BGR to RGB
+
+        blocks.append({
+            'text': item[0].lower(),
+            'bbox': (y_min / y_w, x_min / x_w, (y_max - y_min + 1) / y_w, (x_max - x_min + 1) / x_w),
+            'color': avg_color
+        })
+
+    return blocks
+
+
+def get_blocks_from_html(html_str, image1):
+    """Extract text blocks from HTML and image."""
+    # Process HTML with two different color offsets
+    html_str_1 = process_html_str(html_str, offset=0)
+    html_str_2 = process_html_str(html_str, offset=50)
+
+    # Render both HTML versions to images
+    # TODO: Screenshot html_str_2
+    filter_color = (255, 0, 0)  
+    image2 = Image.new("RGB", image1.size, filter_color)
+
+
+    # Find pixels that differ between the two rendered images
+    different_pixels = find_different_pixels(image1, image2)
+    if different_pixels is None:
+        logger.warning("Unable to get pixels with different colors")
+        return []
+
+    # Extract text and color information from HTML
+    html_text_color_tree = flatten_tree(extract_text_with_color(html_str_1))
+    try:
+        blocks = get_blocks_from_image_diff_pixels(image1, html_text_color_tree, different_pixels)
+    except Exception as e:
+        logger.warning(f"Unable to get blocks: {e}")
+        return []
+
+    return blocks
+
+
+def evaluate(task, generated_img):
+    """Evaluate generated image against reference image using multiple metrics."""
+    # Load reference image
+    post_image = task['post_image']
+
+    # Extract blocks from HTML and images
+    post_blocks = get_blocks_from_html(task['post_html'], post_image)
+    gen_blocks = get_blocks_from_html(task['gen_html'], generated_img)
+
+    print("block details", post_blocks, gen_blocks)
+    if not post_blocks or not gen_blocks:
+        # Fallback to basic CLIP and pixel comparison if no blocks available
+        clip_score = calculate_clip_similarity(post_image, generated_img, [], [])
+        logger.info(f'CLIP similarity score: {clip_score}')
+
+        # Pixel comparison
+        diff = ImageChops.difference(generated_img, post_image)
+        pixel_match = not diff.getbbox()
+        logger.info(
+            f"Pixel difference analysis: {'No difference' if pixel_match else 'Differences found'}"
+        )
+
+        return clip_score > 0.95 or pixel_match
+
+    # Merge blocks with same bounding boxes
+    post_blocks = merge_blocks_by_bbox(post_blocks)
+    gen_blocks = merge_blocks_by_bbox(gen_blocks)
+
+    # Find optimal block matching
+    consecutive_bonus, window_size = 0.1, 1
+    gen_blocks_m, post_blocks_m, matching = find_possible_merge(
+        gen_blocks, deepcopy(post_blocks), consecutive_bonus, window_size
+    )
+
+    # Filter matches with low similarity
+    filtered_matching = []
+    for i, j in matching:
+        text_similarity = calculate_similarity(gen_blocks_m[i], post_blocks_m[j])
+        if text_similarity >= 0.5:
+            filtered_matching.append([i, j, text_similarity])
+    matching = filtered_matching
+
+    if not matching:
+        logger.warning('No matching blocks found')
+        clip_score = calculate_clip_similarity(
+            post_image, generated_img, gen_blocks, post_blocks
+        )
+        return clip_score > 0.95
+
+    # Calculate metrics for matched blocks
+    indices1 = [item[0] for item in matching]
+    indices2 = [item[1] for item in matching]
+
+    # Calculate unmatched areas
+    unmatched_area_1 = sum(
+        block['bbox'][2] * block['bbox'][3]
+        for i, block in enumerate(gen_blocks_m)
+        if i not in indices1
+    )
+    unmatched_area_2 = sum(
+        block['bbox'][2] * block['bbox'][3]
+        for j, block in enumerate(post_blocks_m)
+        if j not in indices2
+    )
+    total_unmatched_area = unmatched_area_1 + unmatched_area_2
+
+    # Calculate metrics for matched blocks
+    matched_areas = []
+    text_scores = []
+    position_scores = []
+    color_scores = []
+
+    for i, j, text_similarity in matching:
+        # Area
+        block_area = (
+            gen_blocks_m[i]['bbox'][2] * gen_blocks_m[i]['bbox'][3]
+            + post_blocks_m[j]['bbox'][2] * post_blocks_m[j]['bbox'][3]
+        )
+        matched_areas.append(block_area)
+
+        # Position similarity
+        position_similarity = 1 - calculate_distance_max_1d(
+            gen_blocks_m[i]['bbox'][0] + gen_blocks_m[i]['bbox'][2] / 2,
+            gen_blocks_m[i]['bbox'][1] + gen_blocks_m[i]['bbox'][3] / 2,
+            post_blocks_m[j]['bbox'][0] + post_blocks_m[j]['bbox'][2] / 2,
+            post_blocks_m[j]['bbox'][1] + post_blocks_m[j]['bbox'][3] / 2,
+        )
+
+        # Color similarity
+        color_similarity = color_similarity_ciede2000(
+            gen_blocks_m[i]['color'], post_blocks_m[j]['color']
+        )
+
+        text_scores.append(text_similarity)
+        position_scores.append(position_similarity)
+        color_scores.append(color_similarity)
+
+    # Calculate final scores
+    total_area = sum(matched_areas) + total_unmatched_area
+    size_score = sum(matched_areas) / total_area if total_area > 0 else 0
+    text_score = np.mean(text_scores) if text_scores else 0
+    position_score = np.mean(position_scores) if position_scores else 0
+    color_score = np.mean(color_scores) if color_scores else 0
+    clip_score = calculate_clip_similarity(
+        post_image, generated_img, gen_blocks, post_blocks
+    )
+
+    # Combine scores with equal weights
+    final_score = 0.2 * (
+        size_score + text_score + position_score + color_score + clip_score
+    )
+
+    logger.info('Evaluation scores:')
+    logger.info(f'- Size score: {size_score:.3f}')
+    logger.info(f'- Text score: {text_score:.3f}')
+    logger.info(f'- Position score: {position_score:.3f}')
+    logger.info(f'- Color score: {color_score:.3f}')
+    logger.info(f'- CLIP score: {clip_score:.3f}')
+    logger.info(f'- Final score: {final_score:.3f}')
+
+    return final_score > 0.8  # Consider it a match if final score > 80%
+
+
+def png_to_bytes(png):
+    buffer = BytesIO()
+    png.save(buffer, format='PNG')
+    image_bytes = buffer.getvalue()
+    return image_bytes
+
+
+def bytes_to_image(image_bytes):
+    """Convert bytes to a Pillow Image object."""
+    return Image.open(BytesIO(image_bytes))
+
+
+if __name__ == '__main__':
+    first_image = Image.open('./evaluation/visualcodebench/data/1/post.png')
+    image = Image.open('./evaluation/visualcodebench/data/1/prev.png')
+    
+    
+    html_file = open('./evaluation/visualcodebench/data/1/post/index.html', 'r')
+    first_html = html_file.read()
+    html_file.close()
+
+    html_file = open('./evaluation/visualcodebench/data/1/prev/index.html', 'r')
+    gen_html = html_file.read()
+    html_file.close()
+
+
+
+    sample = {'post_image': first_image, "post_html": first_html, "gen_html": gen_html}
+
+
+
+    evaluate(sample, image)
+

evaluation/benchmarks/visualcodebench/prepare.py

@@ -0,0 +1,97 @@
+import base64
+import os
+from io import BytesIO
+
+import pandas as pd
+from huggingface_hub import snapshot_download
+from PIL import PngImagePlugin
+from tqdm import tqdm
+
+from openhands.core.logger import openhands_logger as logger
+
+REPO_DOWNLOAD_DIR = (
+    './evaluation/visualcodebench/'  # Directory to store the downloaded repository
+)
+
+
+def download_repository():
+    """
+    Download the entire repository from Hugging Face Hub.
+    This function clones the repository into REPO_DOWNLOAD_DIR.
+    """
+    repo_id = 'rvmalhot/VisualCodeBench'
+    try:
+        logger.info(f"Downloading repository '{repo_id}'...")
+        snapshot_download(
+            repo_id=repo_id,
+            local_dir=REPO_DOWNLOAD_DIR,
+            repo_type='dataset',
+            ignore_patterns=None,  # Download all files
+        )
+        logger.info(f"Repository downloaded to '{REPO_DOWNLOAD_DIR}'.")
+    except Exception as e:
+        logger.error(f"Error downloading repository '{repo_id}': {e}")
+        raise e
+
+
+def format_task_dict(example):
+    instance_id = example['id']
+    prev_remote_path = os.path.join(REPO_DOWNLOAD_DIR, f'data/{instance_id}/prev')
+    post_remote_path = os.path.join(REPO_DOWNLOAD_DIR, f'data/{instance_id}/post')
+
+    # Check if 'prev' and 'post' directories exist
+    prev_exists = os.path.exists(prev_remote_path)
+    post_exists = os.path.exists(post_remote_path)
+
+    if prev_exists and post_exists:
+        skip = False
+    else:
+        skip = True
+
+    task = {
+        'instance_id': instance_id,
+        'prev_image': example['prev_image'],
+        'post_image': example['post_image'],
+        'changes': example['changes'],
+        'prev_code_files': example['prev_code_files'],
+        'post_code_files': example['post_code_files'],
+        'skip': skip,
+    }
+
+    return task
+
+
+def prepare_visualcodebench(dataset):
+    logger.info('Processing dataset')
+    dataset_processed = []
+    for example in tqdm(dataset['train']):
+        formatted_example = format_task_dict(example)
+        if formatted_example['skip']:
+            continue
+        del formatted_example['skip']
+        dataset_processed.append(formatted_example)
+
+    return pd.DataFrame(dataset_processed)
+
+
+def pil_image_to_base64(image: PngImagePlugin.PngImageFile) -> str:
+    """
+    Converts a PIL image to a Base64-encoded string.
+
+    Parameters:
+    - image (PngImagePlugin.PngImageFile): The PIL image to convert.
+
+    Returns:
+    - str: The Base64-encoded string of the image.
+    """
+    if not isinstance(image, PngImagePlugin.PngImageFile):
+        raise ValueError(
+            'The provided image is not a PIL.PngImagePlugin.PngImageFile instance.'
+        )
+
+    buffered = BytesIO()
+    image.save(buffered, format='PNG')
+    img_bytes = buffered.getvalue()
+    img_base64 = base64.b64encode(img_bytes).decode('utf-8')
+    base64_with_prefix = f'data:image/png;base64,{img_base64}'
+    return [base64_with_prefix]

evaluation/benchmarks/visualcodebench/run_infer.py

@@ -0,0 +1,247 @@
+# FILE: run_infer.py
+
+import asyncio
+import os
+import shutil
+import tempfile
+from functools import partial
+
+import pandas as pd
+from datasets import load_dataset
+
+# from evaluation.benchmarks.visualcodebench.eval import capture_screenshot
+from evaluation.benchmarks.visualcodebench.prepare import (
+    REPO_DOWNLOAD_DIR,
+    download_repository,
+    pil_image_to_base64,
+    prepare_visualcodebench,
+)
+from evaluation.utils.shared import (
+    EvalMetadata,
+    assert_and_raise,
+    codeact_user_response,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+)
+from openhands.core.config.utils import parse_arguments
+from openhands.core.logger import openhands_logger as logger  # Import OpenHands logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action.commands import CmdRunAction
+from openhands.events.action.message import MessageAction
+from openhands.events.observation.commands import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+# Define workspace and output directories
+WORKSPACE_DIR = './workspace'
+
+FAKE_RESPONSES = {
+    'CodeActAgent': partial(codeact_user_response, encapsulate_solution=True),
+}
+
+
+def get_config(
+    metadata: EvalMetadata,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='eventstream',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=True,
+            use_host_network=False,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Initialization Fn')
+    logger.info('-' * 30)
+    workspace_dir_name = instance['instance_id']
+    obs: CmdOutputObservation
+
+    action = CmdRunAction(command='mkdir -p /workspace/{workspace_dir_name}')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to create /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    file_path = REPO_DOWNLOAD_DIR + f'data/{workspace_dir_name}/prev/index.html'
+    runtime.copy_to(file_path, f'/workspace/{workspace_dir_name}')
+    logger.info(f'Copied code file for instance {workspace_dir_name}')
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Initialization Fn')
+    logger.info('-' * 30)
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> str:
+    # TODO: extract edited HTML file from agent workspace
+    # temp_zip = runtime.copy_from(f'/workspace/{instance.instance_id}')
+    # file_name = f'/workspace/{instance.instance_id}/index.html'
+    # with zipfile.ZipFile(temp_zip, 'r') as zip_ref:
+    #     if file_name in zip_ref.namelist():
+    #         with zip_ref.open(file_name) as file:
+    #             file_content = file.read().decode('utf-8')  # Decode bytes to string
+    #     else:
+    #         raise FileNotFoundError(f"'{file_name}' not found in the ZIP archive.")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        src_folder = REPO_DOWNLOAD_DIR + f'data/{instance.instance_id}/post/'
+        shutil.copytree(src_folder, tmpdir, dirs_exist_ok=True)
+
+        # image = capture_screenshot(tmpdir)
+        # if image is not None:
+        #     shutil.copy(os.path.join(tmpdir, 'final_screenshot.png'), REPO_DOWNLOAD_DIR)
+
+
+def process_instance(
+    instance: pd.Series, metadata: EvalMetadata, reset_logger: bool = True
+):
+    config = get_config(metadata)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    # =============================================
+    # build instruction
+    # =============================================
+
+    # Prepare instruction
+    instruction = (
+        f"Modify the HTML/CSS according to the following instruction:\n\n"
+        f"{instance['changes']}\n\n"
+    )
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided '
+        'to you AND NEVER ASK FOR HUMAN HELP.\n'
+    )
+
+    # =============================================
+    # create sandbox and run the agent
+    # =============================================
+
+    runtime: Runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    try:
+        initialize_runtime(runtime, instance=instance)
+
+        image_urls = pil_image_to_base64(instance['prev_image'])
+
+        action = MessageAction(content=instruction, image_urls=image_urls)
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=action,
+                runtime=runtime,
+                fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+            )
+        )
+        if state is None:
+            raise ValueError('State should not be None.')
+
+        # =============================================
+        # result evaluation
+        # =============================================
+
+        return_val = complete_runtime(runtime, instance)
+        logger.info(f'Return value {return_val}')
+    finally:
+        runtime.close()
+
+    # TODO: return EVAL output
+
+
+def main():
+    """Main function to run the evaluation."""
+    # args = parse_args()
+    args = parse_arguments()
+
+    logger.info(f"\n{'='*80}\nStarting VisualCodeBench Evaluation\n{'='*80}")
+    logger.info(f'Agent: {args.agent_cls}')
+    logger.info(f'Model: {args.llm_config}')
+    logger.info(f'Max iterations: {args.max_iterations}')
+    logger.info(f'Eval limit: {args.eval_n_limit}')
+    logger.info(f'Num workers: {args.eval_num_workers}\n')
+    logger.info(f'Eval output: {args.eval_output_dir}\n')
+
+    # Step 1: Download the entire repository once
+    logger.info('Downloading repository...')
+    download_repository()
+
+    # Step 2: Load Dataset
+    logger.info('Loading dataset...')
+    dataset = load_dataset(REPO_DOWNLOAD_DIR)
+
+    # Step 3: Prepare dataset
+    llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        logger.error(f'Could not find LLM config: {args.llm_config}')
+        raise ValueError(f'Could not find LLM config: {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        'VisualCodeBench',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        'evaluation/output/',
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    dataset = prepare_visualcodebench(dataset)
+    instances = prepare_dataset(dataset, output_file, eval_n_limit=args.eval_n_limit)
+
+    # Step 4: Run eval
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
+    )
+
+
+if __name__ == '__main__':
+    main()

evaluation/benchmarks/visualcodebench/scripts/run_infer.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+# Check if required arguments are provided
+if [ "$#" -lt 4 ]; then
+    echo "Usage: $0 [model_config] [commit_hash] [agent_cls] [eval_limit] [num_workers]"
+    echo "Example: $0 llm.eval_gpt_4o_mini HEAD CodeActAgent 5 1"
+    exit 1
+fi
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT_CLS=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=${5:-1}  # Default to 1 worker if not specified
+
+# Checkout the specified commit
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+COMMAND="export PYTHONPATH=evaluation/benchmarks/visualcodebench:\$PYTHONPATH && poetry run python evaluation/benchmarks/visualcodebench/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 5 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $OPENHANDS_VERSION" \
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/benchmarks/visualcodebench/server.py

@@ -0,0 +1,167 @@
+import http
+import os
+import socket
+import socketserver
+import threading
+import time
+from io import BytesIO
+
+import requests
+from PIL import Image, ImageChops
+from playwright.sync_api import sync_playwright
+
+from openhands.core.logger import openhands_logger as logger
+
+
+def get_free_port():
+    """Find a free port to run the HTTP server."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(('', 0))
+        return s.getsockname()[1]
+
+
+def start_http_server(tmpdir):
+    port = get_free_port()
+
+    class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+        def translate_path(self, path):
+            # Serve files from the specified directory instead of the current working directory
+            path = super().translate_path(path)
+            relative_path = os.path.relpath(path, os.getcwd())
+            return os.path.join(tmpdir, relative_path)
+
+    handler = CustomHTTPRequestHandler
+    server = socketserver.TCPServer(('', port), handler)
+    return server, port
+
+
+def capture_screenshot(tmpdir):
+    server, port = start_http_server(tmpdir)
+    server_thread = threading.Thread(target=server.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+    time.sleep(10)
+
+    image = None
+    try:
+        server_url = f'http://localhost:{port}/'
+
+        if not is_server_reachable(server_url):
+            raise RuntimeError(f'Server not reachable at {server_url}')
+
+        screenshot_path = os.path.join(tmpdir, 'final_screenshot.png')
+        capture_screenshot_playwright(server_url, screenshot_path)
+        image = Image.open(screenshot_path)
+        image.load()
+    finally:
+        # Shut down the server and clean up
+        server.shutdown()
+        server.server_close()
+
+    return image
+
+
+def is_server_reachable(url):
+    """
+    Check if the local server is reachable.
+    """
+    try:
+        response = requests.get(url, timeout=5)  # Set a 5-second timeout
+        if response.status_code == 200:
+            logger.info(f'Server is reachable at {url}')
+            return True
+        else:
+            logger.warning(
+                f'Server responded with status code {response.status_code} at {url}'
+            )
+            return False
+    except requests.ConnectionError as e:
+        logger.error(f'Failed to connect to server at {url}: {e}')
+        return False
+
+
+def capture_screenshot_playwright(url, screenshot_path):
+    """Capture a screenshot of the given URL using Playwright."""
+    try:
+        with sync_playwright() as p:
+            logger.info('Launching browser...')
+            browser = p.chromium.launch(timeout=10000)  # 10 seconds for browser launch
+
+            logger.info('Creating a new page...')
+            page = browser.new_page()
+
+            logger.info(f'Navigating to URL: {url}')
+            try:
+                page.goto(url, timeout=60 * 1000)  # Set timeout to 5 seconds
+                logger.info('Page navigation completed.')
+            except Exception as e:
+                logger.warning(f'Page navigation timed out. {e}. Continuing...')
+
+            logger.info('Waiting for network to be idle...')
+            try:
+                page.wait_for_load_state(
+                    'networkidle', timeout=60 * 1000
+                )  # Set timeout to 5 seconds
+                logger.info('Page load state reached.')
+            except Exception as e:
+                logger.warning(f'Page load state timed out. {e}. Continuing...')
+
+            logger.info('Capturing screenshot...')
+            page.screenshot(
+                path=screenshot_path, full_page=True
+            )  # Capture full page screenshot
+
+            logger.info(f'Screenshot saved to {screenshot_path}')
+            browser.close()
+            return True
+    except Exception as e:
+        logger.error(f'Error capturing screenshot with Playwright: {e}')
+        return False
+
+
+def evaluate(task, screenshot_path):
+    """Compare generated screenshot with post_image using CLIP score."""
+    try:
+        import torch
+        from transformers import CLIPModel, CLIPProcessor
+
+        # Load CLIP model and processor
+        model = CLIPModel.from_pretrained('openai/clip-vit-base-patch32')
+        processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')
+
+        # Load images
+        post_image = Image.open(BytesIO(task['post_image']))
+        generated_img = Image.open(screenshot_path)
+
+        # Process images
+        inputs = processor(
+            images=[post_image, generated_img], return_tensors='pt', padding=True
+        )
+
+        # Get image features
+        image_features = model.get_image_features(**inputs)
+
+        # Calculate cosine similarity
+        similarity = torch.nn.functional.cosine_similarity(
+            image_features[0].unsqueeze(0), image_features[1].unsqueeze(0)
+        ).item()
+
+        logger.info(f'CLIP similarity score: {similarity}')
+
+        return similarity > 0.95  # Consider it a match if similarity > 95%
+    except Exception as e:
+        logger.error(f'Error in CLIP evaluation: {e}')
+        # Fallback to pixel comparison if CLIP fails
+        try:
+            post_image = Image.open(BytesIO(task['post_image']))
+            generated_img = Image.open(screenshot_path)
+
+            # Compare images directly without converting to bytes
+            diff = ImageChops.difference(generated_img, post_image)
+            logger.info(
+                f"Pixel difference analysis: {'No difference' if not diff.getbbox() else 'Differences found'}"
+            )
+            return not diff.getbbox()
+        except Exception as ex:
+            logger.error(f'Error in fallback evaluation: {ex}')
+            return False

evaluation/benchmarks/visualwebarena/README.md

@@ -0,0 +1,50 @@
+# VisualWebArena Evaluation with OpenHands Browsing Agents
+
+This folder contains evaluation for [VisualWebArena](https://github.com/web-arena-x/visualwebarena) benchmark, powered by [BrowserGym](https://github.com/ServiceNow/BrowserGym) for easy evaluation of how well an agent capable of browsing can perform on realistic web browsing tasks.
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+
+## Setup VisualWebArena Environment
+
+VisualWebArena requires you to set up websites containing pre-populated content that is accessible via URL to the machine running the OpenHands agents.
+Follow [this document](https://github.com/web-arena-x/visualwebarena/blob/main/environment_docker/README.md) to set up your own VisualWebArena environment through local servers or AWS EC2 instances.
+Take note of the base URL (`$VISUALWEBARENA_BASE_URL`) of the machine where the environment is installed.
+
+## Test if your environment works
+
+Access with browser the above VisualWebArena website URLs and see if they load correctly.
+If you cannot access the website, make sure the firewall allows public access of the aforementioned ports on your server
+Check the network security policy if you are using an AWS machine.
+Follow the VisualWebArena environment setup guide carefully, and make sure the URL fields are populated with the correct base URL of your server.
+
+## Run Evaluation
+
+```bash
+export VISUALWEBARENA_BASE_URL=<YOUR_SERVER_URL_HERE>
+export OPENAI_API_KEY="yourkey" # this OpenAI API key is required for some visualWebArena validators that utilize LLMs
+export OPENAI_BASE_URL="https://api.openai.com/v1/" # base URL for OpenAI model used for VisualWebArena evaluation
+bash evaluation/benchmarks/visualwebarena/scripts/run_infer.sh llm.claude HEAD VisualBrowsingAgent
+```
+
+Results will be in `evaluation/evaluation_outputs/outputs/visualwebarena/`
+
+To calculate the success rate, run:
+
+```sh
+poetry run python evaluation/benchmarks/visualwebarena/get_success_rate.py evaluation/evaluation_outputs/outputs/visualwebarena/SOME_AGENT/EXP_NAME/output.jsonl
+```
+
+## Submit your evaluation results
+
+You can start your own fork of [our huggingface evaluation outputs](https://huggingface.co/spaces/OpenHands/evaluation) and submit a PR of your evaluation results following the guide [here](https://huggingface.co/docs/hub/en/repositories-pull-requests-discussions#pull-requests-and-discussions).
+
+## VisualBrowsingAgent V1.0 result
+
+Tested on VisualBrowsingAgent V1.0
+
+VisualWebArena, 910 tasks (high cost, single run due to fixed task), max step 15. Resolve rates are:
+
+- GPT4o: 26.15%
+- Claude-3.5 Sonnet: 25.27%

evaluation/benchmarks/visualwebarena/get_success_rate.py

@@ -0,0 +1,40 @@
+import argparse
+import json
+
+import browsergym.visualwebarena  # noqa F401 register visualwebarena tasks as gym environments
+import gymnasium as gym
+
+parser = argparse.ArgumentParser(description='Calculate average reward.')
+parser.add_argument('output_path', type=str, help='path to output.jsonl')
+
+args = parser.parse_args()
+
+if __name__ == '__main__':
+    env_ids = [
+        id
+        for id in gym.envs.registry.keys()
+        if id.startswith('browsergym/visualwebarena')
+    ]
+    total_num = len(env_ids)
+    print('Total number of tasks: ', total_num)
+    total_reward = 0
+    total_cost = 0
+    actual_num = 0
+    with open(args.output_path, 'r') as f:
+        for line in f:
+            data = json.loads(line)
+            actual_num += 1
+            total_cost += data['metrics']['accumulated_cost']
+            reward = data['test_result']['reward']
+            if reward >= 0:
+                total_reward += data['test_result']['reward']
+            else:
+                actual_num -= 1
+    avg_reward = total_reward / total_num
+    print('Total reward: ', total_reward)
+    print('Success Rate: ', avg_reward)
+
+    avg_cost = total_cost / actual_num
+    print('Avg Cost: ', avg_cost)
+    print('Total Cost: ', total_cost)
+    print('Actual number of tasks finished: ', actual_num)

evaluation/benchmarks/visualwebarena/run_infer.py

@@ -0,0 +1,254 @@
+import asyncio
+import json
+import os
+from typing import Any
+
+import browsergym.visualwebarena  # noqa F401 register visualwebarena tasks as gym environments
+import gymnasium as gym
+import pandas as pd
+
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    parse_arguments,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import (
+    BrowseInteractiveAction,
+    CmdRunAction,
+    MessageAction,
+)
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.runtime.browser.browser_env import (
+    BROWSER_EVAL_GET_GOAL_ACTION,
+    BROWSER_EVAL_GET_REWARDS_ACTION,
+)
+from openhands.utils.async_utils import call_async_from_sync
+
+SUPPORTED_AGENT_CLS = {'VisualBrowsingAgent'}
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'VisualBrowsingAgent': 'Continue the task. IMPORTANT: do not talk to the user until you have finished the task',
+}
+
+
+def get_config(
+    metadata: EvalMetadata,
+    env_id: str,
+) -> AppConfig:
+    base_url = os.environ.get('VISUALWEBARENA_BASE_URL', None)
+    openai_api_key = os.environ.get('OPENAI_API_KEY', None)
+    openai_base_url = os.environ.get('OPENAI_BASE_URL', None)
+    assert base_url is not None, 'VISUALWEBARENA_BASE_URL must be set'
+    assert openai_api_key is not None, 'OPENAI_API_KEY must be set'
+    assert openai_base_url is not None, 'OPENAI_BASE_URL must be set'
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='docker',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=True,
+            use_host_network=False,
+            browsergym_eval_env=env_id,
+            runtime_startup_env_vars={
+                'BASE_URL': base_url,
+                'OPENAI_API_KEY': openai_api_key,
+                'OPENAI_BASE_URL': openai_base_url,
+                'VWA_CLASSIFIEDS': f'{base_url}:9980',
+                'VWA_CLASSIFIEDS_RESET_TOKEN': '4b61655535e7ed388f0d40a93600254c',
+                'VWA_SHOPPING': f'{base_url}:7770',
+                'VWA_SHOPPING_ADMIN': f'{base_url}:7780/admin',
+                'VWA_REDDIT': f'{base_url}:9999',
+                'VWA_GITLAB': f'{base_url}:8023',
+                'VWA_WIKIPEDIA': f'{base_url}:8888',
+                'VWA_HOMEPAGE': f'{base_url}:4399',
+            },
+            timeout=300,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+        attach_to_existing=True,
+    )
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config,
+            metadata.eval_output_dir,
+            env_id,
+        )
+    )
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+) -> tuple[str, list]:
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    # Set instance id
+    action = CmdRunAction(command='mkdir -p /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+    action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_GOAL_ACTION)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    goal = obs.content
+    goal_image_urls = []
+    if hasattr(obs, 'goal_image_urls'):
+        goal_image_urls = obs.goal_image_urls
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+    return goal, goal_image_urls
+
+
+def complete_runtime(
+    runtime: Runtime,
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_REWARDS_ACTION)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+    return {
+        'rewards': json.loads(obs.content),
+    }
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+):
+    env_id = instance.instance_id
+
+    config = get_config(metadata, env_id)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, env_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {env_id}.')
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+    task_str, goal_image_urls = initialize_runtime(runtime)
+    initial_user_action = MessageAction(content=task_str, image_urls=goal_image_urls)
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=initial_user_action,
+            runtime=runtime,
+        )
+    )
+    # ======= Attempt to evaluate the agent's environment impact =======
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    metrics = state.metrics.get() if state.metrics else None
+
+    # Instruction obtained from the first message from the USER
+    instruction = ''
+    for event in state.history:
+        if isinstance(event, MessageAction):
+            instruction = event.content
+            break
+
+    try:
+        return_val = complete_runtime(runtime)
+        logger.info(f'Return value from complete_runtime: {return_val}')
+        reward = max(return_val['rewards'])
+    except Exception:
+        reward = -1.0  # kept -1 to identify instances for which evaluation failed.
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=env_id,
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result={
+            'reward': reward,
+        },
+    )
+    runtime.close()
+    return output
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+
+    dataset = pd.DataFrame(
+        {
+            'instance_id': [
+                id
+                for id in gym.envs.registry.keys()
+                if id.startswith('browsergym/visualwebarena')
+            ]
+        }
+    )
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+    metadata = make_metadata(
+        llm_config,
+        'visualwebarena',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+    )

evaluation/benchmarks/visualwebarena/scripts/run_infer.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+# configure browsing agent
+export USE_NAV="true"
+export USE_CONCISE_ANSWER="true"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=$5
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default VisualBrowsingAgent"
+  AGENT="VisualBrowsingAgent"
+fi
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "AGENT_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+EVAL_NOTE="${OPENHANDS_VERSION}"
+
+COMMAND="poetry run python evaluation/benchmarks/visualwebarena/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 15 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $EVAL_NOTE"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/integration_tests/run_infer.py

@@ -35,6 +35,7 @@ from openhands.utils.async_utils import call_async_from_sync
 FAKE_RESPONSES = {
     'CodeActAgent': fake_user_response,
     'DelegatorAgent': fake_user_response,
+    'VisualBrowsingAgent': fake_user_response,
 }
 
 

frontend/__tests__/components/features/conversation-panel/conversation-panel.test.tsx

@@ -1,4 +1,4 @@
-import { render, screen, within } from "@testing-library/react";
+import { render, screen, waitFor, within } from "@testing-library/react";
 import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   QueryClientProvider,
@@ -7,10 +7,12 @@ import {
 } from "@tanstack/react-query";
 import userEvent from "@testing-library/user-event";
 import { createRoutesStub } from "react-router";
+import React from "react";
 import { ConversationPanel } from "#/components/features/conversation-panel/conversation-panel";
 import OpenHands from "#/api/open-hands";
 import { AuthProvider } from "#/context/auth-context";
 import { clickOnEditButton } from "./utils";
+import { queryClientConfig } from "#/query-client-config";
 
 describe("ConversationPanel", () => {
   const onCloseMock = vi.fn();
@@ -231,4 +233,47 @@ describe("ConversationPanel", () => {
 
     expect(onCloseMock).toHaveBeenCalledOnce();
   });
+
+  it("should refetch data on rerenders", async () => {
+    // We need to simulate the toggling of the component to test the refetching
+    function PanelWithToggle() {
+      const [isOpen, setIsOpen] = React.useState(true);
+      return (
+        <>
+          <button type="button" onClick={() => setIsOpen((prev) => !prev)}>
+            Toggle
+          </button>
+          {isOpen && <ConversationPanel onClose={onCloseMock} />}
+        </>
+      );
+    }
+
+    const MyRouterStub = createRoutesStub([
+      {
+        Component: PanelWithToggle,
+        path: "/",
+      },
+    ]);
+
+    const getUserConversationsSpy = vi.spyOn(OpenHands, "getUserConversations");
+    render(<MyRouterStub />, {
+      wrapper: ({ children }) => (
+        <AuthProvider>
+          <QueryClientProvider client={new QueryClient(queryClientConfig)}>
+            {children}
+          </QueryClientProvider>
+        </AuthProvider>
+      ),
+    });
+
+    await waitFor(() => expect(getUserConversationsSpy).toHaveBeenCalledOnce());
+
+    const button = screen.getByText("Toggle");
+    await userEvent.click(button);
+    await userEvent.click(button);
+
+    await waitFor(() =>
+      expect(getUserConversationsSpy).toHaveBeenCalledTimes(2),
+    );
+  });
 });

frontend/__tests__/components/features/sidebar/sidebar.test.tsx

@@ -8,6 +8,9 @@ import { MULTI_CONVERSATION_UI } from "#/utils/feature-flags";
 import OpenHands from "#/api/open-hands";
 import { MOCK_USER_PREFERENCES } from "#/mocks/handlers";
 
+// These tests will now fail because the conversation panel is rendered through a portal
+// and technically not a child of the Sidebar component.
+
 const renderSidebar = () => {
   const RouterStub = createRoutesStub([
     {
@@ -152,11 +155,13 @@ describe("Sidebar", () => {
       const settingsModal = screen.getByTestId("ai-config-modal");
 
       // Click the advanced options switch to show the API key input
-      const advancedOptionsSwitch = within(settingsModal).getByTestId("advanced-option-switch");
+      const advancedOptionsSwitch = within(settingsModal).getByTestId(
+        "advanced-option-switch",
+      );
       await user.click(advancedOptionsSwitch);
 
       const apiKeyInput = within(settingsModal).getByLabelText(/API\$KEY/i);
-      await user.type(apiKeyInput, "SET");
+      await user.type(apiKeyInput, "**********");
 
       const saveButton = within(settingsModal).getByTestId(
         "save-settings-button",

frontend/__tests__/components/feedback-actions.test.tsx

@@ -1,12 +1,13 @@
 import { render, screen, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { afterEach, describe, expect, it, vi } from "vitest";
-import { FeedbackActions } from "#/components/features/feedback/feedback-actions";
+import { TrajectoryActions } from "#/components/features/trajectory/trajectory-actions";
 
-describe("FeedbackActions", () => {
+describe("TrajectoryActions", () => {
   const user = userEvent.setup();
   const onPositiveFeedback = vi.fn();
   const onNegativeFeedback = vi.fn();
+  const onExportTrajectory = vi.fn();
 
   afterEach(() => {
     vi.clearAllMocks();
@@ -14,9 +15,10 @@ describe("FeedbackActions", () => {
 
   it("should render correctly", () => {
     render(
-      <FeedbackActions
+      <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
+        onExportTrajectory={onExportTrajectory}
       />,
     );
 
@@ -27,9 +29,10 @@ describe("FeedbackActions", () => {
 
   it("should call onPositiveFeedback when positive feedback is clicked", async () => {
     render(
-      <FeedbackActions
+      <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
+        onExportTrajectory={onExportTrajectory}
       />,
     );
 
@@ -41,9 +44,10 @@ describe("FeedbackActions", () => {
 
   it("should call onNegativeFeedback when negative feedback is clicked", async () => {
     render(
-      <FeedbackActions
+      <TrajectoryActions
         onPositiveFeedback={onPositiveFeedback}
         onNegativeFeedback={onNegativeFeedback}
+        onExportTrajectory={onExportTrajectory}
       />,
     );
 
@@ -52,4 +56,19 @@ describe("FeedbackActions", () => {
 
     expect(onNegativeFeedback).toHaveBeenCalled();
   });
+
+  it("should call onExportTrajectory when negative feedback is clicked", async () => {
+    render(
+      <TrajectoryActions
+        onPositiveFeedback={onPositiveFeedback}
+        onNegativeFeedback={onNegativeFeedback}
+        onExportTrajectory={onExportTrajectory}
+      />,
+    );
+
+    const exportButton = screen.getByTestId("export-trajectory");
+    await user.click(exportButton);
+
+    expect(onExportTrajectory).toHaveBeenCalled();
+  });
 });

frontend/__tests__/initial-query.test.tsx

@@ -1,20 +1,20 @@
 import { describe, it, expect } from "vitest";
 import store from "../src/store";
 import {
-  setInitialQuery,
-  clearInitialQuery,
+  setInitialPrompt,
+  clearInitialPrompt,
 } from "../src/state/initial-query-slice";
 
 describe("Initial Query Behavior", () => {
-  it("should clear initial query when clearInitialQuery is dispatched", () => {
+  it("should clear initial query when clearInitialPrompt is dispatched", () => {
     // Set up initial query in the store
-    store.dispatch(setInitialQuery("test query"));
-    expect(store.getState().initialQuery.initialQuery).toBe("test query");
+    store.dispatch(setInitialPrompt("test query"));
+    expect(store.getState().initialQuery.initialPrompt).toBe("test query");
 
     // Clear the initial query
-    store.dispatch(clearInitialQuery());
+    store.dispatch(clearInitialPrompt());
 
     // Verify initial query is cleared
-    expect(store.getState().initialQuery.initialQuery).toBeNull();
+    expect(store.getState().initialQuery.initialPrompt).toBeNull();
   });
 });

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "private": true,
   "type": "module",
   "engines": {
@@ -20,6 +20,7 @@
     "axios": "^1.7.9",
     "clsx": "^2.1.1",
     "eslint-config-airbnb-typescript": "^18.0.0",
+    "framer-motion": "^12.0.1",
     "i18next": "^24.2.1",
     "i18next-browser-languagedetector": "^8.0.2",
     "i18next-http-backend": "^3.0.1",
@@ -42,7 +43,7 @@
     "sirv-cli": "^3.0.0",
     "socket.io-client": "^4.8.1",
     "tailwind-merge": "^2.6.0",
-    "vite": "^6.0.7",
+    "vite": "^5.4.11",
     "web-vitals": "^3.5.2",
     "ws": "^8.18.0"
   },
@@ -79,7 +80,8 @@
     "@playwright/test": "^1.49.1",
     "@react-router/dev": "^7.1.2",
     "@tailwindcss/typography": "^0.5.16",
-    "@tanstack/eslint-plugin-query": "^5.62.16",
+    "@tanstack/eslint-plugin-query": "^5.64.2",
+    "@testing-library/dom": "^10.4.0",
     "@testing-library/jest-dom": "^6.6.1",
     "@testing-library/react": "^16.2.0",
     "@testing-library/user-event": "^14.6.0",
@@ -100,7 +102,7 @@
     "eslint-config-prettier": "^10.0.1",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-jsx-a11y": "^6.10.2",
-    "eslint-plugin-prettier": "^5.2.2",
+    "eslint-plugin-prettier": "^5.2.3",
     "eslint-plugin-react": "^7.37.4",
     "eslint-plugin-react-hooks": "^4.6.2",
     "husky": "^9.1.6",

frontend/src/api/open-hands.ts

@@ -10,6 +10,7 @@ import {
   AuthenticateResponse,
   Conversation,
   ResultSet,
+  GetTrajectoryResponse,
 } from "./open-hands.types";
 import { openHands } from "./open-hands-axios";
 import { ApiSettings } from "#/services/settings";
@@ -243,10 +244,14 @@ class OpenHands {
   static async createConversation(
     githubToken?: string,
     selectedRepository?: string,
+    initialUserMsg?: string,
+    imageUrls?: string[],
   ): Promise<Conversation> {
     const body = {
       github_token: githubToken,
       selected_repository: selectedRepository,
+      initial_user_msg: initialUserMsg,
+      image_urls: imageUrls,
     };
 
     const { data } = await openHands.post<Conversation>(
@@ -354,6 +359,15 @@ class OpenHands {
 
     return response.data.items;
   }
+
+  static async getTrajectory(
+    conversationId: string,
+  ): Promise<GetTrajectoryResponse> {
+    const { data } = await openHands.get<GetTrajectoryResponse>(
+      `/api/conversations/${conversationId}/trajectory`,
+    );
+    return data;
+  }
 }
 
 export default OpenHands;

frontend/src/api/open-hands.types.ts

@@ -55,6 +55,11 @@ export interface GetVSCodeUrlResponse {
   error?: string;
 }
 
+export interface GetTrajectoryResponse {
+  trajectory: unknown[] | null;
+  error?: string;
+}
+
 export interface AuthenticateResponse {
   message?: string;
   error?: string;

frontend/src/components/agent-status-map.constant.ts

@@ -23,7 +23,7 @@ export const AGENT_STATUS_MAP: {
   },
   [AgentState.AWAITING_USER_INPUT]: {
     message: I18nKey.CHAT_INTERFACE$AGENT_AWAITING_USER_INPUT_MESSAGE,
-    indicator: IndicatorColor.ORANGE,
+    indicator: IndicatorColor.BLUE,
   },
   [AgentState.PAUSED]: {
     message: I18nKey.CHAT_INTERFACE$AGENT_PAUSED_MESSAGE,

frontend/src/components/features/chat/chat-interface.tsx

@@ -1,8 +1,10 @@
 import { useDispatch, useSelector } from "react-redux";
+import toast from "react-hot-toast";
 import React from "react";
 import posthog from "posthog-js";
+import { useParams } from "react-router";
 import { convertImageToBase64 } from "#/utils/convert-image-to-base-64";
-import { FeedbackActions } from "../feedback/feedback-actions";
+import { TrajectoryActions } from "../trajectory/trajectory-actions";
 import { createChatMessage } from "#/services/chat-service";
 import { InteractiveChatBox } from "./interactive-chat-box";
 import { addUserMessage } from "#/state/chat-slice";
@@ -19,6 +21,8 @@ import { ActionSuggestions } from "./action-suggestions";
 import { ContinueButton } from "#/components/shared/buttons/continue-button";
 import { ScrollToBottomButton } from "#/components/shared/buttons/scroll-to-bottom-button";
 import { LoadingSpinner } from "#/components/shared/loading-spinner";
+import { useGetTrajectory } from "#/hooks/mutation/use-get-trajectory";
+import { downloadTrajectory } from "#/utils/download-files";
 
 function getEntryPoint(
   hasRepository: boolean | null,
@@ -47,6 +51,8 @@ export function ChatInterface() {
   const { selectedRepository, importedProjectZip } = useSelector(
     (state: RootState) => state.initialQuery,
   );
+  const params = useParams();
+  const { mutate: getTrajectory } = useGetTrajectory();
 
   const handleSendMessage = async (content: string, files: File[]) => {
     if (messages.length === 0) {
@@ -90,6 +96,25 @@ export function ChatInterface() {
     setFeedbackPolarity(polarity);
   };
 
+  const onClickExportTrajectoryButton = () => {
+    if (!params.conversationId) {
+      toast.error("ConversationId unknown, cannot download trajectory");
+      return;
+    }
+
+    getTrajectory(params.conversationId, {
+      onSuccess: async (data) => {
+        await downloadTrajectory(
+          params.conversationId ?? "unknown",
+          data.trajectory,
+        );
+      },
+      onError: (error) => {
+        toast.error(error.message);
+      },
+    });
+  };
+
   const isWaitingForUserInput =
     curAgentState === AgentState.AWAITING_USER_INPUT ||
     curAgentState === AgentState.FINISHED;
@@ -129,13 +154,14 @@ export function ChatInterface() {
 
       <div className="flex flex-col gap-[6px] px-4 pb-4">
         <div className="flex justify-between relative">
-          <FeedbackActions
+          <TrajectoryActions
             onPositiveFeedback={() =>
               onClickShareFeedbackActionButton("positive")
             }
             onNegativeFeedback={() =>
               onClickShareFeedbackActionButton("negative")
             }
+            onExportTrajectory={() => onClickExportTrajectoryButton()}
           />
 
           <div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">

frontend/src/components/features/chat/messages.tsx

@@ -12,15 +12,22 @@ interface MessagesProps {
 export const Messages: React.FC<MessagesProps> = React.memo(
   ({ messages, isAwaitingUserConfirmation }) =>
     messages.map((message, index) => {
+      const shouldShowConfirmationButtons =
+        messages.length - 1 === index &&
+        message.sender === "assistant" &&
+        isAwaitingUserConfirmation;
+
       if (message.type === "error" || message.type === "action") {
         return (
-          <ExpandableMessage
-            key={index}
-            type={message.type}
-            id={message.translationID}
-            message={message.content}
-            success={message.success}
-          />
+          <div key={index}>
+            <ExpandableMessage
+              type={message.type}
+              id={message.translationID}
+              message={message.content}
+              success={message.success}
+            />
+            {shouldShowConfirmationButtons && <ConfirmationButtons />}
+          </div>
         );
       }
 
@@ -33,9 +40,7 @@ export const Messages: React.FC<MessagesProps> = React.memo(
           {message.imageUrls && message.imageUrls.length > 0 && (
             <ImageCarousel size="small" images={message.imageUrls} />
           )}
-          {messages.length - 1 === index &&
-            message.sender === "assistant" &&
-            isAwaitingUserConfirmation && <ConfirmationButtons />}
+          {shouldShowConfirmationButtons && <ConfirmationButtons />}
         </ChatMessage>
       );
     }),

frontend/src/components/features/controls/agent-status-bar.tsx

@@ -43,7 +43,7 @@ export function AgentStatusBar() {
 
   React.useEffect(() => {
     if (status === WsClientProviderStatus.DISCONNECTED) {
-      setStatusMessage("Trying to reconnect...");
+      setStatusMessage("Connecting...");
     } else {
       setStatusMessage(AGENT_STATUS_MAP[curAgentState].message);
     }

frontend/src/components/features/trajectory/trajectory-actions.tsx

@@ -1,28 +1,36 @@
 import ThumbsUpIcon from "#/icons/thumbs-up.svg?react";
 import ThumbDownIcon from "#/icons/thumbs-down.svg?react";
-import { FeedbackActionButton } from "#/components/shared/buttons/feedback-action-button";
+import ExportIcon from "#/icons/export.svg?react";
+import { TrajectoryActionButton } from "#/components/shared/buttons/trajectory-action-button";
 
-interface FeedbackActionsProps {
+interface TrajectoryActionsProps {
   onPositiveFeedback: () => void;
   onNegativeFeedback: () => void;
+  onExportTrajectory: () => void;
 }
 
-export function FeedbackActions({
+export function TrajectoryActions({
   onPositiveFeedback,
   onNegativeFeedback,
-}: FeedbackActionsProps) {
+  onExportTrajectory,
+}: TrajectoryActionsProps) {
   return (
     <div data-testid="feedback-actions" className="flex gap-1">
-      <FeedbackActionButton
+      <TrajectoryActionButton
         testId="positive-feedback"
         onClick={onPositiveFeedback}
         icon={<ThumbsUpIcon width={15} height={15} />}
       />
-      <FeedbackActionButton
+      <TrajectoryActionButton
         testId="negative-feedback"
         onClick={onNegativeFeedback}
         icon={<ThumbDownIcon width={15} height={15} />}
       />
+      <TrajectoryActionButton
+        testId="export-trajectory"
+        onClick={onExportTrajectory}
+        icon={<ExportIcon width={15} height={15} />}
+      />
     </div>
   );
 }

frontend/src/components/shared/buttons/trajectory-action-button.tsx

@@ -1,14 +1,14 @@
-interface FeedbackActionButtonProps {
+interface TrajectoryActionButtonProps {
   testId?: string;
   onClick: () => void;
   icon: React.ReactNode;
 }
 
-export function FeedbackActionButton({
+export function TrajectoryActionButton({
   testId,
   onClick,
   icon,
-}: FeedbackActionButtonProps) {
+}: TrajectoryActionButtonProps) {
   return (
     <button
       type="button"

frontend/src/components/shared/modals/settings/settings-form.tsx

@@ -171,7 +171,7 @@ export function SettingsForm({
 
           <APIKeyInput
             isDisabled={!!disabled}
-            isSet={settings.LLM_API_KEY === "SET"}
+            isSet={settings.LLM_API_KEY === "**********"}
           />
 
           {showAdvancedOptions && (

frontend/src/context/settings-context.tsx

@@ -34,7 +34,7 @@ export function SettingsProvider({ children }: SettingsProviderProps) {
       ...newSettings,
     };
 
-    if (updatedSettings.LLM_API_KEY === "SET") {
+    if (updatedSettings.LLM_API_KEY === "**********") {
       delete updatedSettings.LLM_API_KEY;
     }
 

frontend/src/entry.client.tsx

@@ -11,15 +11,11 @@ import { hydrateRoot } from "react-dom/client";
 import { Provider } from "react-redux";
 import posthog from "posthog-js";
 import "./i18n";
-import {
-  QueryCache,
-  QueryClient,
-  QueryClientProvider,
-} from "@tanstack/react-query";
-import toast from "react-hot-toast";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
 import store from "./store";
 import { useConfig } from "./hooks/query/use-config";
 import { AuthProvider } from "./context/auth-context";
+import { queryClientConfig } from "./query-client-config";
 import { SettingsProvider } from "./context/settings-context";
 
 function PosthogInit() {
@@ -50,27 +46,7 @@ async function prepareApp() {
   }
 }
 
-const QUERY_KEYS_TO_IGNORE = ["authenticated", "hosts"];
-const queryClient = new QueryClient({
-  queryCache: new QueryCache({
-    onError: (error, query) => {
-      if (!QUERY_KEYS_TO_IGNORE.some((key) => query.queryKey.includes(key))) {
-        toast.error(error.message);
-      }
-    },
-  }),
-  defaultOptions: {
-    queries: {
-      staleTime: 1000 * 60 * 5, // 5 minutes
-      gcTime: 1000 * 60 * 15, // 15 minutes
-    },
-    mutations: {
-      onError: (error) => {
-        toast.error(error.message);
-      },
-    },
-  },
-});
+const queryClient = new QueryClient(queryClientConfig);
 
 prepareApp().then(() =>
   startTransition(() => {

frontend/src/hooks/mutation/use-create-conversation.ts

@@ -3,7 +3,7 @@ import { useNavigate } from "react-router";
 import posthog from "posthog-js";
 import { useDispatch, useSelector } from "react-redux";
 import OpenHands from "#/api/open-hands";
-import { setInitialQuery } from "#/state/initial-query-slice";
+import { setInitialPrompt } from "#/state/initial-query-slice";
 import { RootState } from "#/store";
 import { useAuth } from "#/context/auth-context";
 
@@ -18,7 +18,7 @@ export const useCreateConversation = () => {
   );
 
   return useMutation({
-    mutationFn: (variables: { q?: string }) => {
+    mutationFn: async (variables: { q?: string }) => {
       if (
         !variables.q?.trim() &&
         !selectedRepository &&
@@ -28,10 +28,13 @@ export const useCreateConversation = () => {
         throw new Error("No query provided");
       }
 
-      if (variables.q) dispatch(setInitialQuery(variables.q));
+      if (variables.q) dispatch(setInitialPrompt(variables.q));
+
       return OpenHands.createConversation(
         gitHubToken || undefined,
         selectedRepository || undefined,
+        variables.q,
+        files,
       );
     },
     onSuccess: async ({ conversation_id: conversationId }, { q }) => {

frontend/src/hooks/mutation/use-get-trajectory.ts

@@ -0,0 +1,7 @@
+import { useMutation } from "@tanstack/react-query";
+import OpenHands from "#/api/open-hands";
+
+export const useGetTrajectory = () =>
+  useMutation({
+    mutationFn: (cid: string) => OpenHands.getTrajectory(cid),
+  });

frontend/src/hooks/query/use-github-user.ts

@@ -6,7 +6,7 @@ import { useConfig } from "./use-config";
 import OpenHands from "#/api/open-hands";
 
 export const useGitHubUser = () => {
-  const { gitHubToken, setUserId } = useAuth();
+  const { gitHubToken, setUserId, logout } = useAuth();
   const { data: config } = useConfig();
 
   const user = useQuery({
@@ -29,5 +29,11 @@ export const useGitHubUser = () => {
     }
   }, [user.data]);
 
+  React.useEffect(() => {
+    if (user.isError) {
+      logout();
+    }
+  }, [user.isError]);
+
   return user;
 };

frontend/src/hooks/query/use-user-conversations.ts

@@ -9,5 +9,6 @@ export const useUserConversations = () => {
     queryKey: ["user", "conversations"],
     queryFn: OpenHands.getUserConversations,
     enabled: !!userIsAuthenticated,
+    staleTime: 0,
   });
 };

frontend/src/icons/export.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-download"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" x2="12" y1="15" y2="3"/></svg>

frontend/src/mocks/handlers.ts

@@ -141,7 +141,7 @@ export const handlers = [
       { id: 2, full_name: "octocat/earth" },
     ]),
   ),
-  http.get("https://api.github.com/user", () => {
+  http.get("/api/github/user", () => {
     const user: GitHubUser = {
       id: 1,
       login: "octocat",

frontend/src/query-client-config.ts

@@ -0,0 +1,25 @@
+import { QueryClientConfig, QueryCache } from "@tanstack/react-query";
+import toast from "react-hot-toast";
+
+const QUERY_KEYS_TO_IGNORE = ["authenticated", "hosts"];
+
+export const queryClientConfig: QueryClientConfig = {
+  queryCache: new QueryCache({
+    onError: (error, query) => {
+      if (!QUERY_KEYS_TO_IGNORE.some((key) => query.queryKey.includes(key))) {
+        toast.error(error.message);
+      }
+    },
+  }),
+  defaultOptions: {
+    queries: {
+      staleTime: 1000 * 60 * 5, // 5 minutes
+      gcTime: 1000 * 60 * 15, // 15 minutes
+    },
+    mutations: {
+      onError: (error) => {
+        toast.error(error.message);
+      },
+    },
+  },
+};

frontend/src/routes/_oh.app/event-handler.tsx

@@ -1,10 +1,8 @@
 import React from "react";
-import { useWSStatusChange } from "./hooks/use-ws-status-change";
 import { useHandleWSEvents } from "./hooks/use-handle-ws-events";
 import { useHandleRuntimeActive } from "./hooks/use-handle-runtime-active";
 
 export function EventHandler({ children }: React.PropsWithChildren) {
-  useWSStatusChange();
   useHandleWSEvents();
   useHandleRuntimeActive();
 

frontend/src/routes/_oh.app/hooks/use-ws-status-change.ts

@@ -1,68 +0,0 @@
-import React from "react";
-import { useDispatch, useSelector } from "react-redux";
-import {
-  useWsClient,
-  WsClientProviderStatus,
-} from "#/context/ws-client-provider";
-import { createChatMessage } from "#/services/chat-service";
-import { setCurrentAgentState } from "#/state/agent-slice";
-import { addUserMessage } from "#/state/chat-slice";
-import { clearFiles, clearInitialQuery } from "#/state/initial-query-slice";
-import { RootState } from "#/store";
-import { AgentState } from "#/types/agent-state";
-
-export const useWSStatusChange = () => {
-  const { send, status } = useWsClient();
-  const { curAgentState } = useSelector((state: RootState) => state.agent);
-  const dispatch = useDispatch();
-
-  const statusRef = React.useRef<WsClientProviderStatus | null>(null);
-
-  const { files, initialQuery } = useSelector(
-    (state: RootState) => state.initialQuery,
-  );
-
-  const sendInitialQuery = (query: string, base64Files: string[]) => {
-    const timestamp = new Date().toISOString();
-    send(createChatMessage(query, base64Files, timestamp));
-  };
-
-  const dispatchInitialQuery = (query: string) => {
-    sendInitialQuery(query, files);
-    dispatch(clearFiles()); // reset selected files
-    dispatch(clearInitialQuery()); // reset initial query
-  };
-
-  const handleAgentInit = () => {
-    if (initialQuery) {
-      dispatchInitialQuery(initialQuery);
-    }
-  };
-  React.useEffect(() => {
-    if (curAgentState === AgentState.INIT) {
-      handleAgentInit();
-    }
-  }, [curAgentState]);
-
-  React.useEffect(() => {
-    if (statusRef.current === status) {
-      return; // This is a check because of strict mode - if the status did not change, don't do anything
-    }
-    statusRef.current = status;
-
-    if (status !== WsClientProviderStatus.DISCONNECTED && initialQuery) {
-      dispatch(
-        addUserMessage({
-          content: initialQuery,
-          imageUrls: files,
-          timestamp: new Date().toISOString(),
-          pending: true,
-        }),
-      );
-    }
-
-    if (status === WsClientProviderStatus.DISCONNECTED) {
-      dispatch(setCurrentAgentState(AgentState.STOPPED));
-    }
-  }, [status]);
-};

frontend/src/routes/_oh.app/route.tsx

@@ -1,7 +1,7 @@
 import { useDisclosure } from "@nextui-org/react";
 import React from "react";
 import { Outlet } from "react-router";
-import { useDispatch } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
 import { FaServer } from "react-icons/fa";
 import toast from "react-hot-toast";
 import { useTranslation } from "react-i18next";
@@ -11,7 +11,7 @@ import {
   useConversation,
 } from "#/context/conversation-context";
 import { Controls } from "#/components/features/controls/controls";
-import { clearMessages } from "#/state/chat-slice";
+import { clearMessages, addUserMessage } from "#/state/chat-slice";
 import { clearTerminal } from "#/state/command-slice";
 import { useEffectOnce } from "#/hooks/use-effect-once";
 import CodeIcon from "#/icons/code.svg?react";
@@ -36,6 +36,8 @@ import { ServedAppLabel } from "#/components/layout/served-app-label";
 import { TerminalStatusLabel } from "#/components/features/terminal/terminal-status-label";
 import { useSettings } from "#/hooks/query/use-settings";
 import { MULTI_CONVERSATION_UI } from "#/utils/feature-flags";
+import { clearFiles, clearInitialPrompt } from "#/state/initial-query-slice";
+import { RootState } from "#/store";
 
 function AppContent() {
   useConversationConfig();
@@ -46,6 +48,9 @@ function AppContent() {
   const { data: conversation, isFetched } = useUserConversation(
     conversationId || null,
   );
+  const { initialPrompt, files } = useSelector(
+    (state: RootState) => state.initialQuery,
+  );
   const dispatch = useDispatch();
   const endSession = useEndSession();
 
@@ -74,6 +79,18 @@ function AppContent() {
     dispatch(clearMessages());
     dispatch(clearTerminal());
     dispatch(clearJupyter());
+    if (conversationId && (initialPrompt || files.length > 0)) {
+      dispatch(
+        addUserMessage({
+          content: initialPrompt || "",
+          imageUrls: files || [],
+          timestamp: new Date().toISOString(),
+          pending: true,
+        }),
+      );
+      dispatch(clearInitialPrompt());
+      dispatch(clearFiles());
+    }
   }, [conversationId]);
 
   useEffectOnce(() => {

frontend/src/state/initial-query-slice.ts

@@ -2,14 +2,14 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit";
 
 type SliceState = {
   files: string[]; // base64 encoded images
-  initialQuery: string | null;
+  initialPrompt: string | null;
   selectedRepository: string | null;
   importedProjectZip: string | null; // base64 encoded zip
 };
 
 const initialState: SliceState = {
   files: [],
-  initialQuery: null,
+  initialPrompt: null,
   selectedRepository: null,
   importedProjectZip: null,
 };
@@ -27,11 +27,11 @@ export const selectedFilesSlice = createSlice({
     clearFiles(state) {
       state.files = [];
     },
-    setInitialQuery(state, action: PayloadAction<string>) {
-      state.initialQuery = action.payload;
+    setInitialPrompt(state, action: PayloadAction<string>) {
+      state.initialPrompt = action.payload;
     },
-    clearInitialQuery(state) {
-      state.initialQuery = null;
+    clearInitialPrompt(state) {
+      state.initialPrompt = null;
     },
     setSelectedRepository(state, action: PayloadAction<string | null>) {
       state.selectedRepository = action.payload;
@@ -49,8 +49,8 @@ export const {
   addFile,
   removeFile,
   clearFiles,
-  setInitialQuery,
-  clearInitialQuery,
+  setInitialPrompt,
+  clearInitialPrompt,
   setSelectedRepository,
   clearSelectedRepository,
   setImportedProjectZip,

frontend/src/types/file-system.d.ts

@@ -26,6 +26,18 @@ interface FileSystemDirectoryHandle {
   ): Promise<FileSystemFileHandle>;
 }
 
+interface SaveFilePickerOptions {
+  suggestedName?: string;
+  types?: Array<{
+    description?: string;
+    accept: Record<string, string[]>;
+  }>;
+  excludeAcceptAllOption?: boolean;
+}
+
 interface Window {
   showDirectoryPicker(): Promise<FileSystemDirectoryHandle>;
+  showSaveFilePicker(
+    options?: SaveFilePickerOptions,
+  ): Promise<FileSystemFileHandle>;
 }

frontend/src/utils/download-files.ts

@@ -22,6 +22,13 @@ function isFileSystemAccessSupported(): boolean {
   return "showDirectoryPicker" in window;
 }
 
+/**
+ * Checks if the Save File Picker API is supported
+ */
+function isSaveFilePickerSupported(): boolean {
+  return "showSaveFilePicker" in window;
+}
+
 /**
  * Creates subdirectories and returns the final directory handle
  */
@@ -162,6 +169,39 @@ async function processBatch(
   };
 }
 
+export async function downloadTrajectory(
+  conversationId: string,
+  data: unknown[] | null,
+): Promise<void> {
+  try {
+    if (!isSaveFilePickerSupported()) {
+      throw new Error(
+        "Your browser doesn't support downloading folders. Please use Chrome, Edge, or another browser that supports the File System Access API.",
+      );
+    }
+    const options = {
+      suggestedName: `trajectory-${conversationId}.json`,
+      types: [
+        {
+          description: "JSON File",
+          accept: {
+            "application/json": [".json"],
+          },
+        },
+      ],
+    };
+
+    const handle = await window.showSaveFilePicker(options);
+    const writable = await handle.createWritable();
+    await writable.write(JSON.stringify(data, null, 2));
+    await writable.close();
+  } catch (error) {
+    throw new Error(
+      `Failed to download file: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
 /**
  * Downloads files from the workspace one by one
  * @param initialPath Initial path to start downloading from. If not provided, downloads from root

openhands/README.md

@@ -20,7 +20,7 @@ The key classes in OpenHands are:
     * Sandbox: the part of the runtime responsible for running commands, e.g. inside of Docker
 * Server: brokers OpenHands sessions over HTTP, e.g. to drive the frontend
     * Session: holds a single EventStream, a single AgentController, and a single Runtime. Generally represents a single task (but potentially including several user prompts)
-    * SessionManager: keeps a list of active sessions, and ensures requests are routed to the correct Session
+    * ConversationManager: keeps a list of active sessions, and ensures requests are routed to the correct Session
 
 ## Control Flow
 Here's the basic loop (in pseudocode) that drives agents.

openhands/agenthub/__init__.py

@@ -12,6 +12,7 @@ from openhands.agenthub import (  # noqa: E402
     codeact_agent,
     delegator_agent,
     dummy_agent,
+    visualbrowsing_agent,
 )
 
 __all__ = [
@@ -19,6 +20,7 @@ __all__ = [
     'delegator_agent',
     'dummy_agent',
     'browsing_agent',
+    'visualbrowsing_agent',
 ]
 
 for agent in all_microagents.values():

openhands/agenthub/codeact_agent/codeact_agent.py

@@ -277,16 +277,11 @@ class CodeActAgent(Agent):
             # if it doesn't have tool call metadata, it was triggered by a user action
             if obs.tool_call_metadata is None:
                 text = truncate_content(
-                    f'\nObserved result of command executed by user:\n{obs.content}',
+                    f'\nObserved result of command executed by user:\n{obs.to_agent_observation()}',
                     max_message_chars,
                 )
             else:
-                text = truncate_content(
-                    obs.content
-                    + f'\n[Python Interpreter: {obs.metadata.py_interpreter_path}]',
-                    max_message_chars,
-                )
-            text += f'\n[Command finished with exit code {obs.exit_code}]'
+                text = truncate_content(obs.to_agent_observation(), max_message_chars)
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, IPythonRunCellObservation):
             text = obs.content

openhands/agenthub/codeact_agent/function_calling.py

@@ -32,6 +32,7 @@ from openhands.events.tool import ToolCallMetadata
 _BASH_DESCRIPTION = """Execute a bash command in the terminal.
 * Long running commands: For commands that may run indefinitely, it should be run in the background and the output should be redirected to a file, e.g. command = `python3 app.py > server.log 2>&1 &`.
 * Interact with running process: If a bash command returns exit code `-1`, this means the process is not yet finished. By setting `is_input` to `true`, the assistant can interact with the running process and send empty `command` to retrieve any additional logs, or send additional text (set `command` to the text) to STDIN of the running process, or send command like `C-c` (Ctrl+C), `C-d` (Ctrl+D), `C-z` (Ctrl+Z) to interrupt the process.
+* One command at a time: You can only execute one bash command at a time. If you need to run multiple commands sequentially, you can use `&&` or `;` to chain them together.
 """
 
 CmdRunTool = ChatCompletionToolParam(
@@ -44,7 +45,7 @@ CmdRunTool = ChatCompletionToolParam(
             'properties': {
                 'command': {
                     'type': 'string',
-                    'description': 'The bash command to execute. Can be empty string to view additional logs when previous exit code is `-1`. Can be `C-c` (Ctrl+C) to interrupt the currently running process.',
+                    'description': 'The bash command to execute. Can be empty string to view additional logs when previous exit code is `-1`. Can be `C-c` (Ctrl+C) to interrupt the currently running process. Note: You can only execute one bash command at a time. If you need to run multiple commands sequentially, you can use `&&` or `;` to chain them together.',
                 },
                 'is_input': {
                     'type': 'string',
@@ -80,7 +81,7 @@ IPythonTool = ChatCompletionToolParam(
     ),
 )
 
-_FILE_EDIT_DESCRIPTION = """Edit a file.
+_FILE_EDIT_DESCRIPTION = """Edit a file in plain-text format.
 * The assistant can edit files by specifying the file path and providing a draft of the new file content.
 * The draft content doesn't need to be exactly the same as the existing file; the assistant may skip unchanged lines using comments like `# unchanged` to indicate unchanged sections.
 * IMPORTANT: For large files (e.g., > 300 lines), specify the range of lines to edit using `start` and `end` (1-indexed, inclusive). The range should be smaller than 300 lines.
@@ -216,7 +217,7 @@ LLMBasedFileEditTool = ChatCompletionToolParam(
     ),
 )
 
-_STR_REPLACE_EDITOR_DESCRIPTION = """Custom editing tool for viewing, creating and editing files
+_STR_REPLACE_EDITOR_DESCRIPTION = """Custom editing tool for viewing, creating and editing files in plain-text format
 * State is persistent across command calls and discussions with the user
 * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep
 * The `create` command cannot be used if the specified `path` already exists as a file

openhands/agenthub/codeact_agent/prompts/system_prompt.j2

@@ -1,6 +1,7 @@
 You are OpenHands agent, a helpful AI assistant that can interact with a computer to solve tasks.
 <IMPORTANT>
 * If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.
+* You should start exploring the file system with your view command, unless you need to explore more deeply.
 * When configuring git credentials, use "openhands" as the user.name and "openhands@all-hands.dev" as the user.email by default, unless explicitly instructed otherwise.
-* The assistant MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior.
+* You MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior.
 </IMPORTANT>

openhands/agenthub/visualbrowsing_agent/README.md

@@ -0,0 +1,7 @@
+# Browsing Agent Framework
+
+This folder implements the AgentLab [generic agent](https://github.com/ServiceNow/AgentLab/tree/main/src/agentlab/agents/generic_agent) that enables full-featured web browsing. The observations given to the agent include set-of-marks annotated web-page screenshot, accessibility tree of the web-page and all the thoughts and actions from previous steps.
+
+## Test run
+
+Note that for browsing tasks, GPT-4/Claude is usually a requirement to get reasonable results, due to the complexity of the web page structures. This agent has been evaluated on the VisualWebArena benchmark and the CodeAct agent does not call this VisualBrowsingAgent. CodeAct agent uses has in-built support for browsing (e.g., via browse_url and browser tool).

openhands/agenthub/visualbrowsing_agent/__init__.py

@@ -0,0 +1,6 @@
+from openhands.agenthub.visualbrowsing_agent.visualbrowsing_agent import (
+    VisualBrowsingAgent,
+)
+from openhands.controller.agent import Agent
+
+Agent.register('VisualBrowsingAgent', VisualBrowsingAgent)

openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py

@@ -0,0 +1,306 @@
+from browsergym.core.action.highlevel import HighLevelActionSet
+from browsergym.utils.obs import flatten_axtree_to_str
+
+from openhands.agenthub.browsing_agent.response_parser import BrowsingResponseParser
+from openhands.controller.agent import Agent
+from openhands.controller.state.state import State
+from openhands.core.config import AgentConfig
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.message import ImageContent, Message, TextContent
+from openhands.events.action import (
+    Action,
+    AgentFinishAction,
+    BrowseInteractiveAction,
+    MessageAction,
+)
+from openhands.events.event import EventSource
+from openhands.events.observation import BrowserOutputObservation
+from openhands.events.observation.observation import Observation
+from openhands.llm.llm import LLM
+from openhands.runtime.plugins import (
+    PluginRequirement,
+)
+
+
+def get_error_prefix(obs: BrowserOutputObservation) -> str:
+    # temporary fix for OneStopMarket to ignore timeout errors
+    if 'timeout' in obs.last_browser_action_error:
+        return ''
+    return f'## Error from previous action:\n{obs.last_browser_action_error}\n'
+
+
+def create_goal_prompt(goal: str, image_urls: list[str] | None):
+    goal_txt: str = f"""\
+# Instructions
+Review the current state of the page and all other information to find the best possible next action to accomplish your goal. Your answer will be interpreted and executed by a program, make sure to follow the formatting instructions.
+
+## Goal:
+{goal}
+"""
+    goal_image_urls = []
+    if image_urls is not None:
+        for idx, url in enumerate(image_urls):
+            goal_txt = goal_txt + f'Images: Goal input image ({idx+1})\n'
+            goal_image_urls.append(url)
+    goal_txt += '\n'
+    return goal_txt, goal_image_urls
+
+
+def create_observation_prompt(
+    axtree_txt: str,
+    tabs: str,
+    focused_element: str,
+    error_prefix: str,
+    som_screenshot: str | None,
+):
+    txt_observation = f"""
+# Observation of current step:
+{tabs}{axtree_txt}{focused_element}{error_prefix}
+"""
+
+    # screenshot + som: will be a non-empty string if present in observation
+    screenshot_url = None
+    if (som_screenshot is not None) and (len(som_screenshot) > 0):
+        txt_observation += 'Image: Current page screenshot (Note that only visible portion of webpage is present in the screenshot. You may need to scroll to view the remaining portion of the web-page.\n'
+        screenshot_url = som_screenshot
+    else:
+        logger.info('SOM Screenshot not present in observation!')
+    txt_observation += '\n'
+    return txt_observation, screenshot_url
+
+
+def get_tabs(obs: BrowserOutputObservation) -> str:
+    prompt_pieces = ['\n## Currently open tabs:']
+    for page_index, page_url in enumerate(obs.open_pages_urls):
+        active_or_not = ' (active tab)' if page_index == obs.active_page_index else ''
+        prompt_piece = f"""\
+Tab {page_index}{active_or_not}:
+URL: {page_url}
+"""
+        prompt_pieces.append(prompt_piece)
+    return '\n'.join(prompt_pieces) + '\n'
+
+
+def get_axtree(axtree_txt: str) -> str:
+    bid_info = """\
+Note: [bid] is the unique alpha-numeric identifier at the beginning of lines for each element in the AXTree. Always use bid to refer to elements in your actions.
+
+"""
+    visible_tag_info = """\
+Note: You can only interact with visible elements. If the "visible" tag is not present, the element is not visible on the page.
+
+"""
+    return f'\n## AXTree:\n{bid_info}{visible_tag_info}{axtree_txt}\n'
+
+
+def get_action_prompt(action_set: HighLevelActionSet) -> str:
+    action_set_generic_info = """\
+Note: This action set allows you to interact with your environment. Most of them are python function executing playwright code. The primary way of referring to elements in the page is through bid which are specified in your observations.
+
+"""
+    action_description = action_set.describe(
+        with_long_description=False,
+        with_examples=False,
+    )
+    action_prompt = f'# Action space:\n{action_set_generic_info}{action_description}\n'
+    return action_prompt
+
+
+def get_history_prompt(prev_actions: list[BrowseInteractiveAction]) -> str:
+    history_prompt = ['# History of all previous interactions with the task:\n']
+    for i in range(len(prev_actions)):
+        history_prompt.append(f'## step {i+1}')
+        history_prompt.append(
+            f'\nOuput thought and action: {prev_actions[i].thought} ```{prev_actions[i].browser_actions}```\n'
+        )
+    return '\n'.join(history_prompt) + '\n'
+
+
+class VisualBrowsingAgent(Agent):
+    VERSION = '1.0'
+    """
+    VisualBrowsing Agent that can uses webpage screenshots during browsing.
+    """
+
+    sandbox_plugins: list[PluginRequirement] = []
+    response_parser = BrowsingResponseParser()
+
+    def __init__(
+        self,
+        llm: LLM,
+        config: AgentConfig,
+    ) -> None:
+        """Initializes a new instance of the VisualBrowsingAgent class.
+
+        Parameters:
+        - llm (LLM): The llm to be used by this agent
+        """
+        super().__init__(llm, config)
+        # define a configurable action space, with chat functionality, web navigation, and webpage grounding using accessibility tree and HTML.
+        # see https://github.com/ServiceNow/BrowserGym/blob/main/core/src/browsergym/core/action/highlevel.py for more details
+        action_subsets = [
+            'chat',
+            'bid',
+            'nav',
+            'tab',
+            'infeas',
+        ]
+        self.action_space = HighLevelActionSet(
+            subsets=action_subsets,
+            strict=False,  # less strict on the parsing of the actions
+            multiaction=False,
+        )
+        self.action_prompt = get_action_prompt(self.action_space)
+        self.abstract_example = f"""
+# Abstract Example
+
+Here is an abstract version of the answer with description of the content of each tag. Make sure you follow this structure, but replace the content with your answer:
+
+You must mandatorily think step by step. If you need to make calculations such as coordinates, write them here. Describe the effect that your previous action had on the current content of the page. In summary the next action I will perform is ```{self.action_space.example_action(abstract=True)}```
+"""
+        self.concrete_example = """
+# Concrete Example
+
+Here is a concrete example of how to format your answer. Make sure to generate the action in the correct format ensuring that the action is present inside ``````:
+
+Let's think step-by-step. From previous action I tried to set the value of year to "2022", using select_option, but it doesn't appear to be in the form. It may be a dynamic dropdown, I will try using click with the bid "324" and look at the response from the page. In summary the next action I will perform is ```click('324')```
+"""
+        self.hints = """
+Note:
+* Make sure to use bid to identify elements when using commands.
+* Interacting with combobox, dropdowns and auto-complete fields can be tricky, sometimes you need to use select_option, while other times you need to use fill or click and wait for the reaction of the page.
+
+"""
+        self.reset()
+
+    def reset(self) -> None:
+        """Resets the VisualBrowsingAgent."""
+        super().reset()
+        self.cost_accumulator = 0
+        self.error_accumulator = 0
+
+    def step(self, state: State) -> Action:
+        """Performs one step using the VisualBrowsingAgent.
+
+        This includes gathering information on previous steps and prompting the model to make a browsing command to execute.
+
+        Parameters:
+        - state (State): used to get updated info
+
+        Returns:
+        - BrowseInteractiveAction(browsergym_command) - BrowserGym commands to run
+        - MessageAction(content) - Message action to run (e.g. ask for clarification)
+        - AgentFinishAction() - end the interaction
+        """
+        messages: list[Message] = []
+        prev_actions = []
+        cur_axtree_txt = ''
+        error_prefix = ''
+        focused_element = ''
+        tabs = ''
+        last_obs = None
+        last_action = None
+
+        if len(state.history) == 1:
+            # for visualwebarena, webarena and miniwob++ eval, we need to retrieve the initial observation already in browser env
+            # initialize and retrieve the first observation by issuing an noop OP
+            # For non-benchmark browsing, the browser env starts with a blank page, and the agent is expected to first navigate to desired websites
+            return BrowseInteractiveAction(browser_actions='noop(1000)')
+
+        for event in state.history:
+            if isinstance(event, BrowseInteractiveAction):
+                prev_actions.append(event)
+                last_action = event
+            elif isinstance(event, MessageAction) and event.source == EventSource.AGENT:
+                # agent has responded, task finished.
+                return AgentFinishAction(outputs={'content': event.content})
+            elif isinstance(event, Observation):
+                last_obs = event
+
+        if len(prev_actions) >= 1:  # ignore noop()
+            prev_actions = prev_actions[1:]  # remove the first noop action
+
+        # if the final BrowserInteractiveAction exec BrowserGym's send_msg_to_user,
+        # we should also send a message back to the user in OpenHands and call it a day
+        if (
+            isinstance(last_action, BrowseInteractiveAction)
+            and last_action.browsergym_send_msg_to_user
+        ):
+            return MessageAction(last_action.browsergym_send_msg_to_user)
+
+        history_prompt = get_history_prompt(prev_actions)
+        if isinstance(last_obs, BrowserOutputObservation):
+            if last_obs.error:
+                # add error recovery prompt prefix
+                error_prefix = get_error_prefix(last_obs)
+                if len(error_prefix) > 0:
+                    self.error_accumulator += 1
+                    if self.error_accumulator > 5:
+                        return MessageAction(
+                            'Too many errors encountered. Task failed.'
+                        )
+            focused_element = '## Focused element:\nNone\n'
+            if last_obs.focused_element_bid is not None:
+                focused_element = (
+                    f"## Focused element:\nbid='{last_obs.focused_element_bid}'\n"
+                )
+            tabs = get_tabs(last_obs)
+            try:
+                # IMPORTANT: keep AX Tree of full webpage, add visible and clickable tags
+                cur_axtree_txt = flatten_axtree_to_str(
+                    last_obs.axtree_object,
+                    extra_properties=last_obs.extra_element_properties,
+                    with_visible=True,
+                    with_clickable=True,
+                    with_center_coords=False,
+                    with_bounding_box_coords=False,
+                    filter_visible_only=False,
+                    filter_with_bid_only=False,
+                    filter_som_only=False,
+                )
+                cur_axtree_txt = get_axtree(axtree_txt=cur_axtree_txt)
+            except Exception as e:
+                logger.error(
+                    'Error when trying to process the accessibility tree: %s', e
+                )
+                return MessageAction('Error encountered when browsing.')
+            set_of_marks = last_obs.set_of_marks
+        goal, image_urls = state.get_current_user_intent()
+
+        if goal is None:
+            goal = state.inputs['task']
+        goal_txt, goal_images = create_goal_prompt(goal, image_urls)
+        observation_txt, som_screenshot = create_observation_prompt(
+            cur_axtree_txt, tabs, focused_element, error_prefix, set_of_marks
+        )
+        human_prompt = [TextContent(type='text', text=goal_txt)]
+        if len(goal_images) > 0:
+            human_prompt.append(ImageContent(image_urls=goal_images))
+        human_prompt.append(TextContent(type='text', text=observation_txt))
+        if som_screenshot is not None:
+            human_prompt.append(ImageContent(image_urls=[som_screenshot]))
+        remaining_content = f"""
+{history_prompt}\
+{self.action_prompt}\
+{self.hints}\
+{self.abstract_example}\
+{self.concrete_example}\
+"""
+        human_prompt.append(TextContent(type='text', text=remaining_content))
+
+        system_msg = """\
+You are an agent trying to solve a web task based on the content of the page and user instructions. You can interact with the page and explore, and send messages to the user when you finish the task. Each time you submit an action it will be sent to the browser and you will receive a new page.
+""".strip()
+
+        messages.append(Message(role='system', content=[TextContent(text=system_msg)]))
+        messages.append(Message(role='user', content=human_prompt))
+
+        flat_messages = self.llm.format_messages_for_llm(messages)
+
+        response = self.llm.completion(
+            messages=flat_messages,
+            temperature=0.0,
+            stop=[')```', ')\n```'],
+        )
+
+        return self.response_parser.parse(response)

openhands/controller/agent_controller.py

@@ -501,10 +501,6 @@ class AgentController:
             EventSource.ENVIRONMENT,
         )
 
-        if new_state == AgentState.INIT and self.state.resume_state:
-            await self.set_agent_state_to(self.state.resume_state)
-            self.state.resume_state = None
-
     def get_agent_state(self) -> AgentState:
         """Returns the current state of the agent.
 

openhands/core/config/agent_config.py

@@ -27,6 +27,6 @@ class AgentConfig(BaseModel):
     memory_enabled: bool = Field(default=False)
     memory_max_threads: int = Field(default=3)
     llm_config: str | None = Field(default=None)
-    enable_prompt_extensions: bool = Field(default=False)
+    enable_prompt_extensions: bool = Field(default=True)
     disabled_microagents: list[str] | None = Field(default=None)
     condenser: CondenserConfig = Field(default_factory=NoOpCondenserConfig)

openhands/core/config/llm_config.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import os
-
 from typing import Any
+
 from pydantic import BaseModel, Field, SecretStr
 
 from openhands.core.logger import LOG_DIR
@@ -39,12 +39,12 @@ class LLMConfig(BaseModel):
         drop_params: Drop any unmapped (unsupported) params without causing an exception.
         modify_params: Modify params allows litellm to do transformations like adding a default message, when a message is empty.
         disable_vision: If model is vision capable, this option allows to disable image processing (useful for cost reduction).
-        reasoning_effort: The effort to put into reasoning. This is a string that can be one of 'low', 'medium', 'high', or 'none'. Exclusive for o1 models.
         caching_prompt: Use the prompt caching feature if provided by the LLM and supported by the provider.
         log_completions: Whether to log LLM completions to the state.
         log_completions_folder: The folder to log LLM completions to. Required if log_completions is True.
         custom_tokenizer: A custom tokenizer to use for token counting.
         native_tool_calling: Whether to use native tool calling if supported by the model. Can be True, False, or not set.
+        reasoning_effort: The effort to put into reasoning. This is a string that can be one of 'low', 'medium', 'high', or 'none'. Exclusive for o1 models.
     """
 
     model: str = Field(default='claude-3-5-sonnet-20241022')
@@ -85,7 +85,8 @@ class LLMConfig(BaseModel):
     log_completions_folder: str = Field(default=os.path.join(LOG_DIR, 'completions'))
     custom_tokenizer: str | None = Field(default=None)
     native_tool_calling: bool | None = Field(default=None)
-    
+    reasoning_effort: str | None = Field(default=None)
+
     model_config = {'extra': 'forbid'}
 
     def model_post_init(self, __context: Any):

openhands/core/config/sandbox_config.py

@@ -60,7 +60,7 @@ class SandboxConfig(BaseModel):
     runtime_startup_env_vars: dict[str, str] = Field(default_factory=dict)
     browsergym_eval_env: str | None = Field(default=None)
     platform: str | None = Field(default=None)
-    close_delay: int = Field(default=900)
+    close_delay: int = Field(default=15)
     remote_runtime_resource_factor: int = Field(default=1)
     enable_gpu: bool = Field(default=False)
     docker_runtime_kwargs: str | None = Field(default=None)

openhands/core/config/utils.py

@@ -9,7 +9,7 @@ from uuid import uuid4
 
 import toml
 from dotenv import load_dotenv
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, SecretStr, ValidationError
 
 from openhands.core import logger
 from openhands.core.config.agent_config import AgentConfig
@@ -192,7 +192,7 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                                     custom_fields[k] = v
                             merged_llm_dict = generic_llm_fields.copy()
                             merged_llm_dict.update(custom_fields)
-                            
+
                             custom_llm_config = LLMConfig(**merged_llm_dict)
                             cfg.set_llm_config(custom_llm_config, nested_key)
 
@@ -287,8 +287,10 @@ def finalize_config(cfg: AppConfig):
         pathlib.Path(cfg.cache_dir).mkdir(parents=True, exist_ok=True)
 
     if not cfg.jwt_secret:
-        cfg.jwt_secret = get_or_create_jwt_secret(
-            get_file_store(cfg.file_store, cfg.file_store_path)
+        cfg.jwt_secret = SecretStr(
+            get_or_create_jwt_secret(
+                get_file_store(cfg.file_store, cfg.file_store_path)
+            )
         )
 
 

openhands/core/schema/action.py

@@ -4,10 +4,6 @@ __all__ = ['ActionType']
 
 
 class ActionTypeSchema(BaseModel):
-    INIT: str = Field(default='initialize')
-    """Initializes the agent. Only sent by client.
-    """
-
     MESSAGE: str = Field(default='message')
     """Represents a message.
     """

openhands/core/schema/agent.py

@@ -6,10 +6,6 @@ class AgentState(str, Enum):
     """The agent is loading.
     """
 
-    INIT = 'init'
-    """The agent is initialized.
-    """
-
     RUNNING = 'running'
     """The agent is running.
     """

openhands/events/observation/browse.py

@@ -13,8 +13,10 @@ class BrowserOutputObservation(Observation):
     url: str
     trigger_by_action: str
     screenshot: str = field(repr=False, default='')  # don't show in repr
+    set_of_marks: str = field(default='', repr=False)  # don't show in repr
     error: bool = False
     observation: str = ObservationType.BROWSE
+    goal_image_urls: list = field(default_factory=list)
     # do not include in the memory
     open_pages_urls: list = field(default_factory=list)
     active_page_index: int = -1

openhands/events/observation/commands.py

@@ -149,16 +149,18 @@ class CmdOutputObservation(Observation):
             f'**CmdOutputObservation (source={self.source}, exit code={self.exit_code}, '
             f'metadata={json.dumps(self.metadata.model_dump(), indent=2)})**\n'
             '--BEGIN AGENT OBSERVATION--\n'
-            f'{self._to_agent_observation()}\n'
+            f'{self.to_agent_observation()}\n'
             '--END AGENT OBSERVATION--'
         )
 
-    def _to_agent_observation(self) -> str:
+    def to_agent_observation(self) -> str:
         ret = f'{self.metadata.prefix}{self.content}{self.metadata.suffix}'
         if self.metadata.working_dir:
             ret += f'\n[Current working directory: {self.metadata.working_dir}]'
         if self.metadata.py_interpreter_path:
             ret += f'\n[Python interpreter: {self.metadata.py_interpreter_path}]'
+        if self.metadata.exit_code != -1:
+            ret += f'\n[Command finished with exit code {self.metadata.exit_code}]'
         return ret
 
 

openhands/llm/llm.py

@@ -152,6 +152,12 @@ class LLM(RetryMixin, DebugMixin):
             temperature=self.config.temperature,
             top_p=self.config.top_p,
             drop_params=self.config.drop_params,
+            # add reasoning_effort, only if the model is supported
+            **(
+                {'reasoning_effort': self.config.reasoning_effort}
+                if self.config.model.lower() in REASONING_EFFORT_SUPPORTED_MODELS
+                else {}
+            ),
         )
 
         self._completion_unwrapped = self._completion
@@ -217,10 +223,6 @@ class LLM(RetryMixin, DebugMixin):
                         'anthropic-beta': 'prompt-caching-2024-07-31',
                     }
 
-            # Set reasoning effort for models that support it
-            if self.config.model.lower() in REASONING_EFFORT_SUPPORTED_MODELS:
-                kwargs['reasoning_effort'] = self.config.reasoning_effort
-
             # set litellm modify_params to the configured value
             # True by default to allow litellm to do transformations like adding a default message, when a message is empty
             # NOTE: this setting is global; unlike drop_params, it cannot be overridden in the litellm completion partial

openhands/resolver/github_issue.py

@@ -18,3 +18,4 @@ class GithubIssue(BaseModel):
     review_threads: list[ReviewThread] | None = None
     thread_ids: list[str] | None = None
     head_branch: str | None = None
+    base_branch: str | None = None

openhands/resolver/resolve_all_issues.py

@@ -331,9 +331,10 @@ def main():
     if not token:
         raise ValueError('Github token is required.')
 
+    api_key = my_args.llm_api_key or os.environ['LLM_API_KEY']
     llm_config = LLMConfig(
         model=my_args.llm_model or os.environ['LLM_MODEL'],
-        api_key=my_args.llm_api_key or os.environ['LLM_API_KEY'],
+        api_key=str(api_key) if api_key else None,
         base_url=my_args.llm_base_url or os.environ.get('LLM_BASE_URL', None),
     )
 

openhands/resolver/resolve_issue.py

@@ -307,7 +307,6 @@ async def resolve_issue(
     repo_instruction: str | None,
     issue_number: int,
     comment_id: int | None,
-    target_branch: str | None = None,
     reset_logger: bool = False,
 ) -> None:
     """Resolve a single github issue.
@@ -326,7 +325,7 @@ async def resolve_issue(
         repo_instruction: Repository instruction to use.
         issue_number: Issue number to resolve.
         comment_id: Optional ID of a specific comment to focus on.
-        target_branch: Optional target branch to create PR against (for PRs).
+
         reset_logger: Whether to reset the logger for multiprocessing.
     """
     issue_handler = issue_handler_factory(issue_type, owner, repo, token, llm_config)
@@ -424,9 +423,9 @@ async def resolve_issue(
     try:
         # checkout to pr branch if needed
         if issue_type == 'pr':
-            branch_to_use = target_branch if target_branch else issue.head_branch
+            branch_to_use = issue.head_branch
             logger.info(
-                f'Checking out to PR branch {target_branch} for issue {issue.number}'
+                f'Checking out to PR branch {branch_to_use} for issue {issue.number}'
             )
 
             if not branch_to_use:
@@ -446,10 +445,6 @@ async def resolve_issue(
                 cwd=repo_dir,
             )
 
-            # Update issue's base_branch if using custom target branch
-            if target_branch:
-                issue.base_branch = target_branch
-
             base_commit = (
                 subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=repo_dir)
                 .decode('utf-8')
@@ -572,12 +567,6 @@ def main():
         choices=['issue', 'pr'],
         help='Type of issue to resolve, either open issue or pr comments.',
     )
-    parser.add_argument(
-        '--target-branch',
-        type=str,
-        default=None,
-        help="Target branch to pull and create PR against (for PRs). If not specified, uses the PR's base branch.",
-    )
     parser.add_argument(
         '--is-experimental',
         type=lambda x: x.lower() == 'true',
@@ -601,9 +590,10 @@ def main():
     if not token:
         raise ValueError('Github token is required.')
 
+    api_key = my_args.llm_api_key or os.environ['LLM_API_KEY']
     llm_config = LLMConfig(
         model=my_args.llm_model or os.environ['LLM_MODEL'],
-        api_key=my_args.llm_api_key or os.environ['LLM_API_KEY'],
+        api_key=str(api_key) if api_key else None,
         base_url=my_args.llm_base_url or os.environ.get('LLM_BASE_URL', None),
     )
 
@@ -643,7 +633,6 @@ def main():
             repo_instruction=repo_instruction,
             issue_number=my_args.issue_number,
             comment_id=my_args.comment_id,
-            target_branch=my_args.target_branch,
         )
     )
 

openhands/resolver/send_pull_request.py

@@ -719,9 +719,10 @@ def main():
         else os.getenv('GITHUB_USERNAME')
     )
 
+    api_key = my_args.llm_api_key or os.environ['LLM_API_KEY']
     llm_config = LLMConfig(
         model=my_args.llm_model or os.environ['LLM_MODEL'],
-        api_key=my_args.llm_api_key or os.environ['LLM_API_KEY'],
+        api_key=str(api_key) if api_key else None,
         base_url=my_args.llm_base_url or os.environ.get('LLM_BASE_URL', None),
     )
 

openhands/runtime/base.py

@@ -136,6 +136,10 @@ class Runtime(FileEditRuntimeMixin):
     def close(self) -> None:
         pass
 
+    @classmethod
+    async def delete(cls, conversation_id: str) -> None:
+        pass
+
     def log(self, level: str, message: str) -> None:
         message = f'[runtime {self.sid}] {message}'
         getattr(logger, level)(message, stacklevel=2)

openhands/runtime/browser/browser_env.py

@@ -11,7 +11,7 @@ import gymnasium as gym
 import html2text
 import numpy as np
 import tenacity
-from browsergym.utils.obs import flatten_dom_to_str
+from browsergym.utils.obs import flatten_dom_to_str, overlay_som
 from PIL import Image
 
 from openhands.core.exceptions import BrowserInitException
@@ -65,15 +65,22 @@ class BrowserEnv:
             logger.error(f'Failed to start browser process: {e}')
             raise
 
-        if not self.check_alive():
+        if not self.check_alive(timeout=200):
             self.close()
             raise BrowserInitException('Failed to start browser environment.')
 
     def browser_process(self):
         if self.eval_mode:
             assert self.browsergym_eval_env is not None
-            logger.debug('Initializing browser env for web browsing evaluation.')
-            if 'webarena' in self.browsergym_eval_env:
+            logger.info('Initializing browser env for web browsing evaluation.')
+            if not self.browsergym_eval_env.startswith('browsergym/'):
+                self.browsergym_eval_env = 'browsergym/' + self.browsergym_eval_env
+            if 'visualwebarena' in self.browsergym_eval_env:
+                import browsergym.visualwebarena  # noqa F401 register visualwebarena tasks as gym environments
+                import nltk
+
+                nltk.download('punkt_tab')
+            elif 'webarena' in self.browsergym_eval_env:
                 import browsergym.webarena  # noqa F401 register webarena tasks as gym environments
             elif 'miniwob' in self.browsergym_eval_env:
                 import browsergym.miniwob  # noqa F401 register miniwob tasks as gym environments
@@ -81,10 +88,7 @@ class BrowserEnv:
                 raise ValueError(
                     f'Unsupported browsergym eval env: {self.browsergym_eval_env}'
                 )
-            env = gym.make(
-                self.browsergym_eval_env,
-                tags_to_mark='all',
-            )
+            env = gym.make(self.browsergym_eval_env, tags_to_mark='all', timeout=100000)
         else:
             env = gym.make(
                 'browsergym/openended',
@@ -94,17 +98,27 @@ class BrowserEnv:
                 disable_env_checker=True,
                 tags_to_mark='all',
             )
-
         obs, info = env.reset()
 
+        logger.info('Successfully called env.reset')
         # EVAL ONLY: save the goal into file for evaluation
         self.eval_goal = None
+        self.goal_image_urls = []
         self.eval_rewards: list[float] = []
         if self.eval_mode:
-            logger.debug(f"Browsing goal: {obs['goal']}")
             self.eval_goal = obs['goal']
+            if 'goal_object' in obs:
+                if len(obs['goal_object']) > 0:
+                    self.eval_goal = obs['goal_object'][0]['text']
+                for message in obs['goal_object']:
+                    if message['type'] == 'image_url':
+                        image_src = message['image_url']
+                        if isinstance(image_src, dict):
+                            image_src = image_src['url']
+                        self.goal_image_urls.append(image_src)
+            logger.debug(f'Browsing goal: {self.eval_goal}')
+        logger.info('Browser env started.')
 
-        logger.debug('Browser env started.')
         while should_continue():
             try:
                 if self.browser_side.poll(timeout=0.01):
@@ -122,7 +136,13 @@ class BrowserEnv:
                     # EVAL ONLY: Get evaluation info
                     if action_data['action'] == BROWSER_EVAL_GET_GOAL_ACTION:
                         self.browser_side.send(
-                            (unique_request_id, {'text_content': self.eval_goal})
+                            (
+                                unique_request_id,
+                                {
+                                    'text_content': self.eval_goal,
+                                    'image_content': self.goal_image_urls,
+                                },
+                            )
                         )
                         continue
                     elif action_data['action'] == BROWSER_EVAL_GET_REWARDS_ACTION:
@@ -145,7 +165,15 @@ class BrowserEnv:
                     html_str = flatten_dom_to_str(obs['dom_object'])
                     obs['text_content'] = self.html_text_converter.handle(html_str)
                     # make observation serializable
-                    obs['screenshot'] = self.image_to_png_base64_url(obs['screenshot'])
+                    obs['set_of_marks'] = self.image_to_png_base64_url(
+                        overlay_som(
+                            obs['screenshot'], obs.get('extra_element_properties', {})
+                        ),
+                        add_data_prefix=True,
+                    )
+                    obs['screenshot'] = self.image_to_png_base64_url(
+                        obs['screenshot'], add_data_prefix=True
+                    )
                     obs['active_page_index'] = obs['active_page_index'].item()
                     obs['elapsed_time'] = obs['elapsed_time'].item()
                     self.browser_side.send((unique_request_id, obs))
@@ -157,7 +185,7 @@ class BrowserEnv:
                     pass
                 return
 
-    def step(self, action_str: str, timeout: float = 30) -> dict:
+    def step(self, action_str: str, timeout: float = 100) -> dict:
         """Execute an action in the browser environment and return the observation."""
         unique_request_id = str(uuid.uuid4())
         self.agent_side.send((unique_request_id, {'action': action_str}))

openhands/runtime/browser/utils.py

@@ -35,6 +35,10 @@ async def browse(
             content=obs['text_content'],  # text content of the page
             url=obs.get('url', ''),  # URL of the page
             screenshot=obs.get('screenshot', None),  # base64-encoded screenshot, png
+            set_of_marks=obs.get(
+                'set_of_marks', None
+            ),  # base64-encoded Set-of-Marks annotated screenshot, png,
+            goal_image_urls=obs.get('image_content', []),
             open_pages_urls=obs.get('open_pages_urls', []),  # list of open pages
             active_page_index=obs.get(
                 'active_page_index', -1

openhands/runtime/impl/docker/containers.py

@@ -1,18 +1,19 @@
 import docker
 
 
-def remove_all_containers(prefix: str):
+def stop_all_containers(prefix: str):
     docker_client = docker.from_env()
-
     try:
         containers = docker_client.containers.list(all=True)
         for container in containers:
             try:
                 if container.name.startswith(prefix):
-                    container.remove(force=True)
+                    container.stop()
             except docker.errors.APIError:
                 pass
             except docker.errors.NotFound:
                 pass
     except docker.errors.NotFound:  # yes, this can happen!
         pass
+    finally:
+        docker_client.close()

openhands/runtime/impl/docker/docker_runtime.py

@@ -5,6 +5,7 @@ from typing import Callable
 import docker
 import requests
 import tenacity
+from docker.models.containers import Container
 
 from openhands.core.config import AppConfig
 from openhands.core.exceptions import (
@@ -18,7 +19,7 @@ from openhands.runtime.builder import DockerRuntimeBuilder
 from openhands.runtime.impl.action_execution.action_execution_client import (
     ActionExecutionClient,
 )
-from openhands.runtime.impl.docker.containers import remove_all_containers
+from openhands.runtime.impl.docker.containers import stop_all_containers
 from openhands.runtime.plugins import PluginRequirement
 from openhands.runtime.utils import find_available_tcp_port
 from openhands.runtime.utils.command import get_action_execution_server_startup_command
@@ -35,8 +36,8 @@ APP_PORT_RANGE_1 = (50000, 54999)
 APP_PORT_RANGE_2 = (55000, 59999)
 
 
-def remove_all_runtime_containers():
-    remove_all_containers(CONTAINER_NAME_PREFIX)
+def stop_all_runtime_containers():
+    stop_all_containers(CONTAINER_NAME_PREFIX)
 
 
 _atexit_registered = False
@@ -68,7 +69,7 @@ class DockerRuntime(ActionExecutionClient):
         global _atexit_registered
         if not _atexit_registered:
             _atexit_registered = True
-            atexit.register(remove_all_runtime_containers)
+            atexit.register(stop_all_runtime_containers)
 
         self.config = config
         self._runtime_initialized: bool = False
@@ -85,7 +86,7 @@ class DockerRuntime(ActionExecutionClient):
         self.base_container_image = self.config.sandbox.base_container_image
         self.runtime_container_image = self.config.sandbox.runtime_container_image
         self.container_name = CONTAINER_NAME_PREFIX + sid
-        self.container = None
+        self.container: Container | None = None
 
         self.runtime_builder = DockerRuntimeBuilder(self.docker_client)
 
@@ -187,7 +188,6 @@ class DockerRuntime(ActionExecutionClient):
     def _init_container(self):
         self.log('debug', 'Preparing to start container...')
         self.send_status_message('STATUS$PREPARING_CONTAINER')
-
         self._host_port = self._find_available_port(EXECUTION_SERVER_PORT_RANGE)
         self._container_port = self._host_port
         self._vscode_port = self._find_available_port(VSCODE_PORT_RANGE)
@@ -287,7 +287,7 @@ class DockerRuntime(ActionExecutionClient):
                     'warning',
                     f'Container {self.container_name} already exists. Removing...',
                 )
-                remove_all_containers(self.container_name)
+                stop_all_containers(self.container_name)
                 return self._init_container()
 
             else:
@@ -308,20 +308,20 @@ class DockerRuntime(ActionExecutionClient):
 
     def _attach_to_container(self):
         self.container = self.docker_client.containers.get(self.container_name)
-        for port in self.container.attrs['NetworkSettings']['Ports']:  # type: ignore
-            port = int(port.split('/')[0])
-            if (
-                port >= EXECUTION_SERVER_PORT_RANGE[0]
-                and port <= EXECUTION_SERVER_PORT_RANGE[1]
-            ):
-                self._container_port = port
-            if port >= VSCODE_PORT_RANGE[0] and port <= VSCODE_PORT_RANGE[1]:
-                self._vscode_port = port
-            elif port >= APP_PORT_RANGE_1[0] and port <= APP_PORT_RANGE_1[1]:
-                self._app_ports.append(port)
-            elif port >= APP_PORT_RANGE_2[0] and port <= APP_PORT_RANGE_2[1]:
-                self._app_ports.append(port)
-        self._host_port = self._container_port
+        if self.container.status == 'exited':
+            self.container.start()
+        config = self.container.attrs['Config']
+        for env_var in config['Env']:
+            if env_var.startswith('port='):
+                self._host_port = int(env_var.split('port=')[1])
+                self._container_port = self._host_port
+            elif env_var.startswith('VSCODE_PORT='):
+                self._vscode_port = int(env_var.split('VSCODE_PORT=')[1])
+        self._app_ports = []
+        for exposed_port in config['ExposedPorts'].keys():
+            exposed_port = int(exposed_port.split('/tcp')[0])
+            if exposed_port != self._host_port and exposed_port != self._vscode_port:
+                self._app_ports.append(exposed_port)
         self.api_url = f'{self.config.sandbox.local_runtime_url}:{self._container_port}'
         self.log(
             'debug',
@@ -368,7 +368,7 @@ class DockerRuntime(ActionExecutionClient):
         close_prefix = (
             CONTAINER_NAME_PREFIX if rm_all_containers else self.container_name
         )
-        remove_all_containers(close_prefix)
+        stop_all_containers(close_prefix)
 
     def _is_port_in_use_docker(self, port):
         containers = self.docker_client.containers.list()
@@ -404,3 +404,17 @@ class DockerRuntime(ActionExecutionClient):
             hosts[f'http://localhost:{port}'] = port
 
         return hosts
+
+    @classmethod
+    async def delete(cls, conversation_id: str):
+        docker_client = cls._init_docker_client()
+        try:
+            container_name = CONTAINER_NAME_PREFIX + conversation_id
+            container = docker_client.containers.get(container_name)
+            container.remove(force=True)
+        except docker.errors.APIError:
+            pass
+        except docker.errors.NotFound:
+            pass
+        finally:
+            docker_client.close()

openhands/runtime/impl/modal/modal_runtime.py

@@ -40,6 +40,7 @@ class ModalRuntime(ActionExecutionClient):
 
     container_name_prefix = 'openhands-sandbox-'
     sandbox: modal.Sandbox | None
+    sid: str
 
     def __init__(
         self,
@@ -57,6 +58,7 @@ class ModalRuntime(ActionExecutionClient):
 
         self.config = config
         self.sandbox = None
+        self.sid = sid
 
         self.modal_client = modal.Client.from_credentials(
             config.modal_api_token_id.get_secret_value(),
@@ -75,6 +77,8 @@ class ModalRuntime(ActionExecutionClient):
 
         # This value is arbitrary as it's private to the container
         self.container_port = 3000
+        self._vscode_port = 4445
+        self._vscode_url: str | None = None
 
         self.status_callback = status_callback
         self.base_container_image_id = self.config.sandbox.base_container_image
@@ -140,6 +144,7 @@ class ModalRuntime(ActionExecutionClient):
 
         if not self.attach_to_existing:
             self.send_status_message(' ')
+        self._runtime_initialized = True
 
     def _get_action_execution_server_host(self):
         return self.api_url
@@ -208,6 +213,7 @@ echo 'export INPUTRC=/etc/inputrc' >> /etc/bash.bashrc
             environment: dict[str, str | None] = {
                 'port': str(self.container_port),
                 'PYTHONUNBUFFERED': '1',
+                'VSCODE_PORT': str(self._vscode_port),
             }
             if self.config.debug:
                 environment['DEBUG'] = 'true'
@@ -225,7 +231,7 @@ echo 'export INPUTRC=/etc/inputrc' >> /etc/bash.bashrc
                 *sandbox_start_cmd,
                 secrets=[env_secret],
                 workdir='/openhands/code',
-                encrypted_ports=[self.container_port],
+                encrypted_ports=[self.container_port, self._vscode_port],
                 image=self.image,
                 app=self.app,
                 client=self.modal_client,
@@ -248,3 +254,27 @@ echo 'export INPUTRC=/etc/inputrc' >> /etc/bash.bashrc
 
         if not self.attach_to_existing and self.sandbox:
             self.sandbox.terminate()
+
+    @property
+    def vscode_url(self) -> str | None:
+        if self._vscode_url is not None:  # cached value
+            self.log('debug', f'VSCode URL: {self._vscode_url}')
+            return self._vscode_url
+        token = super().get_vscode_token()
+        if not token:
+            self.log('error', 'VSCode token not found')
+            return None
+        if not self.sandbox:
+            self.log('error', 'Sandbox not initialized')
+            return None
+
+        tunnel = self.sandbox.tunnels()[self._vscode_port]
+        tunnel_url = tunnel.url
+        self._vscode_url = tunnel_url + f'/?tkn={token}&folder={self.config.workspace_mount_path_in_sandbox}'
+
+        self.log(
+            'debug',
+            f'VSCode URL: {self._vscode_url}',
+        )
+
+        return self._vscode_url

openhands/runtime/impl/remote/remote_runtime.py

@@ -31,6 +31,9 @@ class RemoteRuntime(ActionExecutionClient):
     """This runtime will connect to a remote oh-runtime-client."""
 
     port: int = 60000  # default port for the remote runtime client
+    runtime_id: str | None = None
+    runtime_url: str | None = None
+    _runtime_initialized: bool = False
 
     def __init__(
         self,
@@ -71,10 +74,7 @@ class RemoteRuntime(ActionExecutionClient):
             self.config.sandbox.api_key,
             self.session,
         )
-        self.runtime_id: str | None = None
-        self.runtime_url: str | None = None
         self.available_hosts: dict[str, int] = {}
-        self._runtime_initialized: bool = False
 
     def log(self, level: str, message: str) -> None:
         message = f'[runtime session_id={self.sid} runtime_id={self.runtime_id or "unknown"}] {message}'
@@ -306,6 +306,12 @@ class RemoteRuntime(ActionExecutionClient):
         assert 'pod_status' in runtime_data
         pod_status = runtime_data['pod_status'].lower()
         self.log('debug', f'Pod status: {pod_status}')
+        restart_count = runtime_data.get('restart_count', 0)
+        if restart_count != 0:
+            restart_reasons = runtime_data.get('restart_reasons')
+            self.log(
+                'debug', f'Pod restarts: {restart_count}, reasons: {restart_reasons}'
+            )
 
         # FIXME: We should fix it at the backend of /start endpoint, make sure
         # the pod is created before returning the response.
@@ -405,8 +411,13 @@ class RemoteRuntime(ActionExecutionClient):
                         f'Runtime is temporarily unavailable. This may be due to a restart or network issue, please try again. Original error: {e}'
                     ) from e
             elif e.response.status_code == 503:
-                self.log('warning', 'Runtime appears to be paused. Resuming...')
-                self._resume_runtime()
-                return super()._send_action_server_request(method, url, **kwargs)
+                if self.config.sandbox.keep_runtime_alive:
+                    self.log('warning', 'Runtime appears to be paused. Resuming...')
+                    self._resume_runtime()
+                    return super()._send_action_server_request(method, url, **kwargs)
+                else:
+                    raise AgentRuntimeDisconnectedError(
+                        f'Runtime is temporarily unavailable. This may be due to a restart or network issue, please try again. Original error: {e}'
+                    ) from e
             else:
                 raise e

openhands/runtime/utils/runtime_build.py

@@ -303,17 +303,12 @@ def truncate_hash(hash: str) -> str:
 
 def get_hash_for_lock_files(base_image: str):
     openhands_source_dir = Path(openhands.__file__).parent
-    project_root = openhands_source_dir.parent
     md5 = hashlib.md5()
     md5.update(base_image.encode())
     for file in ['pyproject.toml', 'poetry.lock']:
-        # First try project root
-        src = Path(project_root, file)
+        src = Path(openhands_source_dir, file)
         if not src.exists():
-            # Then try source dir
-            src = Path(openhands_source_dir, file)
-            if not src.exists():
-                raise FileNotFoundError(f"Could not find {file} in either {project_root} or {openhands_source_dir}")
+            src = Path(openhands_source_dir.parent, file)
         with open(src, 'rb') as f:
             for chunk in iter(lambda: f.read(4096), b''):
                 md5.update(chunk)

openhands/server/README.md

@@ -125,13 +125,13 @@ The `agent_session.py` file contains the `AgentSession` class, which manages the
 - Handling security analysis
 - Managing the event stream
 
-### 3. session/manager.py
+### 3. session/conversation_manager/conversation_manager.py
 
-The `manager.py` file defines the `SessionManager` class, which is responsible for managing multiple client sessions. Key features include:
+The `conversation_manager.py` file defines the `ConversationManager` class, which is responsible for managing multiple client conversations. Key features include:
 
-- Adding and restarting sessions
-- Sending messages to specific sessions
-- Cleaning up inactive sessions
+- Adding and restarting conversations
+- Sending messages to specific conversations
+- Cleaning up inactive conversations
 
 ### 4. listen.py
 
@@ -148,7 +148,7 @@ The `listen.py` file is the main server file that sets up the FastAPI applicatio
 1. **Server Initialization**:
    - The FastAPI application is created and configured in `listen.py`.
    - CORS middleware and static file serving are set up.
-   - The `SessionManager` is initialized.
+   - The `ConversationManager` is initialized.
 
 2. **Client Connection**:
    - When a client connects via WebSocket, a new `Session` is created or an existing one is restarted.
@@ -173,7 +173,7 @@ The `listen.py` file is the main server file that sets up the FastAPI applicatio
    - Security-related API requests are forwarded to the security analyzer.
 
 7. **Session Management**:
-   - The `SessionManager` periodically cleans up inactive sessions.
+   - The `ConversationManager` periodically cleans up inactive sessions.
    - It also handles sending messages to specific sessions when needed.
 
 8. **API Endpoints**:

openhands/server/app.py

@@ -9,13 +9,7 @@ from fastapi import (
 )
 
 import openhands.agenthub  # noqa F401 (we import this to get the agents registered)
-from openhands.server.middleware import (
-    AttachConversationMiddleware,
-    InMemoryRateLimiter,
-    LocalhostCORSMiddleware,
-    NoCacheMiddleware,
-    RateLimitMiddleware,
-)
+from openhands import __version__
 from openhands.server.routes.conversation import app as conversation_api_router
 from openhands.server.routes.feedback import app as feedback_api_router
 from openhands.server.routes.files import app as files_api_router
@@ -26,28 +20,23 @@ from openhands.server.routes.manage_conversations import (
 from openhands.server.routes.public import app as public_api_router
 from openhands.server.routes.security import app as security_api_router
 from openhands.server.routes.settings import app as settings_router
-from openhands.server.shared import openhands_config, session_manager
-from openhands.utils.import_utils import get_impl
+from openhands.server.routes.trajectory import app as trajectory_router
+from openhands.server.shared import conversation_manager, openhands_config
 
 
 @asynccontextmanager
 async def _lifespan(app: FastAPI):
-    async with session_manager:
+    async with conversation_manager:
         yield
 
 
-app = FastAPI(lifespan=_lifespan)
-app.add_middleware(
-    LocalhostCORSMiddleware,
-    allow_credentials=True,
-    allow_methods=['*'],
-    allow_headers=['*'],
-)
-
-app.add_middleware(NoCacheMiddleware)
-app.add_middleware(
-    RateLimitMiddleware, rate_limiter=InMemoryRateLimiter(requests=10, seconds=1)
+app = FastAPI(
+    title='OpenHands',
+    description='OpenHands: Code Less, Make More',
+    version=__version__,
+    lifespan=_lifespan,
 )
+openhands_config.attach_middleware(app)
 
 
 @app.get('/health')
@@ -63,8 +52,4 @@ app.include_router(conversation_api_router)
 app.include_router(manage_conversation_api_router)
 app.include_router(settings_router)
 app.include_router(github_api_router)
-
-AttachConversationMiddlewareImpl = get_impl(
-    AttachConversationMiddleware, openhands_config.attach_conversation_middleware_path
-)
-app.middleware('http')(AttachConversationMiddlewareImpl(app))
+app.include_router(trajectory_router)

openhands/server/auth.py

@@ -1,44 +1,5 @@
-import jwt
 from fastapi import Request
-from jwt.exceptions import InvalidTokenError
-
-from openhands.core.logger import openhands_logger as logger
 
 
 def get_user_id(request: Request) -> str | None:
     return getattr(request.state, 'github_user_id', None)
-
-
-def get_sid_from_token(token: str, jwt_secret: str) -> str:
-    """Retrieves the session id from a JWT token.
-
-    Parameters:
-        token (str): The JWT token from which the session id is to be extracted.
-
-    Returns:
-        str: The session id if found and valid, otherwise an empty string.
-    """
-    try:
-        # Decode the JWT using the specified secret and algorithm
-        payload = jwt.decode(token, jwt_secret, algorithms=['HS256'])
-
-        # Ensure the payload contains 'sid'
-        if 'sid' in payload:
-            return payload['sid']
-        else:
-            logger.error('SID not found in token')
-            return ''
-    except InvalidTokenError:
-        logger.error('Invalid token')
-    except Exception as e:
-        logger.exception('Unexpected error decoding token: %s', e)
-    return ''
-
-
-def sign_token(payload: dict[str, object], jwt_secret: str, algorithm='HS256') -> str:
-    """Signs a JWT token."""
-    # payload = {
-    #     "sid": sid,
-    #     # "exp": datetime.now(timezone.utc) + timedelta(minutes=15),
-    # }
-    return jwt.encode(payload, jwt_secret, algorithm=algorithm)