update sys prompt

Co-authored-by: amanape <83104063+amanape@users.noreply.github.com>

.github/workflows/ghcr-build.yml

@@ -286,7 +286,6 @@ jobs:
           image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
-          SKIP_CONTAINER_LOGS=true \
           TEST_RUNTIME=eventstream \
           SANDBOX_USER_ID=$(id -u) \
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
@@ -364,7 +363,6 @@ jobs:
           image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
-          SKIP_CONTAINER_LOGS=true \
           TEST_RUNTIME=eventstream \
           SANDBOX_USER_ID=$(id -u) \
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \

.github/workflows/openhands-resolver.yml

@@ -1,15 +1,267 @@
-name: Resolve Issues with OpenHands
+name: Auto-Fix Tagged Issue with OpenHands
 
 on:
+  workflow_call:
+    inputs:
+      max_iterations:
+        required: false
+        type: number
+        default: 50
+      macro:
+        required: false
+        type: string
+        default: "@openhands-agent"
+    secrets:
+      LLM_MODEL:
+        required: true
+      LLM_API_KEY:
+        required: true
+      LLM_BASE_URL:
+        required: false
+      PAT_TOKEN:
+        required: true
+      PAT_USERNAME:
+        required: true
+
   issues:
     types: [labeled]
   pull_request:
     types: [labeled]
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  pull_request_review:
+    types: [submitted]
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
 
 jobs:
-  call-openhands-resolver:
-    uses: All-Hands-AI/openhands-resolver/.github/workflows/openhands-resolver.yml@main
-    if: github.event.label.name == 'fix-me'
-    with:
-      max_iterations: 50
-    secrets: inherit
+
+  auto-fix:
+    if: |
+      github.event_name == 'workflow_call' ||
+      github.event.label.name == 'fix-me' ||
+      github.event.label.name == 'fix-me-experimental' ||
+
+      (
+        ((github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
+        startsWith(github.event.comment.body, inputs.macro || '@openhands-agent') &&
+        (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER')
+        ) ||
+
+        (github.event_name == 'pull_request_review' &&
+        startsWith(github.event.review.body, inputs.macro || '@openhands-agent') &&
+        (github.event.review.author_association == 'OWNER' || github.event.review.author_association == 'COLLABORATOR' || github.event.review.author_association == 'MEMBER')
+        )
+      )
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Get latest versions and create requirements.txt
+        run: |
+          python -m pip index versions openhands-ai > openhands_versions.txt
+          OPENHANDS_VERSION=$(head -n 1 openhands_versions.txt | awk '{print $2}' | tr -d '()')
+          echo "openhands-ai==${OPENHANDS_VERSION}" >> requirements.txt
+          cat requirements.txt
+
+      - name: Cache pip dependencies
+        if: github.event.label.name != 'fix-me-experimental'
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/*
+          key: ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
+
+      - name: Check required environment variables
+        env:
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
+          PAT_USERNAME: ${{ secrets.PAT_USERNAME }}
+        run: |
+          required_vars=("LLM_MODEL" "LLM_API_KEY" "PAT_TOKEN" "PAT_USERNAME")
+          for var in "${required_vars[@]}"; do
+            if [ -z "${!var}" ]; then
+              echo "Error: Required environment variable $var is not set."
+              exit 1
+            fi
+          done
+
+      - name: Set environment variables
+        run: |
+          if [ -n "${{ github.event.review.body }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          elif [ -n "${{ github.event.issue.pull_request }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          elif [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          else
+            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=issue" >> $GITHUB_ENV
+          fi
+
+          if [ -n "${{ github.event.review.body }}" ]; then
+            echo "COMMENT_ID=${{ github.event.review.id || 'None' }}" >> $GITHUB_ENV
+          else
+            echo "COMMENT_ID=${{ github.event.comment.id || 'None' }}" >> $GITHUB_ENV
+          fi
+
+          echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV
+          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
+
+      - name: Comment on issue with start message
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const issueType = process.env.ISSUE_TYPE;
+            github.rest.issues.createComment({
+              issue_number: ${{ env.ISSUE_NUMBER }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
+            });
+
+      - name: Install OpenHands
+        run: |
+          if [ "${{ github.event.label.name }}" == "fix-me-experimental" ]; then
+            python -m pip install --upgrade pip
+            pip install git+https://github.com/all-hands-ai/openhands.git
+          else
+            python -m pip install --upgrade -r requirements.txt
+          fi
+
+      - name: Attempt to resolve issue
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PYTHONPATH: ""
+        run: |
+          cd /tmp && python -m openhands.resolver.resolve_issue \
+            --repo ${{ github.repository }} \
+            --issue-number ${{ env.ISSUE_NUMBER }} \
+            --issue-type ${{ env.ISSUE_TYPE }} \
+            --max-iterations ${{ env.MAX_ITERATIONS }} \
+            --comment-id ${{ env.COMMENT_ID }}
+
+      - name: Check resolution result
+        id: check_result
+        run: |
+          if cd /tmp && grep -q '"success":true' output/output.jsonl; then
+            echo "RESOLUTION_SUCCESS=true" >> $GITHUB_OUTPUT
+          else
+            echo "RESOLUTION_SUCCESS=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload output.jsonl as artifact
+        uses: actions/upload-artifact@v4
+        if: always() # Upload even if the previous steps fail
+        with:
+          name: resolver-output
+          path: /tmp/output/output.jsonl
+          retention-days: 30 # Keep the artifact for 30 days
+
+      - name: Create draft PR or push branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PYTHONPATH: ""
+        run: |
+          if [ "${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}" == "true" ]; then
+            cd /tmp && python -m openhands.resolver.send_pull_request \
+              --issue-number ${{ env.ISSUE_NUMBER }} \
+              --pr-type draft | tee pr_result.txt && \
+              grep "draft created" pr_result.txt | sed 's/.*\///g' > pr_number.txt
+          else
+            cd /tmp && python -m openhands.resolver.send_pull_request \
+              --issue-number ${{ env.ISSUE_NUMBER }} \
+              --pr-type branch \
+              --send-on-failure | tee branch_result.txt && \
+              grep "branch created" branch_result.txt | sed 's/.*\///g; s/.expand=1//g' > branch_name.txt
+          fi
+
+      - name: Comment on issue
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const fs = require('fs');
+            const issueNumber = ${{ env.ISSUE_NUMBER }};
+            const success = ${{ steps.check_result.outputs.RESOLUTION_SUCCESS }};
+
+            let prNumber = '';
+            let branchName = '';
+            let logContent = '';
+            const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`;
+
+            try {
+              if (success){
+                logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim();
+              } else {
+                logContent = fs.readFileSync('/tmp/branch_result.txt', 'utf8').trim();
+              }
+            } catch (error) {
+              console.error('Error reading results file:', error);
+            }
+
+            try {
+              if (success) {
+                prNumber = fs.readFileSync('/tmp/pr_number.txt', 'utf8').trim();
+              } else {
+                branchName = fs.readFileSync('/tmp/branch_name.txt', 'utf8').trim();
+              }
+            } catch (error) {
+              console.error('Error reading file:', error);
+            }
+
+            if (logContent.includes(noChangesMessage)) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.`
+              });
+            } else if (success && prNumber) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `A potential fix has been generated and a draft PR #${prNumber} has been created. Please review the changes.`
+              });
+            } else if (!success && branchName) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`
+              });
+            } else {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.`
+              });
+            }

.gitignore

@@ -176,6 +176,9 @@ evaluation/gorilla/data
 evaluation/toolqa/data
 evaluation/scienceagentbench/benchmark
 
+# openhands resolver
+output/
+
 # frontend
 
 # dependencies

Development.md

@@ -100,7 +100,7 @@ poetry run pytest ./tests/unit/test_*.py
 ### 9. Use existing Docker image
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image. Follow these steps:
 1. Set the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
-2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.13-nikolaik
+2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.14-nikolaik
 
 ## Develop inside Docker container
 

README.md

@@ -38,16 +38,16 @@ See the [Installation](https://docs.all-hands.dev/modules/usage/installation) gu
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
 
 docker run -it --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     -e LOG_ALL_EVENTS=true \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.13
+    docker.all-hands.dev/all-hands-ai/openhands:0.14
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!

compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.13-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.13-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/modules/usage/how-to/cli-mode.md

@@ -50,7 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -59,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.13 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
     python -m openhands.core.cli
 ```
 

docs/modules/usage/how-to/headless-mode.md

@@ -44,7 +44,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -54,6 +54,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.13 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```

docs/modules/usage/installation.mdx

@@ -11,16 +11,16 @@
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     -e LOG_ALL_EVENTS=true \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.13
+    docker.all-hands.dev/all-hands-ai/openhands:0.14
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/modules/usage/llms/litellm-proxy.md

@@ -0,0 +1,20 @@
+# LiteLLM Proxy
+
+OpenHands supports using the [LiteLLM proxy](https://docs.litellm.ai/docs/proxy/quick_start) to access various LLM providers.
+
+## Configuration
+
+To use LiteLLM proxy with OpenHands, you need to:
+
+1. Set up a LiteLLM proxy server (see [LiteLLM documentation](https://docs.litellm.ai/docs/proxy/quick_start))
+2. When running OpenHands, you'll need to set the following in the OpenHands UI through the Settings:
+  * Enable `Advanced Options`
+  * `Custom Model` to the prefix `litellm_proxy/` + the model you will be using (e.g. `litellm_proxy/anthropic.claude-3-5-sonnet-20241022-v2:0`)
+  * `Base URL` to your LiteLLM proxy URL (e.g. `https://your-litellm-proxy.com`)
+  * `API Key` to your LiteLLM proxy API key
+
+## Supported Models
+
+The supported models depend on your LiteLLM proxy configuration. OpenHands supports any model that your LiteLLM proxy is configured to handle.
+
+Refer to your LiteLLM proxy configuration for the list of available models and their names.

docs/modules/usage/llms/llms.md

@@ -63,6 +63,7 @@ We have a few guides for running OpenHands with specific model providers:
 - [Azure](llms/azure-llms)
 - [Google](llms/google-llms)
 - [Groq](llms/groq)
+- [LiteLLM Proxy](llms/litellm-proxy)
 - [OpenAI](llms/openai-llms)
 - [OpenRouter](llms/openrouter)
 

docs/modules/usage/runtimes.md

@@ -59,7 +59,7 @@ docker run # ...
     -e RUNTIME=remote \
     -e SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.app.all-hands.dev" \
     -e SANDBOX_API_KEY="your-all-hands-api-key" \
-    -e SANDBOX_KEEP_REMOTE_RUNTIME_ALIVE="true" \
+    -e SANDBOX_KEEP_RUNTIME_ALIVE="true" \
     # ...
 ```
 

docs/sidebars.ts

@@ -76,6 +76,11 @@ const sidebars: SidebarsConfig = {
                   label: 'Groq',
                   id: 'usage/llms/groq',
                 },
+                {
+                  type: 'doc',
+                  label: 'LiteLLM Proxy',
+                  id: 'usage/llms/litellm-proxy',
+                },
                 {
                   type: 'doc',
                   label: 'OpenAI',

evaluation/EDA/game.py

@@ -87,9 +87,7 @@ class Q20Game:
         # others
         bingo, anwser_reply = self.judge_winner(response)
         if bingo:
-            return (
-                'You are bingo! quit now, run: <execute_bash> exit </execute_bash>.\n'
-            )
+            return 'You are bingo! Use the "finish" tool to finish the interaction.\n'
         if self.curr_turn == self.num_turns - 2:
             anwser_reply += " You must guess now, what's it?"
         return anwser_reply

evaluation/aider_bench/README.md

@@ -56,6 +56,20 @@ You can update the arguments in the script
 ./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
 ```
 
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+
+# Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
+export ALLHANDS_API_KEY="YOUR-API-KEY"
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+./evaluation/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
+```
+
 ## Summarize Results
 
 ```bash

evaluation/aider_bench/run_infer.py

@@ -58,6 +58,9 @@ def get_config(
             use_host_network=False,
             timeout=100,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_runtime_alive=False,
+            remote_runtime_init_timeout=1800,
         ),
         # do not mount workspace
         workspace_base=None,

evaluation/biocoder/run_infer.py

@@ -40,7 +40,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 FILE_EXT_MAP = {

evaluation/bird/run_infer.py

@@ -40,7 +40,7 @@ from openhands.utils.async_utils import call_async_from_sync
 def codeact_user_response(state: State) -> str:
     msg = (
         'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have completed the SQL, please run the following command: <execute_bash> exit </execute_bash>.\n'
+        'If you think you have completed the SQL, please finish the interaction using the "finish" tool.\n'
         'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
     )
     if state.history:
@@ -54,7 +54,7 @@ def codeact_user_response(state: State) -> str:
             # let the agent know that it can give up when it has tried 3 times
             return (
                 msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
+                + 'If you want to give up, use the "finish" tool to finish the interaction.\n'
             )
     return msg
 
@@ -64,7 +64,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 

evaluation/discoverybench/run_infer.py

@@ -55,7 +55,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 

evaluation/gorilla/run_infer.py

@@ -33,7 +33,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the request, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have completed the request, please finish the interaction using the "finish" tool.\n'
 }
 
 

evaluation/gpqa/run_infer.py

@@ -87,11 +87,10 @@ def gpqa_codeact_user_response(
     msg = (
         'Please continue working on the task on whatever approach you think is suitable.\n'
         'Feel free to use all tools for calculations and solving the problem, and web-search for finding relevant facts during the process if needed\n'
-        'If you have finished reporting the answer in the expected format, (and only once that is done), please run the following command to submit: <execute_bash> exit </execute_bash>.\n'
+        'If you have finished reporting the answer in the expected format, (and only once that is done), please use the "finish" tool to finish the interaction.\n'
         'Again you are being told a million times to first report the answer in the requested format (see again below for reference) before exiting. DO NOT EXIT WITHOUT REPORTING THE ANSWER FIRST.\n'
         'That is, when you have decided on the answer report in the following format:\n'
         f'{ACTION_FORMAT}\n'
-        '<execute_bash> exit </execute_bash>\n'
         'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP TO SOLVE THIS TASK.\n'
     )
     return msg
@@ -100,7 +99,7 @@ def gpqa_codeact_user_response(
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {'CodeActAgent': gpqa_codeact_user_response}
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': '\n\n SUPER IMPORTANT: When you think you have solved the question, first report it back to the user in the requested format. Only once that is done, in the next turn, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': '\n\n SUPER IMPORTANT: When you think you have solved the question, first report it back to the user in the requested format. Only once that is done, in the next turn, please finish the interaction using the "finish" tool.\n'
 }
 
 
@@ -205,12 +204,11 @@ Additional Instructions:
 - Do not try to solve the question in a single step. Break it down into smaller steps.
 - You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.
 
-- SUPER IMPORTANT: When you have reported the answer to the user in the requested format, (and only once that is done) in the next turn, please run the following command: <execute_bash> exit </execute_bash>.
+- SUPER IMPORTANT: When you have reported the answer to the user in the requested format, (and only once that is done) in the next turn, please finish the interaction using the "finish" tool.
 - Again you are being told a million times to first report the answer in the requested format (see again below for reference) before exiting. DO NOT EXIT WITHOUT REPORTING THE ANSWER FIRST.
     That is, when you have decided on the answer report in the following format:
 
 {ACTION_FORMAT}
-<execute_bash> exit </execute_bash>
 
 Again do not quit without reporting the answer first.
 Ok now its time to start solving the question. Good luck!

evaluation/humanevalfix/README.md

@@ -23,7 +23,7 @@ For each problem, OpenHands is given a set number of iterations to fix the faili
 ```
 {
     "task_id": "Python/2",
-    "instruction": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
+    "instruction": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
     "metadata": {
         "agent_class": "CodeActAgent",
         "model_name": "gpt-4",
@@ -38,10 +38,10 @@ For each problem, OpenHands is given a set number of iterations to fix the faili
                 "id": 27,
                 "timestamp": "2024-05-22T20:57:24.688651",
                 "source": "user",
-                "message": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
+                "message": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
                 "action": "message",
                 "args": {
-                    "content": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
+                    "content": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
                     "wait_for_response": false
                 }
             },

evaluation/humanevalfix/run_infer.py

@@ -75,7 +75,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 

evaluation/miniwob/README.md

@@ -16,6 +16,20 @@ Access with browser the above MiniWoB URLs and see if they load correctly.
 ./evaluation/miniwob/scripts/run_infer.sh llm.claude-35-sonnet-eval
 ```
 
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+./evaluation/miniwob/scripts/run_infer.sh [model_config] [git-version] [agent] [note] [eval_limit] [num_workers]
+
+# Example - This runs evaluation on BrowsingAgent for 125 instances on miniwob, with 2 workers running in parallel
+export ALLHANDS_API_KEY="YOUR-API-KEY"
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+./evaluation/miniwob/scripts/run_infer.sh llm.eval HEAD BrowsingAgent "" 125 2
+```
+
 Results will be in `evaluation/evaluation_outputs/outputs/miniwob/`
 
 To calculate the average reward, run:

evaluation/miniwob/get_avg_reward.py

@@ -23,7 +23,7 @@ if __name__ == '__main__':
             data = json.loads(line)
             actual_num += 1
             total_cost += data['metrics']['accumulated_cost']
-            total_reward += data['test_result']
+            total_reward += data['test_result']['reward']
 
     avg_reward = total_reward / total_num
     print('Avg Reward: ', avg_reward)

evaluation/miniwob/run_infer.py

@@ -47,6 +47,7 @@ SUPPORTED_AGENT_CLS = {'BrowsingAgent', 'CodeActAgent'}
 
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
     'CodeActAgent': codeact_user_response,
+    'BrowsingAgent': 'Continue the task. IMPORTANT: do not talk to the user until you have finished the task',
 }
 
 
@@ -66,7 +67,9 @@ def get_config(
             browsergym_eval_env=env_id,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_remote_runtime_alive=False,
+            remote_runtime_init_timeout=1800,
+            keep_runtime_alive=False,
+            timeout=120,
         ),
         # do not mount workspace
         workspace_base=None,

evaluation/miniwob/scripts/run_infer.sh

@@ -33,7 +33,7 @@ echo "MODEL_CONFIG: $MODEL_CONFIG"
 
 EVAL_NOTE="${AGENT_VERSION}_${NOTE}"
 
-COMMAND="poetry run python evaluation/miniwob/run_infer.py \
+COMMAND="export PYTHONPATH=evaluation/miniwob:\$PYTHONPATH && poetry run python evaluation/miniwob/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 10 \

evaluation/mint/run_infer.py

@@ -70,7 +70,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': '\nIMPORTANT: When your answer is confirmed by the user to be correct, you can exit using the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'IMPORTANT: When your answer is confirmed by the user to be correct, you can use the "finish" tool to finish the interaction.\n'
 }
 
 with open(os.path.join(os.path.dirname(__file__), 'requirements.txt'), 'r') as f:

evaluation/ml_bench/README.md

@@ -55,7 +55,7 @@ Here's an example of the evaluation output for a single task instance:
 {
   "instance_id": 3,
   "repo": "https://github.com/dmlc/dgl",
-  "instruction": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
+  "instruction": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
   "metadata": {
     "agent_class": "CodeActAgent",
     "model_name": "gpt-4-1106-preview",
@@ -70,10 +70,10 @@ Here's an example of the evaluation output for a single task instance:
         "id": 0,
         "timestamp": "2024-05-26T17:40:41.060009",
         "source": "user",
-        "message": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
+        "message": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
         "action": "message",
         "args": {
-          "content": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
+          "content": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
           "wait_for_response": false
         }
       },

evaluation/ml_bench/run_infer.py

@@ -52,7 +52,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have completed the task, please finish the interaction using the "finish" tool.\n'
 }
 
 ID2CONDA = {

evaluation/scienceagentbench/run_infer.py

@@ -72,7 +72,7 @@ def get_config(
             timeout=300,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_remote_runtime_alive=False,
+            keep_runtime_alive=False,
         ),
         # do not mount workspace
         workspace_base=None,

evaluation/swe_bench/eval_infer.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import time
+from functools import partial
 
 import pandas as pd
 from swebench.harness.grading import get_eval_report
@@ -83,7 +84,7 @@ def get_config(instance: pd.Series) -> AppConfig:
             timeout=1800,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            remote_runtime_init_timeout=1800,
+            remote_runtime_init_timeout=3600,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -94,13 +95,28 @@ def get_config(instance: pd.Series) -> AppConfig:
 
 def process_instance(
     instance: pd.Series,
-    metadata: EvalMetadata | None = None,
+    metadata: EvalMetadata,
     reset_logger: bool = True,
+    log_dir: str | None = None,
 ) -> EvalOutput:
+    """
+    Evaluate agent performance on a SWE-bench problem instance.
+
+    Note that this signature differs from the expected input to `run_evaluation`. Use
+    `functools.partial` to provide optional arguments before passing to the evaluation harness.
+
+    Args:
+        log_dir (str | None, default=None): Path to directory where log files will be written. Must
+        be provided if `reset_logger` is set.
+
+    Raises:
+        AssertionError: if the `reset_logger` flag is set without a provided log directory.
+    """
     # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
     if reset_logger:
-        global output_file
-        log_dir = output_file.replace('.jsonl', '.logs')
+        assert (
+            log_dir is not None
+        ), "Can't reset logger without a provided log directory."
         os.makedirs(log_dir, exist_ok=True)
         reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
     else:
@@ -127,6 +143,7 @@ def process_instance(
         return EvalOutput(
             instance_id=instance_id,
             test_result=instance['test_result'],
+            metadata=metadata,
         )
 
     runtime = create_runtime(config)
@@ -176,6 +193,7 @@ def process_instance(
             return EvalOutput(
                 instance_id=instance_id,
                 test_result=instance['test_result'],
+                metadata=metadata,
             )
         elif 'APPLY_PATCH_PASS' in apply_patch_output:
             logger.info(f'[{instance_id}] {APPLY_PATCH_PASS}:\n{apply_patch_output}')
@@ -245,23 +263,29 @@ def process_instance(
                         test_output_path = os.path.join(log_dir, 'test_output.txt')
                         with open(test_output_path, 'w') as f:
                             f.write(test_output)
-
-                        _report = get_eval_report(
-                            test_spec=test_spec,
-                            prediction={
-                                'model_patch': model_patch,
-                                'instance_id': instance_id,
-                            },
-                            log_path=test_output_path,
-                            include_tests_status=True,
-                        )
-                        report = _report[instance_id]
-                        logger.info(
-                            f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}"
-                        )
-                        instance['test_result']['report']['resolved'] = report[
-                            'resolved'
-                        ]
+                        try:
+                            _report = get_eval_report(
+                                test_spec=test_spec,
+                                prediction={
+                                    'model_patch': model_patch,
+                                    'instance_id': instance_id,
+                                },
+                                log_path=test_output_path,
+                                include_tests_status=True,
+                            )
+                            report = _report[instance_id]
+                            logger.info(
+                                f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}"
+                            )
+                            instance['test_result']['report']['resolved'] = report[
+                                'resolved'
+                            ]
+                        except Exception as e:
+                            logger.error(
+                                f'[{instance_id}] Error when getting eval report: {e}'
+                            )
+                            instance['test_result']['report']['resolved'] = False
+                            instance['test_result']['report']['error_eval'] = True
             else:
                 logger.info(f'[{instance_id}] Error when starting eval:\n{obs.content}')
                 instance['test_result']['report']['error_eval'] = True
@@ -269,6 +293,7 @@ def process_instance(
             return EvalOutput(
                 instance_id=instance_id,
                 test_result=instance['test_result'],
+                metadata=metadata,
             )
         else:
             logger.info(
@@ -336,7 +361,7 @@ if __name__ == '__main__':
 
     if 'model_patch' not in predictions.columns:
         predictions['model_patch'] = predictions['test_result'].apply(
-            lambda x: x['git_patch']
+            lambda x: x.get('git_patch', '')
         )
     assert {'instance_id', 'model_patch'}.issubset(
         set(predictions.columns)
@@ -355,12 +380,26 @@ if __name__ == '__main__':
     output_file = args.input_file.replace('.jsonl', '.swebench_eval.jsonl')
     instances = prepare_dataset(predictions, output_file, args.eval_n_limit)
 
+    # If possible, load the relevant metadata to avoid issues with `run_evaluation`.
+    metadata: EvalMetadata | None = None
+    metadata_filepath = os.path.join(os.path.dirname(args.input_file), 'metadata.json')
+    if os.path.exists(metadata_filepath):
+        with open(metadata_filepath, 'r') as metadata_file:
+            data = metadata_file.read()
+            metadata = EvalMetadata.model_validate_json(data)
+
+    # The evaluation harness constrains the signature of `process_instance_func` but we need to
+    # pass extra information. Build a new function object to avoid issues with multiprocessing.
+    process_instance_func = partial(
+        process_instance, log_dir=output_file.replace('.jsonl', '.logs')
+    )
+
     run_evaluation(
         instances,
-        metadata=None,
+        metadata=metadata,
         output_file=output_file,
         num_workers=args.eval_num_workers,
-        process_instance_func=process_instance,
+        process_instance_func=process_instance_func,
     )
 
     # Load evaluated predictions & print number of resolved predictions

evaluation/swe_bench/prompt.py

@@ -1,6 +1,6 @@
 CODEACT_SWE_PROMPT = """Now, you're going to solve this issue on your own. Your terminal session has started and you're in the repository's root directory. You can use any bash commands or the special interface to help you. Edit all the files you need to and run any checks or tests that you want.
 Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command.
-When you're satisfied with all of the changes you've made, you can run the following command: <execute_bash> exit </execute_bash>.
+When you're satisfied with all of the changes you've made, you can use the "finish" tool to finish the interaction.
 Note however that you cannot use any interactive session commands (e.g. vim) in this environment, but you can write scripts and run them. E.g. you can write a python script and then run it with `python <script_name>.py`.
 
 NOTE ABOUT THE EDIT COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate indentation before each line!

evaluation/swe_bench/run_infer.py

@@ -36,8 +36,8 @@ from openhands.events.action import CmdRunAction, MessageAction
 from openhands.events.observation import CmdOutputObservation, ErrorObservation
 from openhands.events.serialization.event import event_to_dict
 from openhands.runtime.base import Runtime
-from openhands.runtime.utils.shutdown_listener import sleep_if_should_continue
 from openhands.utils.async_utils import call_async_from_sync
+from openhands.utils.shutdown_listener import sleep_if_should_continue
 
 USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
 USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
@@ -146,7 +146,7 @@ def get_config(
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
             keep_remote_runtime_alive=False,
-            remote_runtime_init_timeout=1800,
+            remote_runtime_init_timeout=3600,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -534,5 +534,10 @@ if __name__ == '__main__':
             instances[col] = instances[col].apply(lambda x: str(x))
 
     run_evaluation(
-        instances, metadata, output_file, args.eval_num_workers, process_instance
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+        timeout_seconds=120 * 60,  # 2 hour PER instance should be more than enough
     )

evaluation/toolqa/run_infer.py

@@ -34,7 +34,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the request, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have completed the request, please finish the interaction using the "finish" tool.\n'
 }
 
 

evaluation/utils/shared.py

@@ -137,7 +137,7 @@ def codeact_user_response(
             # let the agent know that it can give up when it has tried 3 times
             return (
                 msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
+                + 'If you want to give up, use the "finish" tool to finish the interaction.\n'
             )
     return msg
 
@@ -346,6 +346,7 @@ def run_evaluation(
             f'model {metadata.llm_config.model}, max iterations {metadata.max_iterations}.\n'
         )
     else:
+        logger.warning('Running evaluation without metadata.')
         logger.info(f'Evaluation started with {num_workers} workers.')
 
     total_instances = len(dataset)

frontend/__tests__/clear-session.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { clearSession } from "../src/utils/clear-session";
+import store from "../src/store";
+import { initialState as browserInitialState } from "../src/state/browserSlice";
+
+describe("clearSession", () => {
+  beforeEach(() => {
+    // Mock localStorage
+    const localStorageMock = {
+      getItem: vi.fn(),
+      setItem: vi.fn(),
+      removeItem: vi.fn(),
+      clear: vi.fn(),
+    };
+    vi.stubGlobal("localStorage", localStorageMock);
+
+    // Set initial browser state to non-default values
+    store.dispatch({
+      type: "browser/setUrl",
+      payload: "https://example.com",
+    });
+    store.dispatch({
+      type: "browser/setScreenshotSrc",
+      payload: "base64screenshot",
+    });
+  });
+
+  it("should clear localStorage and reset browser state", () => {
+    clearSession();
+
+    // Verify localStorage items were removed
+    expect(localStorage.removeItem).toHaveBeenCalledWith("token");
+    expect(localStorage.removeItem).toHaveBeenCalledWith("repo");
+
+    // Verify browser state was reset
+    const state = store.getState();
+    expect(state.browser.url).toBe(browserInitialState.url);
+    expect(state.browser.screenshotSrc).toBe(browserInitialState.screenshotSrc);
+  });
+});

frontend/__tests__/components/chat/chat-interface.test.tsx

@@ -21,6 +21,11 @@ describe("Empty state", () => {
   }));
 
   beforeAll(() => {
+    vi.mock("@remix-run/react", async (importActual) => ({
+      ...(await importActual<typeof import("@remix-run/react")>()),
+      useRouteLoaderData: vi.fn(() => ({})),
+    }));
+
     vi.mock("#/context/socket", async (importActual) => ({
       ...(await importActual<typeof import("#/context/ws-client-provider")>()),
       useWsClient: useWsClientMock,

frontend/__tests__/hooks/use-rate.test.ts

@@ -0,0 +1,93 @@
+import { act, renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { useRate } from "#/utils/use-rate";
+
+describe("useRate", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("should initialize", () => {
+    const { result } = renderHook(() => useRate());
+
+    expect(result.current.items).toHaveLength(0);
+    expect(result.current.rate).toBeNull();
+    expect(result.current.lastUpdated).toBeNull();
+    expect(result.current.isUnderThreshold).toBe(true);
+  });
+
+  it("should handle the case of a single element", () => {
+    const { result } = renderHook(() => useRate());
+
+    act(() => {
+      result.current.record(123);
+    });
+
+    expect(result.current.items).toHaveLength(1);
+    expect(result.current.lastUpdated).not.toBeNull();
+  });
+
+  it("should return the difference between the last two elements", () => {
+    const { result } = renderHook(() => useRate());
+
+    vi.setSystemTime(500);
+    act(() => {
+      result.current.record(4);
+    });
+
+    vi.advanceTimersByTime(500);
+    act(() => {
+      result.current.record(9);
+    });
+
+    expect(result.current.items).toHaveLength(2);
+    expect(result.current.rate).toBe(5);
+    expect(result.current.lastUpdated).toBe(1000);
+  });
+
+  it("should update isUnderThreshold after [threshold]ms of no activity", () => {
+    const { result } = renderHook(() => useRate({ threshold: 500 }));
+
+    expect(result.current.isUnderThreshold).toBe(true);
+
+    act(() => {
+       // not sure if fake timers is buggy with intervals,
+       // but I need to call it twice to register
+      vi.advanceTimersToNextTimer();
+      vi.advanceTimersToNextTimer();
+    });
+
+    expect(result.current.isUnderThreshold).toBe(false);
+  });
+
+  it("should return an isUnderThreshold boolean", () => {
+    const { result } = renderHook(() => useRate({ threshold: 500 }));
+
+    vi.setSystemTime(500);
+    act(() => {
+      result.current.record(400);
+    });
+    act(() => {
+      result.current.record(1000);
+    });
+
+    expect(result.current.isUnderThreshold).toBe(false);
+
+    act(() => {
+      result.current.record(1500);
+    });
+
+    expect(result.current.isUnderThreshold).toBe(true);
+
+    act(() => {
+      vi.advanceTimersToNextTimer();
+      vi.advanceTimersToNextTimer();
+    });
+
+    expect(result.current.isUnderThreshold).toBe(false);
+  });
+});

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "openhands-frontend",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "openhands-frontend",
-      "version": "0.13.0",
+      "version": "0.14.0",
       "dependencies": {
         "@monaco-editor/react": "^4.6.0",
         "@nextui-org/react": "^2.4.8",
@@ -26,7 +26,7 @@
         "isbot": "^5.1.17",
         "jose": "^5.9.4",
         "monaco-editor": "^0.52.0",
-        "posthog-js": "^1.176.0",
+        "posthog-js": "^1.184.1",
         "react": "^18.3.1",
         "react-dom": "^18.3.1",
         "react-highlight": "^0.15.0",
@@ -19749,9 +19749,9 @@
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
     "node_modules/posthog-js": {
-      "version": "1.176.0",
-      "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.176.0.tgz",
-      "integrity": "sha512-T5XKNtRzp7q6CGb7Vc7wAI76rWap9fiuDUPxPsyPBPDkreKya91x9RIsSapAVFafwD1AEin1QMczCmt9Le9BWw==",
+      "version": "1.184.1",
+      "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.184.1.tgz",
+      "integrity": "sha512-q/1Kdard5SZnL2smrzeKcD+RuUi2PnbidiN4D3ThK20bNrhy5Z2heIy9SnRMvEiARY5lcQ7zxmDCAKPBKGSOtQ==",
       "dependencies": {
         "core-js": "^3.38.1",
         "fflate": "^0.4.8",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "private": true,
   "type": "module",
   "engines": {
@@ -25,7 +25,7 @@
     "isbot": "^5.1.17",
     "jose": "^5.9.4",
     "monaco-editor": "^0.52.0",
-    "posthog-js": "^1.176.0",
+    "posthog-js": "^1.184.1",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-highlight": "^0.15.0",

frontend/public/config.json

@@ -1,4 +1,5 @@
 {
   "APP_MODE": "oss",
-  "GITHUB_CLIENT_ID": ""
-}
+  "GITHUB_CLIENT_ID": "",
+  "POSTHOG_CLIENT_KEY": "phc_3ESMmY9SgqEAGBB6sMGK5ayYHkeUuknH2vP6FmWH9RA"
+}

frontend/src/api/open-hands.ts

@@ -8,6 +8,7 @@ import {
   GitHubAccessTokenResponse,
   ErrorResponse,
   GetConfigResponse,
+  GetVSCodeUrlResponse,
 } from "./open-hands.types";
 
 class OpenHands {
@@ -174,6 +175,21 @@ class OpenHands {
       true,
     );
   }
+
+  /**
+   * Get the VSCode URL
+   * @returns VSCode URL
+   */
+  static async getVSCodeUrl(): Promise<GetVSCodeUrlResponse> {
+    return request(`/api/vscode-url`, {}, false, false, 1);
+  }
+
+  static async getRuntimeId(): Promise<{ runtime_id: string }> {
+    const response = await request("/api/config");
+    const data = await response.json();
+
+    return data;
+  }
 }
 
 export default OpenHands;

frontend/src/api/open-hands.types.ts

@@ -43,5 +43,11 @@ export interface Feedback {
 
 export interface GetConfigResponse {
   APP_MODE: "saas" | "oss";
-  GITHUB_CLIENT_ID: string | null;
+  GITHUB_CLIENT_ID: string;
+  POSTHOG_CLIENT_KEY: string;
+}
+
+export interface GetVSCodeUrlResponse {
+  vscode_url: string | null;
+  error?: string;
 }

frontend/src/assets/vscode-alt.svg

@@ -0,0 +1,57 @@
+<svg width="100" height="100" viewBox="0 0 100 100" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0)">
+<g filter="url(#filter0_d)">
+<mask id="mask0" mask-type="alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="100" height="100">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M70.9119 99.5723C72.4869 100.189 74.2828 100.15 75.8725 99.3807L96.4604 89.4231C98.624 88.3771 100 86.1762 100 83.7616V16.2392C100 13.8247 98.624 11.6238 96.4604 10.5774L75.8725 0.619067C73.7862 -0.389991 71.3446 -0.142885 69.5135 1.19527C69.252 1.38636 69.0028 1.59985 68.769 1.83502L29.3551 37.9795L12.1872 24.88C10.5891 23.6607 8.35365 23.7606 6.86938 25.1178L1.36302 30.1525C-0.452603 31.8127 -0.454583 34.6837 1.35854 36.3466L16.2471 50.0001L1.35854 63.6536C-0.454583 65.3164 -0.452603 68.1876 1.36302 69.8477L6.86938 74.8824C8.35365 76.2395 10.5891 76.34 12.1872 75.1201L29.3551 62.0207L68.769 98.1651C69.3925 98.7923 70.1246 99.2645 70.9119 99.5723ZM75.0152 27.1813L45.1092 50.0001L75.0152 72.8189V27.1813Z" fill="white"/>
+</mask>
+<g mask="url(#mask0)">
+<path d="M96.4614 10.593L75.8567 0.62085C73.4717 -0.533437 70.6215 -0.0465506 68.7498 1.83492L1.29834 63.6535C-0.515935 65.3164 -0.513852 68.1875 1.30281 69.8476L6.8125 74.8823C8.29771 76.2395 10.5345 76.339 12.1335 75.1201L93.3604 13.18C96.0854 11.102 100 13.0557 100 16.4939V16.2535C100 13.84 98.6239 11.64 96.4614 10.593Z" fill="#D9D9D9"/>
+<g filter="url(#filter1_d)">
+<path d="M96.4614 89.4074L75.8567 99.3797C73.4717 100.534 70.6215 100.047 68.7498 98.1651L1.29834 36.3464C-0.515935 34.6837 -0.513852 31.8125 1.30281 30.1524L6.8125 25.1177C8.29771 23.7605 10.5345 23.6606 12.1335 24.88L93.3604 86.8201C96.0854 88.8985 100 86.9447 100 83.5061V83.747C100 86.1604 98.6239 88.3603 96.4614 89.4074Z" fill="#E6E6E6"/>
+</g>
+<g filter="url(#filter2_d)">
+<path d="M75.8578 99.3807C73.4721 100.535 70.6219 100.047 68.75 98.1651C71.0564 100.483 75 98.8415 75 95.5631V4.43709C75 1.15852 71.0565 -0.483493 68.75 1.83492C70.6219 -0.0467614 73.4721 -0.534276 75.8578 0.618963L96.4583 10.5773C98.6229 11.6237 100 13.8246 100 16.2391V83.7616C100 86.1762 98.6229 88.3761 96.4583 89.4231L75.8578 99.3807Z" fill="white"/>
+</g>
+<g style="mix-blend-mode:overlay" opacity="0.25">
+<path style="mix-blend-mode:overlay" opacity="0.25" fill-rule="evenodd" clip-rule="evenodd" d="M70.8508 99.5723C72.4258 100.189 74.2218 100.15 75.8115 99.3807L96.4 89.4231C98.5635 88.3771 99.9386 86.1762 99.9386 83.7616V16.2391C99.9386 13.8247 98.5635 11.6239 96.4 10.5774L75.8115 0.618974C73.7252 -0.390085 71.2835 -0.142871 69.4525 1.19518C69.1909 1.38637 68.9418 1.59976 68.7079 1.83493L29.2941 37.9795L12.1261 24.88C10.528 23.6606 8.2926 23.7605 6.80833 25.1177L1.30198 30.1524C-0.51354 31.8126 -0.515625 34.6837 1.2975 36.3465L16.186 50L1.2975 63.6536C-0.515625 65.3164 -0.51354 68.1875 1.30198 69.8476L6.80833 74.8824C8.2926 76.2395 10.528 76.339 12.1261 75.1201L29.2941 62.0207L68.7079 98.1651C69.3315 98.7923 70.0635 99.2645 70.8508 99.5723ZM74.9542 27.1812L45.0481 50L74.9542 72.8188V27.1812Z" fill="url(#paint0_linear)"/>
+</g>
+</g>
+</g>
+</g>
+<defs>
+<filter id="filter0_d" x="-6.25" y="-4.16667" width="112.5" height="112.5" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset dy="2.08333"/>
+<feGaussianBlur stdDeviation="3.125"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.15 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<filter id="filter1_d" x="-8.39436" y="15.6951" width="116.728" height="92.6376" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="4.16667"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/>
+<feBlend mode="overlay" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<filter id="filter2_d" x="60.4167" y="-8.33346" width="47.9167" height="116.667" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="4.16667"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/>
+<feBlend mode="overlay" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<linearGradient id="paint0_linear" x1="49.939" y1="-5.19792e-05" x2="49.939" y2="100.001" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="white" stop-opacity="0"/>
+</linearGradient>
+<clipPath id="clip0">
+<rect width="100" height="100" fill="white"/>
+</clipPath>
+</defs>
+</svg>

frontend/src/components/chat-input.tsx

@@ -18,6 +18,7 @@ interface ChatInputProps {
   onBlur?: () => void;
   onImagePaste?: (files: File[]) => void;
   className?: React.HTMLAttributes<HTMLDivElement>["className"];
+  buttonClassName?: React.HTMLAttributes<HTMLButtonElement>["className"];
 }
 
 export function ChatInput({
@@ -35,6 +36,7 @@ export function ChatInput({
   onBlur,
   onImagePaste,
   className,
+  buttonClassName,
 }: ChatInputProps) {
   const textareaRef = React.useRef<HTMLTextAreaElement>(null);
   const [isDraggingOver, setIsDraggingOver] = React.useState(false);
@@ -100,7 +102,7 @@ export function ChatInput({
   return (
     <div
       data-testid="chat-input"
-      className="flex items-end justify-end grow gap-1 min-h-6"
+      className="flex items-end justify-end grow gap-1 min-h-6 w-full"
     >
       <TextareaAutosize
         ref={textareaRef}
@@ -128,7 +130,7 @@ export function ChatInput({
         )}
       />
       {showButton && (
-        <>
+        <div className={buttonClassName}>
           {button === "submit" && (
             <button
               aria-label="Send"
@@ -152,7 +154,7 @@ export function ChatInput({
               <div className="w-[10px] h-[10px] bg-white" />
             </button>
           )}
-        </>
+        </div>
       )}
     </div>
   );

frontend/src/components/chat-interface.tsx

@@ -1,6 +1,7 @@
 import { useDispatch, useSelector } from "react-redux";
 import React from "react";
 import posthog from "posthog-js";
+import { useRouteLoaderData } from "@remix-run/react";
 import { convertImageToBase64 } from "#/utils/convert-image-to-base-64";
 import { ChatMessage } from "./chat-message";
 import { FeedbackActions } from "./feedback-actions";
@@ -21,18 +22,27 @@ import { ScrollToBottomButton } from "./scroll-to-bottom-button";
 import { Suggestions } from "./suggestions";
 import { SUGGESTIONS } from "#/utils/suggestions";
 import BuildIt from "#/icons/build-it.svg?react";
-import { useWsClient } from "#/context/ws-client-provider";
+import {
+  useWsClient,
+  WsClientProviderStatus,
+} from "#/context/ws-client-provider";
+import OpenHands from "#/api/open-hands";
+import { clientLoader } from "#/routes/_oh";
+import { downloadWorkspace } from "#/utils/download-workspace";
+import { SuggestionItem } from "./suggestion-item";
 
 const isErrorMessage = (
   message: Message | ErrorMessage,
 ): message is ErrorMessage => "error" in message;
 
 export function ChatInterface() {
-  const { send } = useWsClient();
+  const { send, status, isLoadingMessages } = useWsClient();
+
   const dispatch = useDispatch();
   const scrollRef = React.useRef<HTMLDivElement>(null);
   const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
     useScrollToBottom(scrollRef);
+  const rootLoaderData = useRouteLoaderData<typeof clientLoader>("routes/_oh");
 
   const { messages } = useSelector((state: RootState) => state.chat);
   const { curAgentState } = useSelector((state: RootState) => state.agent);
@@ -42,6 +52,24 @@ export function ChatInterface() {
   >("positive");
   const [feedbackModalIsOpen, setFeedbackModalIsOpen] = React.useState(false);
   const [messageToSend, setMessageToSend] = React.useState<string | null>(null);
+  const [isDownloading, setIsDownloading] = React.useState(false);
+
+  React.useEffect(() => {
+    if (status === WsClientProviderStatus.ACTIVE) {
+      try {
+        OpenHands.getRuntimeId().then(({ runtime_id }) => {
+          // eslint-disable-next-line no-console
+          console.log(
+            "Runtime ID: %c%s",
+            "background: #444; color: #ffeb3b; font-weight: bold; padding: 2px 4px; border-radius: 4px;",
+            runtime_id,
+          );
+        });
+      } catch (e) {
+        console.warn("Runtime ID not available in this environment");
+      }
+    }
+  }, [status]);
 
   const handleSendMessage = async (content: string, files: File[]) => {
     posthog.capture("user_message_sent", {
@@ -72,6 +100,17 @@ export function ChatInterface() {
     setFeedbackPolarity(polarity);
   };
 
+  const handleDownloadWorkspace = async () => {
+    setIsDownloading(true);
+    try {
+      await downloadWorkspace();
+    } catch (error) {
+      // TODO: Handle error
+    } finally {
+      setIsDownloading(false);
+    }
+  };
+
   return (
     <div className="h-full flex flex-col justify-between">
       {messages.length === 0 && (
@@ -101,29 +140,64 @@ export function ChatInterface() {
         onScroll={(e) => onChatBodyScroll(e.currentTarget)}
         className="flex flex-col grow overflow-y-auto overflow-x-hidden px-4 pt-4 gap-2"
       >
-        {messages.map((message, index) =>
-          isErrorMessage(message) ? (
-            <ErrorMessage
-              key={index}
-              id={message.id}
-              message={message.message}
-            />
-          ) : (
-            <ChatMessage
-              key={index}
-              type={message.sender}
-              message={message.content}
-            >
-              {message.imageUrls.length > 0 && (
-                <ImageCarousel size="small" images={message.imageUrls} />
-              )}
-              {messages.length - 1 === index &&
-                message.sender === "assistant" &&
-                curAgentState === AgentState.AWAITING_USER_CONFIRMATION && (
-                  <ConfirmationButtons />
+        {isLoadingMessages && (
+          <div className="flex justify-center">
+            <div className="w-6 h-6 border-2 border-t-[4px] border-primary-500 rounded-full animate-spin" />
+          </div>
+        )}
+
+        {!isLoadingMessages &&
+          messages.map((message, index) =>
+            isErrorMessage(message) ? (
+              <ErrorMessage
+                key={index}
+                id={message.id}
+                message={message.message}
+              />
+            ) : (
+              <ChatMessage
+                key={index}
+                type={message.sender}
+                message={message.content}
+              >
+                {message.imageUrls.length > 0 && (
+                  <ImageCarousel size="small" images={message.imageUrls} />
                 )}
-            </ChatMessage>
-          ),
+                {messages.length - 1 === index &&
+                  message.sender === "assistant" &&
+                  curAgentState === AgentState.AWAITING_USER_CONFIRMATION && (
+                    <ConfirmationButtons />
+                  )}
+              </ChatMessage>
+            ),
+          )}
+
+        {(curAgentState === AgentState.AWAITING_USER_INPUT ||
+          curAgentState === AgentState.FINISHED) && (
+          <div className="flex flex-col gap-2 mb-2">
+            {rootLoaderData?.ghToken ? (
+              <SuggestionItem
+                suggestion={{
+                  label: "Push to GitHub",
+                  value:
+                    "Please push the changes to GitHub and open a pull request.",
+                }}
+                onClick={(value) => {
+                  handleSendMessage(value, []);
+                }}
+              />
+            ) : (
+              <SuggestionItem
+                suggestion={{
+                  label: !isDownloading
+                    ? "Download .zip"
+                    : "Downloading, please wait...",
+                  value: "Download .zip",
+                }}
+                onClick={handleDownloadWorkspace}
+              />
+            )}
+          </div>
         )}
       </div>
 

frontend/src/components/event-handler.tsx

@@ -14,13 +14,13 @@ import {
 } from "#/context/ws-client-provider";
 import { ErrorObservation } from "#/types/core/observations";
 import { addErrorMessage, addUserMessage } from "#/state/chatSlice";
-import { handleAssistantMessage } from "#/services/actions";
 import {
   getCloneRepoCommand,
   getGitHubTokenCommand,
 } from "#/services/terminalService";
 import {
   clearFiles,
+  clearInitialQuery,
   clearSelectedRepository,
   setImportedProjectZip,
 } from "#/state/initial-query-slice";
@@ -34,6 +34,7 @@ import { base64ToBlob } from "#/utils/base64-to-blob";
 import { setCurrentAgentState } from "#/state/agentSlice";
 import AgentState from "#/types/AgentState";
 import { getSettings } from "#/services/settings";
+import { generateAgentStateChangeEvent } from "#/services/agentStateService";
 
 interface ServerError {
   error: boolean | string;
@@ -52,13 +53,10 @@ export function EventHandler({ children }: React.PropsWithChildren) {
   const runtimeActive = status === WsClientProviderStatus.ACTIVE;
   const fetcher = useFetcher();
   const dispatch = useDispatch();
-  const { files, importedProjectZip } = useSelector(
+  const { files, importedProjectZip, initialQuery } = useSelector(
     (state: RootState) => state.initalQuery,
   );
   const { ghToken, repo } = useLoaderData<typeof appClientLoader>();
-  const initialQueryRef = React.useRef<string | null>(
-    store.getState().initalQuery.initialQuery,
-  );
 
   const sendInitialQuery = (query: string, base64Files: string[]) => {
     const timestamp = new Date().toISOString();
@@ -96,6 +94,14 @@ export function EventHandler({ children }: React.PropsWithChildren) {
       return;
     }
 
+    if (event.type === "error") {
+      const message: string = `${event.message}`;
+      if (message.startsWith("Agent reached maximum")) {
+        // We set the agent state to paused here - if the user clicks resume, it auto updates the max iterations
+        send(generateAgentStateChangeEvent(AgentState.PAUSED));
+      }
+    }
+
     if (isErrorObservation(event)) {
       dispatch(
         addErrorMessage({
@@ -103,9 +109,7 @@ export function EventHandler({ children }: React.PropsWithChildren) {
           message: event.message,
         }),
       );
-      return;
     }
-    handleAssistantMessage(event);
   }, [events.length]);
 
   React.useEffect(() => {
@@ -113,7 +117,6 @@ export function EventHandler({ children }: React.PropsWithChildren) {
       return; // This is a check because of strict mode - if the status did not change, don't do anything
     }
     statusRef.current = status;
-    const initialQuery = initialQueryRef.current;
 
     if (status === WsClientProviderStatus.ACTIVE) {
       let additionalInfo = "";
@@ -134,7 +137,7 @@ export function EventHandler({ children }: React.PropsWithChildren) {
           sendInitialQuery(initialQuery, files);
         }
         dispatch(clearFiles()); // reset selected files
-        initialQueryRef.current = null;
+        dispatch(clearInitialQuery()); // reset initial query
       }
     }
 

frontend/src/components/file-explorer/FileExplorer.tsx

@@ -12,6 +12,7 @@ import { useTranslation } from "react-i18next";
 import { twMerge } from "tailwind-merge";
 import AgentState from "#/types/AgentState";
 import { setRefreshID } from "#/state/codeSlice";
+import { addAssistantMessage } from "#/state/chatSlice";
 import IconButton from "../IconButton";
 import ExplorerTree from "./ExplorerTree";
 import toast from "#/utils/toast";
@@ -20,6 +21,7 @@ import { I18nKey } from "#/i18n/declaration";
 import OpenHands from "#/api/open-hands";
 import { useFiles } from "#/context/files";
 import { isOpenHandsErrorResponse } from "#/api/open-hands.utils";
+import VSCodeIcon from "#/assets/vscode-alt.svg?react";
 
 interface ExplorerActionsProps {
   onRefresh: () => void;
@@ -168,6 +170,35 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
     }
   };
 
+  const handleVSCodeClick = async (e: React.MouseEvent) => {
+    e.preventDefault();
+    try {
+      const response = await OpenHands.getVSCodeUrl();
+      if (response.vscode_url) {
+        dispatch(
+          addAssistantMessage(
+            "You opened VS Code. Please inform the agent of any changes you made to the workspace or environment. To avoid conflicts, it's best to pause the agent before making any changes.",
+          ),
+        );
+        window.open(response.vscode_url, "_blank");
+      } else {
+        toast.error(
+          `open-vscode-error-${new Date().getTime()}`,
+          t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+            error: response.error,
+          }),
+        );
+      }
+    } catch (exp_error) {
+      toast.error(
+        `open-vscode-error-${new Date().getTime()}`,
+        t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+          error: String(exp_error),
+        }),
+      );
+    }
+  };
+
   React.useEffect(() => {
     refreshWorkspace();
   }, [curAgentState]);
@@ -210,7 +241,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
           !isOpen ? "w-12" : "w-60",
         )}
       >
-        <div className="flex flex-col relative h-full px-3 py-2">
+        <div className="flex flex-col relative h-full px-3 py-2 overflow-hidden">
           <div className="sticky top-0 bg-neutral-800">
             <div
               className={twMerge(
@@ -232,7 +263,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
             </div>
           </div>
           {!error && (
-            <div className="overflow-auto flex-grow">
+            <div className="overflow-auto flex-grow min-h-0">
               <div style={{ display: !isOpen ? "none" : "block" }}>
                 <ExplorerTree files={paths} />
               </div>
@@ -243,6 +274,27 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
               <p className="text-neutral-300 text-sm">{error}</p>
             </div>
           )}
+          {isOpen && (
+            <button
+              type="button"
+              onClick={handleVSCodeClick}
+              disabled={
+                curAgentState === AgentState.INIT ||
+                curAgentState === AgentState.LOADING
+              }
+              className={twMerge(
+                "mt-auto mb-2 w-full h-10 text-white rounded flex items-center justify-center gap-2 transition-colors",
+                curAgentState === AgentState.INIT ||
+                  curAgentState === AgentState.LOADING
+                  ? "bg-neutral-600 cursor-not-allowed"
+                  : "bg-[#4465DB] hover:bg-[#3451C7]",
+              )}
+              aria-label="Open in VS Code"
+            >
+              <VSCodeIcon width={20} height={20} />
+              Open in VS Code
+            </button>
+          )}
         </div>
         <input
           data-testid="file-input"

frontend/src/components/github-repositories-suggestion-box.tsx

@@ -10,32 +10,8 @@ import { GitHubRepositorySelector } from "#/routes/_oh._index/github-repo-select
 import ModalButton from "./buttons/ModalButton";
 import GitHubLogo from "#/assets/branding/github-logo.svg?react";
 
-interface GitHubAuthProps {
-  onConnectToGitHub: () => void;
-  repositories: GitHubRepository[];
-  isLoggedIn: boolean;
-}
-
-function GitHubAuth({
-  onConnectToGitHub,
-  repositories,
-  isLoggedIn,
-}: GitHubAuthProps) {
-  if (isLoggedIn) {
-    return <GitHubRepositorySelector repositories={repositories} />;
-  }
-
-  return (
-    <ModalButton
-      text="Connect to GitHub"
-      icon={<GitHubLogo width={20} height={20} />}
-      className="bg-[#791B80] w-full"
-      onClick={onConnectToGitHub}
-    />
-  );
-}
-
 interface GitHubRepositoriesSuggestionBoxProps {
+  handleSubmit: () => void;
   repositories: Awaited<
     ReturnType<typeof retrieveAllGitHubUserRepositories>
   > | null;
@@ -44,6 +20,7 @@ interface GitHubRepositoriesSuggestionBoxProps {
 }
 
 export function GitHubRepositoriesSuggestionBox({
+  handleSubmit,
   repositories,
   gitHubAuthUrl,
   user,
@@ -70,16 +47,26 @@ export function GitHubRepositoriesSuggestionBox({
     );
   }
 
+  const isLoggedIn = !!user && !isGitHubErrorReponse(user);
+
   return (
     <>
       <SuggestionBox
         title="Open a Repo"
         content={
-          <GitHubAuth
-            isLoggedIn={!!user && !isGitHubErrorReponse(user)}
-            repositories={repositories || []}
-            onConnectToGitHub={handleConnectToGitHub}
-          />
+          isLoggedIn ? (
+            <GitHubRepositorySelector
+              onSelect={handleSubmit}
+              repositories={repositories || []}
+            />
+          ) : (
+            <ModalButton
+              text="Connect to GitHub"
+              icon={<GitHubLogo width={20} height={20} />}
+              className="bg-[#791B80] w-full"
+              onClick={handleConnectToGitHub}
+            />
+          )
         }
       />
       {connectToGitHubModalOpen && (

frontend/src/components/interactive-chat-box.tsx

@@ -56,14 +56,9 @@ export function InteractiveChatBox({
       <div
         className={cn(
           "flex items-end gap-1",
-          "bg-neutral-700 border border-neutral-600 rounded-lg px-2 py-[10px]",
+          "bg-neutral-700 border border-neutral-600 rounded-lg px-2",
           "transition-colors duration-200",
           "hover:border-neutral-500 focus-within:border-neutral-500",
-          "group relative",
-          "before:pointer-events-none before:absolute before:inset-0 before:rounded-lg before:transition-colors",
-          "before:border-2 before:border-dashed before:border-transparent",
-          "[&:has(*:focus-within)]:before:border-neutral-500/50",
-          "[&:has(*[data-dragging-over='true'])]:before:border-neutral-500/50",
         )}
       >
         <UploadImageInput onUpload={handleUpload} />
@@ -76,6 +71,8 @@ export function InteractiveChatBox({
           onStop={onStop}
           value={value}
           onImagePaste={handleUpload}
+          className="py-[10px]"
+          buttonClassName="py-[10px]"
         />
       </div>
     </div>

frontend/src/components/markdown/list.tsx

@@ -4,8 +4,8 @@ import { ExtraProps } from "react-markdown";
 // Custom component to render <ul> in markdown
 export function ul({
   children,
-}: React.ClassAttributes<HTMLElement> &
-  React.HTMLAttributes<HTMLElement> &
+}: React.ClassAttributes<HTMLUListElement> &
+  React.HTMLAttributes<HTMLUListElement> &
   ExtraProps) {
   return <ul className="list-disc ml-5 pl-2 whitespace-normal">{children}</ul>;
 }
@@ -13,10 +13,13 @@ export function ul({
 // Custom component to render <ol> in markdown
 export function ol({
   children,
-}: React.ClassAttributes<HTMLElement> &
-  React.HTMLAttributes<HTMLElement> &
+  start,
+}: React.ClassAttributes<HTMLOListElement> &
+  React.OlHTMLAttributes<HTMLOListElement> &
   ExtraProps) {
   return (
-    <ol className="list-decimal ml-5 pl-2 whitespace-normal">{children}</ol>
+    <ol className="list-decimal ml-5 pl-2 whitespace-normal" start={start}>
+      {children}
+    </ol>
   );
 }

frontend/src/components/modals/AccountSettingsModal.tsx

@@ -1,7 +1,10 @@
 import { useFetcher, useRouteLoaderData } from "@remix-run/react";
 import React from "react";
 import { useTranslation } from "react-i18next";
-import { BaseModalTitle } from "./confirmation-modals/BaseModal";
+import {
+  BaseModalDescription,
+  BaseModalTitle,
+} from "./confirmation-modals/BaseModal";
 import ModalBody from "./ModalBody";
 import ModalButton from "../buttons/ModalButton";
 import FormFieldset from "../form/FormFieldset";
@@ -87,6 +90,17 @@ function AccountSettingsModal({
             type="password"
             defaultValue={data?.ghToken ?? ""}
           />
+          <BaseModalDescription>
+            {t(I18nKey.CONNECT_TO_GITHUB_MODAL$GET_YOUR_TOKEN)}{" "}
+            <a
+              href="https://github.com/settings/tokens/new?description=openhands-app&scopes=repo,user,workflow"
+              target="_blank"
+              rel="noreferrer noopener"
+              className="text-[#791B80] underline"
+            >
+              {t(I18nKey.CONNECT_TO_GITHUB_MODAL$HERE)}
+            </a>
+          </BaseModalDescription>
           {gitHubError && (
             <p className="text-danger text-xs">
               {t(I18nKey.ACCOUNT_SETTINGS_MODAL$GITHUB_TOKEN_INVALID)}

frontend/src/components/suggestion-item.tsx

@@ -7,12 +7,12 @@ interface SuggestionItemProps {
 
 export function SuggestionItem({ suggestion, onClick }: SuggestionItemProps) {
   return (
-    <li className="border border-neutral-600 rounded-xl hover:bg-neutral-700">
+    <li className="list-none border border-neutral-600 rounded-xl hover:bg-neutral-700">
       <button
         type="button"
         data-testid="suggestion"
         onClick={() => onClick(suggestion.value)}
-        className="text-[16px] leading-6 -tracking-[0.01em] text-center w-full p-4 font-semibold"
+        className="text-[16px] leading-6 -tracking-[0.01em] text-center w-full p-3 font-semibold"
       >
         {suggestion.label}
       </button>

frontend/src/components/upload-image-input.tsx

@@ -11,7 +11,7 @@ export function UploadImageInput({ onUpload, label }: UploadImageInputProps) {
   };
 
   return (
-    <label className="cursor-pointer">
+    <label className="cursor-pointer py-[10px]">
       {label || <Clip data-testid="default-label" width={24} height={24} />}
       <input
         data-testid="upload-image-input"

frontend/src/context/ws-client-provider.tsx

@@ -4,6 +4,13 @@ import { Settings } from "#/services/settings";
 import ActionType from "#/types/ActionType";
 import EventLogger from "#/utils/event-logger";
 import AgentState from "#/types/AgentState";
+import { handleAssistantMessage } from "#/services/actions";
+import { useRate } from "#/utils/use-rate";
+
+const isOpenHandsMessage = (event: Record<string, unknown>) =>
+  event.action === "message";
+
+const RECONNECT_RETRIES = 5;
 
 export enum WsClientProviderStatus {
   STOPPED,
@@ -14,12 +21,14 @@ export enum WsClientProviderStatus {
 
 interface UseWsClient {
   status: WsClientProviderStatus;
+  isLoadingMessages: boolean;
   events: Record<string, unknown>[];
   send: (event: Record<string, unknown>) => void;
 }
 
 const WsClientContext = React.createContext<UseWsClient>({
   status: WsClientProviderStatus.STOPPED,
+  isLoadingMessages: true,
   events: [],
   send: () => {
     throw new Error("not connected");
@@ -46,6 +55,9 @@ export function WsClientProvider({
   const closeRef = React.useRef<ReturnType<typeof setTimeout> | null>(null);
   const [status, setStatus] = React.useState(WsClientProviderStatus.STOPPED);
   const [events, setEvents] = React.useState<Record<string, unknown>[]>([]);
+  const [retryCount, setRetryCount] = React.useState(RECONNECT_RETRIES);
+
+  const messageRateHandler = useRate({ threshold: 500 });
 
   function send(event: Record<string, unknown>) {
     if (!wsRef.current) {
@@ -56,6 +68,7 @@ export function WsClientProvider({
   }
 
   function handleOpen() {
+    setRetryCount(RECONNECT_RETRIES);
     setStatus(WsClientProviderStatus.OPENING);
     const initEvent = {
       action: ActionType.INIT,
@@ -66,6 +79,9 @@ export function WsClientProvider({
 
   function handleMessage(messageEvent: MessageEvent) {
     const event = JSON.parse(messageEvent.data);
+    if (isOpenHandsMessage(event)) {
+      messageRateHandler.record(new Date().getTime());
+    }
     setEvents((prevEvents) => [...prevEvents, event]);
     if (event.extras?.agent_state === AgentState.INIT) {
       setStatus(WsClientProviderStatus.ACTIVE);
@@ -76,11 +92,19 @@ export function WsClientProvider({
     ) {
       setStatus(WsClientProviderStatus.ERROR);
     }
+
+    handleAssistantMessage(event);
   }
 
   function handleClose() {
-    setStatus(WsClientProviderStatus.STOPPED);
-    setEvents([]);
+    if (retryCount) {
+      setTimeout(() => {
+        setRetryCount(retryCount - 1);
+      }, 1000);
+    } else {
+      setStatus(WsClientProviderStatus.STOPPED);
+      setEvents([]);
+    }
     wsRef.current = null;
   }
 
@@ -95,7 +119,7 @@ export function WsClientProvider({
     let ws = wsRef.current;
 
     // If disabled close any existing websockets...
-    if (!enabled) {
+    if (!enabled || !retryCount) {
       if (ws) {
         ws.close();
       }
@@ -116,7 +140,11 @@ export function WsClientProvider({
       const baseUrl =
         import.meta.env.VITE_BACKEND_BASE_URL || window?.location.host;
       const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
-      ws = new WebSocket(`${protocol}//${baseUrl}/ws`, [
+      let wsUrl = `${protocol}//${baseUrl}/ws`;
+      if (events.length) {
+        wsUrl += `?latest_event_id=${events[events.length - 1].id}`;
+      }
+      ws = new WebSocket(wsUrl, [
         "openhands",
         token || "NO_JWT",
         ghToken || "NO_GITHUB",
@@ -136,7 +164,7 @@ export function WsClientProvider({
       ws.removeEventListener("error", handleError);
       ws.removeEventListener("close", handleClose);
     };
-  }, [enabled, token, ghToken]);
+  }, [enabled, token, ghToken, retryCount]);
 
   // Strict mode mounts and unmounts each component twice, so we have to wait in the destructor
   // before actually closing the socket and cancel the operation if the component gets remounted.
@@ -148,7 +176,11 @@ export function WsClientProvider({
 
     return () => {
       closeRef.current = setTimeout(() => {
-        wsRef.current?.close();
+        const ws = wsRef.current;
+        if (ws) {
+          ws.removeEventListener("close", handleClose);
+          ws.close();
+        }
       }, 100);
     };
   }, []);
@@ -156,10 +188,11 @@ export function WsClientProvider({
   const value = React.useMemo<UseWsClient>(
     () => ({
       status,
+      isLoadingMessages: messageRateHandler.isUnderThreshold,
       events,
       send,
     }),
-    [status, events],
+    [status, messageRateHandler.isUnderThreshold, events],
   );
 
   return (

frontend/src/entry.client.tsx

@@ -12,15 +12,26 @@ import { Provider } from "react-redux";
 import posthog from "posthog-js";
 import "./i18n";
 import store from "./store";
+import OpenHands from "./api/open-hands";
 
 function PosthogInit() {
+  const [key, setKey] = React.useState<string | null>(null);
+
   React.useEffect(() => {
-    posthog.init("phc_3ESMmY9SgqEAGBB6sMGK5ayYHkeUuknH2vP6FmWH9RA", {
-      api_host: "https://us.i.posthog.com",
-      person_profiles: "identified_only",
+    OpenHands.getConfig().then((config) => {
+      setKey(config.POSTHOG_CLIENT_KEY);
     });
   }, []);
 
+  React.useEffect(() => {
+    if (key) {
+      posthog.init(key, {
+        api_host: "https://us.i.posthog.com",
+        person_profiles: "identified_only",
+      });
+    }
+  }, [key]);
+
   return null;
 }
 

frontend/src/i18n/translation.json

@@ -678,6 +678,16 @@
     "tr": "Sunucudan beklenmeyen yanıt yapısı",
     "no": "Uventet responsstruktur fra serveren"
   },
+  "EXPLORER$VSCODE_SWITCHING_MESSAGE": {
+    "en": "Switching to VS Code in 3 seconds...\nImportant: Please inform the agent of any changes you make in VS Code. To avoid conflicts, wait for the assistant to complete its work before making your own changes.",
+    "zh-CN": "3 秒后切换到 VS Code\n重要提示：请告知 OpenHands 您在 VS Code 中进行的任何更改。为了避免冲突，请在 OpenHands 完成工作后再进行自己的更改。",
+    "zh-TW": "3 秒後切換到 VS Code\n重要提示：請告知 OpenHands 您在 VS Code 中進行的任何更改。為避免衝突，請在 OpenHands 完成工作後再進行自己的更改。"
+  },
+  "EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE": {
+    "en": "Error switching to VS Code: {{error}}",
+    "zh-CN": "切换到 VS Code 时发生错误: {{error}}",
+    "zh-TW": "切換到 VS Code 時發生錯誤: {{error}}"
+  },
   "LOAD_SESSION$MODAL_TITLE": {
     "en": "Return to existing session?",
     "de": "Zurück zu vorhandener Sitzung?",

frontend/src/mocks/handlers.ts

@@ -71,8 +71,6 @@ const openHandsHandlers = [
 export const handlers = [
   ...openHandsHandlers,
   http.get("https://api.github.com/user/repos", async ({ request }) => {
-    if (import.meta.env.MODE !== "test") await delay(3500);
-
     const token = request.headers
       .get("Authorization")
       ?.replace("Bearer", "")

frontend/src/mocks/handlers.ws.ts

@@ -29,7 +29,7 @@ const generateAgentResponse = (message: string): AssistantMessageAction => ({
   action: "message",
   args: {
     content: message,
-    images_urls: [],
+    image_urls: [],
     wait_for_response: false,
   },
 });

frontend/src/routes/_oh._index/github-repo-selector.tsx

@@ -1,16 +1,16 @@
 import { Autocomplete, AutocompleteItem } from "@nextui-org/react";
 import { useDispatch } from "react-redux";
-import { useNavigate } from "react-router-dom";
 import { setSelectedRepository } from "#/state/initial-query-slice";
 
 interface GitHubRepositorySelectorProps {
+  onSelect: () => void;
   repositories: GitHubRepository[];
 }
 
 export function GitHubRepositorySelector({
+  onSelect,
   repositories,
 }: GitHubRepositorySelectorProps) {
-  const navigate = useNavigate();
   const dispatch = useDispatch();
 
   const handleRepoSelection = (id: string | null) => {
@@ -18,7 +18,7 @@ export function GitHubRepositorySelector({
     if (repo) {
       // set query param
       dispatch(setSelectedRepository(repo.full_name));
-      navigate("/app");
+      onSelect();
     }
   };
 

frontend/src/routes/_oh._index/route.tsx

@@ -5,7 +5,6 @@ import {
   defer,
   redirect,
   useLoaderData,
-  useNavigate,
   useRouteLoaderData,
 } from "@remix-run/react";
 import React from "react";
@@ -73,10 +72,10 @@ export const clientAction = async ({ request }: ClientActionFunctionArgs) => {
 };
 
 function Home() {
-  const navigate = useNavigate();
   const dispatch = useDispatch();
   const rootData = useRouteLoaderData<typeof rootClientLoader>("routes/_oh");
   const { repositories, githubAuthUrl } = useLoaderData<typeof clientLoader>();
+  const formRef = React.useRef<HTMLFormElement>(null);
 
   return (
     <div
@@ -86,7 +85,7 @@ function Home() {
       <HeroHeading />
       <div className="flex flex-col gap-16 w-[600px] items-center">
         <div className="flex flex-col gap-2 w-full">
-          <TaskForm />
+          <TaskForm ref={formRef} />
         </div>
         <div className="flex gap-4 w-full">
           <React.Suspense
@@ -100,6 +99,7 @@ function Home() {
             <Await resolve={repositories}>
               {(resolvedRepositories) => (
                 <GitHubRepositoriesSuggestionBox
+                  handleSubmit={() => formRef.current?.requestSubmit()}
                   repositories={resolvedRepositories}
                   gitHubAuthUrl={githubAuthUrl}
                   user={rootData?.user || null}
@@ -129,7 +129,7 @@ function Home() {
                       dispatch(
                         setImportedProjectZip(await convertZipToBase64(zip)),
                       );
-                      navigate("/app");
+                      formRef.current?.requestSubmit();
                     } else {
                       // TODO: handle error
                     }

frontend/src/routes/_oh._index/task-form.tsx

@@ -13,7 +13,7 @@ import { getRandomKey } from "#/utils/get-random-key";
 import { AttachImageLabel } from "#/components/attach-image-label";
 import { cn } from "#/utils/utils";
 
-export function TaskForm() {
+export const TaskForm = React.forwardRef<HTMLFormElement>((_, ref) => {
   const dispatch = useDispatch();
   const navigation = useNavigation();
 
@@ -21,7 +21,6 @@ export function TaskForm() {
     (state: RootState) => state.initalQuery,
   );
 
-  const formRef = React.useRef<HTMLFormElement>(null);
   const [text, setText] = React.useState("");
   const [suggestion, setSuggestion] = React.useState(
     getRandomKey(SUGGESTIONS["non-repo"]),
@@ -55,7 +54,7 @@ export function TaskForm() {
   return (
     <div className="flex flex-col gap-2 w-full">
       <Form
-        ref={formRef}
+        ref={ref}
         method="post"
         className="flex flex-col items-center gap-2"
         replace
@@ -67,20 +66,15 @@ export function TaskForm() {
         />
         <div
           className={cn(
-            "border border-neutral-600 px-4 py-[17px] rounded-lg text-[17px] leading-5 w-full transition-colors duration-200",
+            "border border-neutral-600 px-4 rounded-lg text-[17px] leading-5 w-full transition-colors duration-200",
             inputIsFocused ? "bg-neutral-600" : "bg-neutral-700",
             "hover:border-neutral-500 focus-within:border-neutral-500",
-            "group relative",
-            "before:pointer-events-none before:absolute before:inset-0 before:rounded-lg before:transition-colors",
-            "before:border-2 before:border-dashed before:border-transparent",
-            "[&:has(*:focus-within)]:before:border-neutral-500/50",
-            "[&:has(*[data-dragging-over='true'])]:before:border-neutral-500/50",
           )}
         >
           <ChatInput
             name="q"
             onSubmit={() => {
-              formRef.current?.requestSubmit();
+              if (typeof ref !== "function") ref?.current?.requestSubmit();
             }}
             onChange={(message) => setText(message)}
             onFocus={() => setInputIsFocused(true)}
@@ -96,7 +90,8 @@ export function TaskForm() {
             value={text}
             maxRows={15}
             showButton={!!text}
-            className="text-[17px] leading-5"
+            className="text-[17px] leading-5 py-[17px]"
+            buttonClassName="pb-[17px]"
             disabled={navigation.state === "submitting"}
           />
         </div>
@@ -120,4 +115,6 @@ export function TaskForm() {
       )}
     </div>
   );
-}
+});
+
+TaskForm.displayName = "TaskForm";

frontend/src/routes/_oh.app._index/code-editor-component.tsx

@@ -7,15 +7,15 @@ import { I18nKey } from "#/i18n/declaration";
 import { useFiles } from "#/context/files";
 import OpenHands from "#/api/open-hands";
 
-interface CodeEditorCompoonentProps {
+interface CodeEditorComponentProps {
   onMount: EditorProps["onMount"];
   isReadOnly: boolean;
 }
 
-function CodeEditorCompoonent({
+function CodeEditorComponent({
   onMount,
   isReadOnly,
-}: CodeEditorCompoonentProps) {
+}: CodeEditorComponentProps) {
   const { t } = useTranslation();
   const {
     files,
@@ -107,4 +107,4 @@ function CodeEditorCompoonent({
   );
 }
 
-export default React.memo(CodeEditorCompoonent);
+export default React.memo(CodeEditorComponent);

frontend/src/routes/_oh.app._index/route.tsx

@@ -8,10 +8,22 @@ import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import FileExplorer from "#/components/file-explorer/FileExplorer";
 import OpenHands from "#/api/open-hands";
-import CodeEditorCompoonent from "./code-editor-component";
+import CodeEditorComponent from "./code-editor-component";
 import { useFiles } from "#/context/files";
 import { EditorActions } from "#/components/editor-actions";
 
+const ASSET_FILE_TYPES = [
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".bmp",
+  ".gif",
+  ".pdf",
+  ".mp4",
+  ".webm",
+  ".ogg",
+];
+
 export const clientLoader = async () => {
   const token = localStorage.getItem("token");
   return json({ token });
@@ -104,6 +116,10 @@ function CodeEditor() {
     if (selectedPath) discardChanges(selectedPath);
   };
 
+  const isAssetFileType = selectedPath
+    ? ASSET_FILE_TYPES.some((ext) => selectedPath.endsWith(ext))
+    : false;
+
   return (
     <div className="flex h-full bg-neutral-900 relative">
       <FileExplorer
@@ -112,7 +128,7 @@ function CodeEditor() {
         error={errors.getFiles}
       />
       <div className="w-full">
-        {selectedPath && (
+        {selectedPath && !isAssetFileType && (
           <div className="flex w-full items-center justify-between self-end p-2">
             <span className="text-sm text-neutral-500">{selectedPath}</span>
             <EditorActions
@@ -122,7 +138,7 @@ function CodeEditor() {
             />
           </div>
         )}
-        <CodeEditorCompoonent
+        <CodeEditorComponent
           onMount={handleEditorDidMount}
           isReadOnly={!isEditingAllowed}
         />

frontend/src/routes/_oh.app.tsx

@@ -18,7 +18,6 @@ import { useEffectOnce } from "#/utils/use-effect-once";
 import CodeIcon from "#/icons/code.svg?react";
 import GlobeIcon from "#/icons/globe.svg?react";
 import ListIcon from "#/icons/list-type-number.svg?react";
-import { clearInitialQuery } from "#/state/initial-query-slice";
 import { isGitHubErrorReponse, retrieveLatestGitHubCommit } from "#/api/github";
 import { clearJupyter } from "#/state/jupyterSlice";
 import { FilesProvider } from "#/context/files";
@@ -28,8 +27,6 @@ import { EventHandler } from "#/components/event-handler";
 
 export const clientLoader = async () => {
   const ghToken = localStorage.getItem("ghToken");
-
-  const q = store.getState().initalQuery.initialQuery;
   const repo =
     store.getState().initalQuery.selectedRepository ||
     localStorage.getItem("repo");
@@ -55,7 +52,6 @@ export const clientLoader = async () => {
     token,
     ghToken,
     repo,
-    q,
     lastCommit,
   });
 };
@@ -91,7 +87,6 @@ function App() {
     dispatch(clearMessages());
     dispatch(clearTerminal());
     dispatch(clearJupyter());
-    dispatch(clearInitialQuery()); // Clear initial query when navigating to /app
   });
 
   const {

frontend/src/routes/_oh.tsx

@@ -171,6 +171,8 @@ export default function MainApp() {
         company: user.company,
         name: user.name,
         email: user.email,
+        user: user.login,
+        mode: window.__APP_MODE__ || "oss",
       });
     }
   }, [user]);
@@ -241,7 +243,7 @@ export default function MainApp() {
               type="button"
               aria-label="All Hands Logo"
               onClick={() => {
-                if (location.pathname === "/app")
+                if (location.pathname.startsWith("/app"))
                   setStartNewProjectModalIsOpen(true);
               }}
             >

frontend/src/services/chatService.ts

@@ -2,12 +2,12 @@ import ActionType from "#/types/ActionType";
 
 export function createChatMessage(
   message: string,
-  images_urls: string[],
+  image_urls: string[],
   timestamp: string,
 ) {
   const event = {
     action: ActionType.MESSAGE,
-    args: { content: message, images_urls, timestamp },
+    args: { content: message, image_urls, timestamp },
   };
   return event;
 }

frontend/src/types/core/actions.ts

@@ -4,7 +4,7 @@ export interface UserMessageAction extends OpenHandsActionEvent<"message"> {
   source: "user";
   args: {
     content: string;
-    images_urls: string[];
+    image_urls: string[];
   };
 }
 
@@ -23,7 +23,7 @@ export interface AssistantMessageAction
   source: "agent";
   args: {
     content: string;
-    images_urls: string[] | null;
+    image_urls: string[] | null;
     wait_for_response: boolean;
   };
 }

frontend/src/types/core/variances.ts

@@ -27,7 +27,7 @@ interface LocalUserMessageAction {
   action: "message";
   args: {
     content: string;
-    images_urls: string[];
+    image_urls: string[];
   };
 }
 

frontend/src/utils/clear-session.ts

@@ -1,7 +1,21 @@
+import store from "#/store";
+import { initialState as browserInitialState } from "#/state/browserSlice";
+
 /**
- * Clear the session data from the local storage. This will remove the token and repo
+ * Clear the session data from the local storage and reset relevant Redux state
  */
 export const clearSession = () => {
+  // Clear local storage
   localStorage.removeItem("token");
   localStorage.removeItem("repo");
+
+  // Reset browser state to initial values
+  store.dispatch({
+    type: "browser/setUrl",
+    payload: browserInitialState.url,
+  });
+  store.dispatch({
+    type: "browser/setScreenshotSrc",
+    payload: browserInitialState.screenshotSrc,
+  });
 };

frontend/src/utils/suggestions/repo-suggestions.ts

@@ -13,14 +13,14 @@ const KEY_2 = "Auto-merge Dependabot PRs";
 const VALUE_2 = `Please add a GitHub action to this repository which automatically merges pull requests from Dependabot so long as the tests are passing.`;
 
 const KEY_3 = "Fix up my README";
-const VALUE_3 = `"Please look at the README and make the following improvements, if they make sense:
+const VALUE_3 = `Please look at the README and make the following improvements, if they make sense:
 * correct any typos that you find
 * add missing language annotations on codeblocks
 * if there are references to other files or other sections of the README, turn them into links
 * make sure the readme has an h1 title towards the top
 * make sure any existing sections in the readme are appropriately separated with headings
 
-If there are no obvious ways to improve the README, make at least one small change to make the wording clearer or friendlier"`;
+If there are no obvious ways to improve the README, make at least one small change to make the wording clearer or friendlier`;
 
 const KEY_4 = "Clean up my dependencies";
 const VALUE_4 = `Examine the dependencies of the current codebase. Make sure you can run the code and any tests.

frontend/src/utils/toast.tsx

@@ -10,7 +10,6 @@ export default {
       style: {
         background: "#ef4444",
         color: "#fff",
-        lineBreak: "anywhere",
       },
       iconTheme: {
         primary: "#ef4444",
@@ -19,25 +18,20 @@ export default {
     });
     idMap.set(id, toastId);
   },
-  success: (id: string, msg: string) => {
-    const toastId = idMap.get(id);
-    if (toastId === undefined) return;
-    if (toastId) {
-      toast.success(msg, {
-        id: toastId,
-        duration: 4000,
-        style: {
-          background: "#333",
-          color: "#fff",
-          lineBreak: "anywhere",
-        },
-        iconTheme: {
-          primary: "#333",
-          secondary: "#fff",
-        },
-      });
-    }
-    idMap.delete(id);
+  success: (id: string, msg: string, duration: number = 4000) => {
+    if (idMap.has(id)) return; // prevent duplicate toast
+    const toastId = toast.success(msg, {
+      duration,
+      style: {
+        background: "#333",
+        color: "#fff",
+      },
+      iconTheme: {
+        primary: "#333",
+        secondary: "#fff",
+      },
+    });
+    idMap.set(id, toastId);
   },
   settingsChanged: (msg: string) => {
     toast(msg, {
@@ -48,7 +42,6 @@ export default {
       style: {
         background: "#333",
         color: "#fff",
-        lineBreak: "anywhere",
       },
     });
   },

frontend/src/utils/use-rate.ts

@@ -0,0 +1,67 @@
+import React from "react";
+
+interface UseRateProps {
+  threshold: number;
+}
+
+const DEFAULT_CONFIG: UseRateProps = { threshold: 1000 };
+
+export const useRate = (config = DEFAULT_CONFIG) => {
+  const [items, setItems] = React.useState<number[]>([]);
+  const [rate, setRate] = React.useState<number | null>(null);
+  const [lastUpdated, setLastUpdated] = React.useState<number | null>(null);
+  const [isUnderThreshold, setIsUnderThreshold] = React.useState(true);
+
+  /**
+   * Record an entry in order to calculate the rate
+   * @param entry Entry to record
+   *
+   * @example
+   * record(new Date().getTime());
+   */
+  const record = (entry: number) => {
+    setItems((prev) => [...prev, entry]);
+    setLastUpdated(new Date().getTime());
+  };
+
+  /**
+   * Update the rate based on the last two entries (if available)
+   */
+  const updateRate = () => {
+    if (items.length > 1) {
+      const newRate = items[items.length - 1] - items[items.length - 2];
+      setRate(newRate);
+
+      if (newRate <= config.threshold) setIsUnderThreshold(true);
+      else setIsUnderThreshold(false);
+    }
+  };
+
+  React.useEffect(() => {
+    updateRate();
+  }, [items]);
+
+  React.useEffect(() => {
+    // Set up an interval to check if the time since the last update exceeds the threshold
+    // If it does, set isUnderThreshold to false, otherwise set it to true
+    // This ensures that the component can react to periods of inactivity
+    const intervalId = setInterval(() => {
+      if (lastUpdated !== null) {
+        const timeSinceLastUpdate = new Date().getTime() - lastUpdated;
+        setIsUnderThreshold(timeSinceLastUpdate <= config.threshold);
+      } else {
+        setIsUnderThreshold(false);
+      }
+    }, config.threshold);
+
+    return () => clearInterval(intervalId);
+  }, [lastUpdated, config.threshold]);
+
+  return {
+    items,
+    rate,
+    lastUpdated,
+    isUnderThreshold,
+    record,
+  };
+};

frontend/tests/redirect.spec.ts

@@ -59,3 +59,29 @@ test("should redirect to /app after selecting a repo", async ({ page }) => {
   await page.waitForURL("/app");
   expect(page.url()).toBe("http://127.0.0.1:3000/app");
 });
+
+// FIXME: This fails because the MSW WS mocks change state too quickly,
+// missing the OPENING status where the initial query is rendered.
+test.fail(
+  "should redirect the user to /app with their initial query after selecting a project",
+  async ({ page }) => {
+    await page.goto("/");
+    await confirmSettings(page);
+
+    // enter query
+    const testQuery = "this is my test query";
+    const textbox = page.getByPlaceholder(/what do you want to build/i);
+    expect(textbox).not.toBeNull();
+    await textbox.fill(testQuery);
+
+    const fileInput = page.getByLabel("Upload a .zip");
+    const filePath = path.join(dirname, "fixtures/project.zip");
+    await fileInput.setInputFiles(filePath);
+
+    await page.waitForURL("/app");
+
+    // get user message
+    const userMessage = page.getByTestId("user-message");
+    expect(await userMessage.textContent()).toBe(testQuery);
+  },
+);

openhands/agenthub/codeact_agent/action_parser.py

@@ -1,304 +0,0 @@
-import re
-
-from openhands.controller.action_parser import (
-    ActionParser,
-    ResponseParser,
-)
-from openhands.core.exceptions import LLMMalformedActionError
-from openhands.core.logger import openhands_logger as logger
-from openhands.events.action import (
-    Action,
-    AgentDelegateAction,
-    AgentFinishAction,
-    CmdRunAction,
-    FileEditAction,
-    IPythonRunCellAction,
-    MessageAction,
-)
-
-
-class CodeActResponseParser(ResponseParser):
-    """Parser action:
-    - CmdRunAction(command) - bash command to run
-    - FileEditAction(path, content) - edit a file
-    - IPythonRunCellAction(code) - IPython code to run
-    - AgentDelegateAction(agent, inputs) - delegate action for (sub)task
-    - MessageAction(content) - Message action to run (e.g. ask for clarification)
-    - AgentFinishAction() - end the interaction
-    """
-
-    def __init__(self):
-        # Need pay attention to the item order in self.action_parsers
-        super().__init__()
-        self.action_parsers = [
-            CodeActActionParserFinish(),
-            CodeActActionParserFileEdit(),
-            CodeActActionParserCmdRun(),
-            CodeActActionParserIPythonRunCell(),
-            CodeActActionParserAgentDelegate(),
-        ]
-        self.default_parser = CodeActActionParserMessage()
-
-    def parse(self, response) -> Action:
-        action_str = self.parse_response(response)
-        return self.parse_action(action_str)
-
-    def parse_response(self, response) -> str:
-        action = response.choices[0].message.content
-        if action is None:
-            return ''
-        for lang in ['bash', 'ipython', 'browse']:
-            # special handling for DeepSeek: it has stop-word bug and returns </execute_ipython instead of </execute_ipython>
-            if f'</execute_{lang}' in action and f'</execute_{lang}>' not in action:
-                action = action.replace(f'</execute_{lang}', f'</execute_{lang}>')
-
-            if f'<execute_{lang}>' in action and f'</execute_{lang}>' not in action:
-                action += f'</execute_{lang}>'
-
-        # special handling for DeepSeek: it has stop-word bug and returns </execute_ipython instead of </execute_ipython>
-        if '</file_edit' in action and '</file_edit>' not in action:
-            action = action.replace('</file_edit', '</file_edit>')
-
-        if '<file_edit' in action and '</file_edit>' not in action:
-            action += '</file_edit>'
-        return action
-
-    def parse_action(self, action_str: str) -> Action:
-        for action_parser in self.action_parsers:
-            if action_parser.check_condition(action_str):
-                return action_parser.parse(action_str)
-        return self.default_parser.parse(action_str)
-
-    def action_to_str(self, action: Action) -> str:
-        if isinstance(action, CmdRunAction):
-            return (
-                f'{action.thought}\n<execute_bash>\n{action.command}\n</execute_bash>'
-            )
-        elif isinstance(action, IPythonRunCellAction):
-            return f'{action.thought}\n<execute_ipython>\n{action.code}\n</execute_ipython>'
-        elif isinstance(action, AgentDelegateAction):
-            return f'{action.thought}\n<execute_browse>\n{action.inputs["task"]}\n</execute_browse>'
-        elif isinstance(action, FileEditAction):
-            return f'{action.thought}\n<file_edit path={action.path}>\n{action.content}\n</file_edit>'
-        elif isinstance(action, MessageAction):
-            return action.content
-        elif isinstance(action, AgentFinishAction) and action.source == 'agent':
-            return action.thought
-        return ''
-
-
-class CodeActActionParserFinish(ActionParser):
-    """Parser action:
-    - AgentFinishAction() - end the interaction
-    """
-
-    def __init__(
-        self,
-    ):
-        self.finish_command = None
-
-    def check_condition(self, action_str: str) -> bool:
-        self.finish_command = re.search(r'<finish>.*</finish>', action_str, re.DOTALL)
-        return self.finish_command is not None
-
-    def parse(self, action_str: str) -> Action:
-        assert (
-            self.finish_command is not None
-        ), 'self.finish_command should not be None when parse is called'
-        thought = action_str.replace(self.finish_command.group(0), '').strip()
-        return AgentFinishAction(thought=thought)
-
-
-class CodeActActionParserCmdRun(ActionParser):
-    """Parser action:
-    - CmdRunAction(command) - bash command to run
-    - AgentFinishAction() - end the interaction
-    """
-
-    def __init__(
-        self,
-    ):
-        self.bash_command = None
-
-    def check_condition(self, action_str: str) -> bool:
-        self.bash_command = re.search(
-            r'<execute_bash>(.*?)</execute_bash>', action_str, re.DOTALL
-        )
-        return self.bash_command is not None
-
-    def parse(self, action_str: str) -> Action:
-        assert (
-            self.bash_command is not None
-        ), 'self.bash_command should not be None when parse is called'
-        thought = action_str.replace(self.bash_command.group(0), '').strip()
-        # a command was found
-        command_group = self.bash_command.group(1).strip()
-        if command_group.strip() == 'exit':
-            return AgentFinishAction(thought=thought)
-        return CmdRunAction(command=command_group, thought=thought)
-
-
-class CodeActActionParserIPythonRunCell(ActionParser):
-    """Parser action:
-    - IPythonRunCellAction(code) - IPython code to run
-    """
-
-    def __init__(
-        self,
-    ):
-        self.python_code = None
-        self.jupyter_kernel_init_code: str = 'from agentskills import *'
-
-    def check_condition(self, action_str: str) -> bool:
-        self.python_code = re.search(
-            r'<execute_ipython>(.*?)</execute_ipython>', action_str, re.DOTALL
-        )
-        return self.python_code is not None
-
-    def parse(self, action_str: str) -> Action:
-        assert (
-            self.python_code is not None
-        ), 'self.python_code should not be None when parse is called'
-        code_group = self.python_code.group(1).strip()
-        thought = action_str.replace(self.python_code.group(0), '').strip()
-        return IPythonRunCellAction(
-            code=code_group,
-            thought=thought,
-            kernel_init_code=self.jupyter_kernel_init_code,
-        )
-
-
-class CodeActActionParserAgentDelegate(ActionParser):
-    """Parser action:
-    - AgentDelegateAction(agent, inputs) - delegate action for (sub)task
-    """
-
-    def __init__(
-        self,
-    ):
-        self.agent_delegate = None
-
-    def check_condition(self, action_str: str) -> bool:
-        self.agent_delegate = re.search(
-            r'<execute_browse>(.*)</execute_browse>', action_str, re.DOTALL
-        )
-        return self.agent_delegate is not None
-
-    def parse(self, action_str: str) -> Action:
-        assert (
-            self.agent_delegate is not None
-        ), 'self.agent_delegate should not be None when parse is called'
-        thought = action_str.replace(self.agent_delegate.group(0), '').strip()
-        browse_actions = self.agent_delegate.group(1).strip()
-        thought = (
-            f'{thought}\nI should start with: {browse_actions}'
-            if thought
-            else f'I should start with: {browse_actions}'
-        )
-
-        return AgentDelegateAction(
-            agent='BrowsingAgent', thought=thought, inputs={'task': browse_actions}
-        )
-
-
-class CodeActActionParserMessage(ActionParser):
-    """Parser action:
-    - MessageAction(content) - Message action to run (e.g. ask for clarification)
-    """
-
-    def __init__(
-        self,
-    ):
-        pass
-
-    def check_condition(self, action_str: str) -> bool:
-        # We assume the LLM is GOOD enough that when it returns pure natural language
-        # it wants to talk to the user
-        return True
-
-    def parse(self, action_str: str) -> Action:
-        return MessageAction(content=action_str, wait_for_response=True)
-
-
-class CodeActActionParserFileEdit(ActionParser):
-    """Parser action:
-    - FileEditAction(path, content) - edit a file
-    """
-
-    def __init__(self):
-        self.file_edit_match: re.Match | None = None
-
-    def check_condition(self, action_str: str) -> bool:
-        if '<file_edit' not in action_str:
-            return False
-
-        # Updated regex to make start and end optional
-        self.file_edit_match = re.search(
-            r'<file_edit\s+path=(["\']?)(.*?)\1(?:\s+start=(["\']?)(.*?)\3)?(?:\s+end=(["\']?)(.*?)\5)?\s*>(.*?)</file_edit>',
-            action_str,
-            re.DOTALL,
-        )
-
-        if self.file_edit_match is None:
-            logger.error(
-                f'FileEditAction detected but the format is incorrect. Unable to match for <file_edit> in:\n{"-" * 80}\n{action_str}\n{"-" * 80}'
-            )
-            raise LLMMalformedActionError(
-                'FileEditAction detected but the format is incorrect. Usage:\n'
-                '<file_edit path="[path]" start=[start_line] end=[end_line]>\n'
-                '[content_to_edit]\n'
-                '</file_edit>\n'
-            )
-
-        path = self.file_edit_match.group(2)
-        start = self.file_edit_match.group(4)
-        end = self.file_edit_match.group(6)
-
-        if not path:
-            raise LLMMalformedActionError(
-                'FileEditAction detected but no `path` specified. You should specify the path of the file to edit.'
-            )
-
-        if start:
-            try:
-                int(start)
-            except ValueError:
-                raise LLMMalformedActionError(
-                    f'FileEditAction detected but `start` is not a valid integer: {start}'
-                )
-
-        if end:
-            try:
-                int(end)
-            except ValueError:
-                raise LLMMalformedActionError(
-                    f'FileEditAction detected but `end` is not a valid integer: {end}'
-                )
-
-        return True
-
-    def parse(self, action_str: str) -> Action:
-        assert (
-            self.file_edit_match is not None
-        ), 'self.file_edit_match should not be None when parse is called'
-
-        file_path = self.file_edit_match.group(2).strip()
-        start_line = (
-            int(self.file_edit_match.group(4))
-            if self.file_edit_match.group(4)
-            else None
-        )
-        end_line = (
-            int(self.file_edit_match.group(6))
-            if self.file_edit_match.group(6)
-            else None
-        )
-        content = self.file_edit_match.group(7)
-        thought = action_str.replace(self.file_edit_match.group(0), '').strip()
-
-        action = FileEditAction(path=file_path, content=content, thought=thought)
-        if start_line is not None:
-            action.start = start_line
-        if end_line is not None:
-            action.end = end_line
-        return action

openhands/agenthub/codeact_agent/codeact_agent.py

@@ -1,12 +1,10 @@
 import json
 import os
 from collections import deque
-from itertools import islice
 
 from litellm import ModelResponse
 
 import openhands.agenthub.codeact_agent.function_calling as codeact_function_calling
-from openhands.agenthub.codeact_agent.action_parser import CodeActResponseParser
 from openhands.controller.agent import Agent
 from openhands.controller.state.state import State
 from openhands.core.config import AgentConfig
@@ -22,6 +20,7 @@ from openhands.events.action import (
     IPythonRunCellAction,
     MessageAction,
 )
+from openhands.events.event import EventSource
 from openhands.events.observation import (
     AgentDelegateObservation,
     BrowserOutputObservation,
@@ -70,7 +69,6 @@ class CodeActAgent(Agent):
         AgentSkillsRequirement(),
         JupyterRequirement(),
     ]
-    obs_prefix = 'OBSERVATION:\n'
 
     def __init__(
         self,
@@ -85,34 +83,30 @@ class CodeActAgent(Agent):
         super().__init__(llm, config)
         self.reset()
 
-        self.function_calling_active = self.config.function_calling
-        if self.function_calling_active and not self.llm.is_function_calling_active():
-            logger.warning(
-                f'Function calling not supported for model {self.llm.config.model}. '
-                'Disabling function calling.'
+        self.mock_function_calling = False
+        if not self.llm.is_function_calling_active():
+            logger.info(
+                f'Function calling not enabled for model {self.llm.config.model}. '
+                'Mocking function calling via prompting.'
             )
-            self.function_calling_active = False
+            self.mock_function_calling = True
 
-        if self.function_calling_active:
-            self.tools = codeact_function_calling.get_tools(
-                codeact_enable_browsing=self.config.codeact_enable_browsing,
-                codeact_enable_jupyter=self.config.codeact_enable_jupyter,
-                codeact_enable_llm_editor=self.config.codeact_enable_llm_editor,
-            )
-            logger.debug(
-                f'TOOLS loaded for CodeActAgent: {json.dumps(self.tools, indent=2)}'
-            )
-            self.prompt_manager = PromptManager(
-                microagent_dir=os.path.join(os.path.dirname(__file__), 'micro'),
-                prompt_dir=os.path.join(os.path.dirname(__file__), 'prompts', 'tools'),
-            )
-        else:
-            self.action_parser = CodeActResponseParser()
-            self.prompt_manager = PromptManager(
-                microagent_dir=os.path.join(os.path.dirname(__file__), 'micro'),
-                prompt_dir=os.path.join(os.path.dirname(__file__), 'prompts', 'default'),
-                agent_skills_docs=AgentSkillsRequirement.documentation,
-            )
+        # Function calling mode
+        self.tools = codeact_function_calling.get_tools(
+            codeact_enable_browsing=self.config.codeact_enable_browsing,
+            codeact_enable_jupyter=self.config.codeact_enable_jupyter,
+            codeact_enable_llm_editor=self.config.codeact_enable_llm_editor,
+        )
+        logger.debug(
+            f'TOOLS loaded for CodeActAgent: {json.dumps(self.tools, indent=2)}'
+        )
+        self.prompt_manager = PromptManager(
+            microagent_dir=os.path.join(os.path.dirname(__file__), 'micro')
+            if self.config.use_microagents
+            else None,
+            prompt_dir=os.path.join(os.path.dirname(__file__), 'prompts'),
+            disabled_microagents=self.config.disabled_microagents,
+        )
 
         self.pending_actions: deque[Action] = deque()
 
@@ -155,55 +149,54 @@ class CodeActAgent(Agent):
             action,
             (
                 AgentDelegateAction,
-                CmdRunAction,
                 IPythonRunCellAction,
                 FileEditAction,
                 BrowseInteractiveAction,
             ),
-        ) or (isinstance(action, AgentFinishAction) and action.source == 'agent'):
-            if self.function_calling_active:
-                tool_metadata = action.tool_call_metadata
-                assert tool_metadata is not None, (
-                    'Tool call metadata should NOT be None when function calling is enabled. Action: '
-                    + str(action)
-                )
+        ) or (
+            isinstance(action, (AgentFinishAction, CmdRunAction))
+            and action.source == 'agent'
+        ):
+            tool_metadata = action.tool_call_metadata
+            assert tool_metadata is not None, (
+                'Tool call metadata should NOT be None when function calling is enabled. Action: '
+                + str(action)
+            )
 
-                llm_response: ModelResponse = tool_metadata.model_response
-                assistant_msg = llm_response.choices[0].message
-                # Add the LLM message (assistant) that initiated the tool calls
-                # (overwrites any previous message with the same response_id)
-                pending_tool_call_action_messages[llm_response.id] = Message(
-                    role=assistant_msg.role,
-                    # tool call content SHOULD BE a string
-                    content=[TextContent(text=assistant_msg.content or '')]
-                    if assistant_msg.content is not None
-                    else [],
-                    tool_calls=assistant_msg.tool_calls,
-                )
-                return []
-            else:
-                assert not isinstance(action, BrowseInteractiveAction), (
-                    'BrowseInteractiveAction is not supported in non-function calling mode. Action: '
-                    + str(action)
-                )
-                content = [TextContent(text=self.action_parser.action_to_str(action))]
-                return [
-                    Message(
-                        role='user' if action.source == 'user' else 'assistant',
-                        content=content,
-                    )
-                ]
+            llm_response: ModelResponse = tool_metadata.model_response
+            assistant_msg = llm_response.choices[0].message
+            # Add the LLM message (assistant) that initiated the tool calls
+            # (overwrites any previous message with the same response_id)
+            pending_tool_call_action_messages[llm_response.id] = Message(
+                role=assistant_msg.role,
+                # tool call content SHOULD BE a string
+                content=[TextContent(text=assistant_msg.content or '')]
+                if assistant_msg.content is not None
+                else [],
+                tool_calls=assistant_msg.tool_calls,
+            )
+            return []
         elif isinstance(action, MessageAction):
             role = 'user' if action.source == 'user' else 'assistant'
             content = [TextContent(text=action.content or '')]
-            if self.llm.vision_is_active() and action.images_urls:
-                content.append(ImageContent(image_urls=action.images_urls))
+            if self.llm.vision_is_active() and action.image_urls:
+                content.append(ImageContent(image_urls=action.image_urls))
             return [
                 Message(
                     role=role,
                     content=content,
                 )
             ]
+        elif isinstance(action, CmdRunAction) and action.source == 'user':
+            content = [
+                TextContent(text=f'User executed the command:\n{action.command}')
+            ]
+            return [
+                Message(
+                    role='user',
+                    content=content,
+                )
+            ]
         return []
 
     def get_observation_message(
@@ -238,15 +231,21 @@ class CodeActAgent(Agent):
         """
         message: Message
         max_message_chars = self.llm.config.max_message_chars
-        obs_prefix = 'OBSERVATION:\n'
         if isinstance(obs, CmdOutputObservation):
-            text = obs_prefix + truncate_content(
-                obs.content + obs.interpreter_details, max_message_chars
-            )
+            # if it doesn't have tool call metadata, it was triggered by a user action
+            if obs.tool_call_metadata is None:
+                text = truncate_content(
+                    f'\nObserved result of command executed by user:\n{obs.content}',
+                    max_message_chars,
+                )
+            else:
+                text = truncate_content(
+                    obs.content + obs.interpreter_details, max_message_chars
+                )
             text += f'\n[Command finished with exit code {obs.exit_code}]'
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, IPythonRunCellObservation):
-            text = obs_prefix + obs.content
+            text = obs.content
             # replace base64 images with a placeholder
             splitted = text.split('\n')
             for i, line in enumerate(splitted):
@@ -258,22 +257,24 @@ class CodeActAgent(Agent):
             text = truncate_content(text, max_message_chars)
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, FileEditObservation):
-            text = obs_prefix + truncate_content(str(obs), max_message_chars)
+            text = truncate_content(str(obs), max_message_chars)
+            if obs.source == EventSource.USER:
+                text = '[User has edited a file]\n' + text
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, BrowserOutputObservation):
             text = obs.get_agent_obs_text()
             message = Message(
                 role='user',
-                content=[TextContent(text=obs_prefix + text)],
+                content=[TextContent(text=text)],
             )
         elif isinstance(obs, AgentDelegateObservation):
-            text = obs_prefix + truncate_content(
+            text = truncate_content(
                 obs.outputs['content'] if 'content' in obs.outputs else '',
                 max_message_chars,
             )
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, ErrorObservation):
-            text = obs_prefix + truncate_content(obs.content, max_message_chars)
+            text = truncate_content(obs.content, max_message_chars)
             text += '\n[Error occurred in processing last action]'
             message = Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, UserRejectObservation):
@@ -285,19 +286,18 @@ class CodeActAgent(Agent):
             # when the LLM tries to return the next message
             raise ValueError(f'Unknown observation type: {type(obs)}')
 
-        if self.function_calling_active:
-            # Update the message as tool response properly
-            if (tool_call_metadata := obs.tool_call_metadata) is not None:
-                tool_call_id_to_message[tool_call_metadata.tool_call_id] = Message(
-                    role='tool',
-                    content=message.content,
-                    tool_call_id=tool_call_metadata.tool_call_id,
-                    name=tool_call_metadata.function_name,
-                )
-                # No need to return the observation message
-                # because it will be added by get_action_message when all the corresponding
-                # tool calls in the SAME request are processed
-                return []
+        # Update the message as tool response properly
+        if (tool_call_metadata := obs.tool_call_metadata) is not None:
+            tool_call_id_to_message[tool_call_metadata.tool_call_id] = Message(
+                role='tool',
+                content=message.content,
+                tool_call_id=tool_call_metadata.tool_call_id,
+                name=tool_call_metadata.function_name,
+            )
+            # No need to return the observation message
+            # because it will be added by get_action_message when all the corresponding
+            # tool calls in the SAME request are processed
+            return []
 
         return [message]
 
@@ -333,25 +333,14 @@ class CodeActAgent(Agent):
         params: dict = {
             'messages': self.llm.format_messages_for_llm(messages),
         }
-        if self.function_calling_active:
-            params['tools'] = self.tools
-            params['parallel_tool_calls'] = False
-        else:
-            params['stop'] = [
-                '</execute_ipython>',
-                '</execute_bash>',
-                '</execute_browse>',
-                '</file_edit>',
-            ]
+        params['tools'] = self.tools
+        if self.mock_function_calling:
+            params['mock_function_calling'] = True
         response = self.llm.completion(**params)
-
-        if self.function_calling_active:
-            actions = codeact_function_calling.response_to_actions(response)
-            for action in actions:
-                self.pending_actions.append(action)
-            return self.pending_actions.popleft()
-        else:
-            return self.action_parser.parse(response)
+        actions = codeact_function_calling.response_to_actions(response)
+        for action in actions:
+            self.pending_actions.append(action)
+        return self.pending_actions.popleft()
 
     def _get_messages(self, state: State) -> list[Message]:
         """Constructs the message history for the LLM conversation.
@@ -482,7 +471,4 @@ class CodeActAgent(Agent):
                     else:
                         break
 
-        if not self.function_calling_active:
-            self.prompt_manager.add_turns_left_reminder(messages, state)
-
         return messages

openhands/agenthub/codeact_agent/function_calling.py

@@ -53,9 +53,6 @@ _IPYTHON_DESCRIPTION = """Run a cell of Python code in an IPython environment.
 * The assistant should define variables and import packages before using them.
 * The variable defined in the IPython environment will not be available outside the IPython environment (e.g., in terminal).
 """
-# We are not using agentskills's file_ops for viewing files now because StrReplaceEditorTool already supports viewing files
-# """* Apart from the standard Python library, the assistant can also use the following functions (already imported):
-# {AgentSkillsRequirement.documentation}"""
 
 IPythonTool = ChatCompletionToolParam(
     type='function',

openhands/agenthub/codeact_agent/prompts/system_prompt.j2

@@ -1,7 +1,11 @@
 You are OpenHands agent, a helpful AI assistant that can interact with a computer to solve tasks.
+You also observe user actions, like "User has edited a file", and
+infer the user's long-term intentons based on these edits.
+If you think you can help the user finish the task at hand,
+you should offer a suggestion as to how you can
+help, and wait for the user to confirm.
 <IMPORTANT>
 * If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.
 * When configuring git credentials, use "openhands" as the user.name and "openhands@all-hands.dev" as the user.email by default, unless explicitly instructed otherwise.
 * The assistant MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior.
 </IMPORTANT>
-

openhands/agenthub/codeact_agent/system_prompt.j2

@@ -1,5 +1,10 @@
 {% set MINIMAL_SYSTEM_PREFIX %}
-A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed answers to the user's questions.
+A chat between a curious user and an artificial intelligence assistant.
+The assistant gives helpful, detailed answers to the user's questions.
+It also observes user actions, like "User has edited a file", and
+infers the user's long-term intentons based on these edits. If the agent thinks
+it can help the user finish the task at hand, it offers a suggestion as to how it can
+help, and waits for the user to confirm.
 
 [1] The assistant can use a Python environment with <execute_ipython>, e.g.:
 <execute_ipython>
@@ -163,9 +168,6 @@ IMPORTANT: Execute code using <execute_ipython>, <execute_bash>, or <execute_bro
 The assistant should utilize full file paths and the `pwd` command to prevent path-related errors.
 The assistant MUST NOT apologize to the user or thank the user after running commands or editing files. It should only address the user in response to an explicit message from the user, or to ask for more information.
 The assistant MUST NOT push any changes to GitHub unless explicitly requested to do so.
-The assistant MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior, or
-to describe precisely how to apply proposed edits. Comments about applying edits should always have blank lines above
-and below.
 
 {% endset %}
 {# Combine all parts without newlines between them #}

openhands/agenthub/codeact_agent/user_prompt.j2

@@ -197,6 +197,58 @@ Observation:
 (this is the end of the file)
 [File updated (edited at line 7). Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
 
+USER:
+[User has edited a file]
+OBSERVATION:
+[Existing file app.py is edited with 1 changes.]
+[begin of edit 1 / 1]
+(content before edit)
+1|from flask import Flask
+2|app = Flask(__name__)
+3|
+4|@app.route('/')
+5|def index():
+6|    numbers = list(range(1, 11))
+7|    ret = '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
+8|    return ret
+9|
+10|if __name__ == '__main__':
+11|    app.run(port=5000)
+(content after edit)
+1|from flask import Flask
+2|app = Flask(__name__)
+3|
+4|@app.route('/')
+5|def index():
+6|    numbers = list(range(1, 11))
+7|    ret = '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
+8|    return ret
+9|
++10|@app.route('/reversed')
++11|
+12|if __name__ == '__main__':
+13|    app.run(port=5000)
+
+[end of edit 1 / 1]
+
+ASSISTANT:
+I see you've started to add a new route '/reversed'. Would you like me to
+implement the logic for this route by reversing the numbers?
+
+USER:
+continue
+
+ASSISTANT:
+I should edit the file to display the reversed numbers in a table format. I should include correct indentation. Let me update the file:
+<file_edit path="/workspace/app.py" start=10 end=12>
+@app.route('/reversed')
+def index():
+    numbers = reversed(list(range(1, 11)))
+    ret = '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
+    return ret
+</file_edit>
+
+
 ASSISTANT:
 Running the updated file:
 <execute_bash>
@@ -215,5 +267,12 @@ The server is running on port 5000 with PID 126. You can access the list of numb
 {% endset %}
 Here is an example of how you can interact with the environment for task solving:
 {{ DEFAULT_EXAMPLE }}
+{% if micro_agent %}
+--- BEGIN OF GUIDELINE ---
+The following information may assist you in completing your task:
+
+{{ micro_agent }}
+--- END OF GUIDELINE ---
+{% endif %}
 
 NOW, LET'S START!

openhands/agenthub/codeact_swe_agent/codeact_swe_agent.py

@@ -95,9 +95,9 @@ class CodeActSWEAgent(Agent):
             if (
                 self.llm.vision_is_active()
                 and isinstance(action, MessageAction)
-                and action.images_urls
+                and action.image_urls
             ):
-                content.append(ImageContent(image_urls=action.images_urls))
+                content.append(ImageContent(image_urls=action.image_urls))
 
             return Message(
                 role='user' if action.source == 'user' else 'assistant', content=content

openhands/controller/agent_controller.py

@@ -5,12 +5,14 @@ import traceback
 from typing import Callable, ClassVar, Type
 
 import litellm
+from litellm.exceptions import ContextWindowExceededError
 
 from openhands.controller.agent import Agent
 from openhands.controller.state.state import State, TrafficControlState
 from openhands.controller.stuck import StuckDetector
 from openhands.core.config import AgentConfig, LLMConfig
 from openhands.core.exceptions import (
+    FunctionCallValidationError,
     LLMMalformedActionError,
     LLMNoActionError,
     LLMResponseError,
@@ -42,7 +44,7 @@ from openhands.events.observation import (
 )
 from openhands.events.serialization.event import truncate_content
 from openhands.llm.llm import LLM
-from openhands.runtime.utils.shutdown_listener import should_continue
+from openhands.utils.shutdown_listener import should_continue
 
 # note: RESUME is only available on web GUI
 TRAFFIC_CONTROL_REMINDER = (
@@ -63,6 +65,7 @@ class AgentController:
     parent: 'AgentController | None' = None
     delegate: 'AgentController | None' = None
     _pending_action: Action | None = None
+    _closed: bool = False
     filter_out: ClassVar[tuple[type[Event], ...]] = (
         NullAction,
         NullObservation,
@@ -158,6 +161,7 @@ class AgentController:
 
         # unsubscribe from the event stream
         self.event_stream.unsubscribe(EventStreamSubscriber.AGENT_CONTROLLER, self.id)
+        self._closed = True
 
     def log(self, level: str, message: str, extra: dict | None = None):
         """Logs a message to the agent controller's logger.
@@ -192,6 +196,8 @@ class AgentController:
 
         self.log('info', 'Starting step loop...')
         while should_continue():
+            if self._closed:
+                break
             try:
                 await self._step()
             except asyncio.CancelledError:
@@ -276,6 +282,11 @@ class AgentController:
             if self.state.agent_state == AgentState.USER_REJECTED:
                 await self.set_agent_state_to(AgentState.AWAITING_USER_INPUT)
             return
+
+        if observation.source == EventSource.USER:
+            if self.state.agent_state == AgentState.AWAITING_USER_INPUT:
+                await self.set_agent_state_to(AgentState.RUNNING)
+
         elif isinstance(observation, ErrorObservation):
             if self.state.agent_state == AgentState.ERROR:
                 self.state.metrics.merge(self.state.local_metrics)
@@ -477,7 +488,12 @@ class AgentController:
             action = self.agent.step(self.state)
             if action is None:
                 raise LLMNoActionError('No action was returned')
-        except (LLMMalformedActionError, LLMNoActionError, LLMResponseError) as e:
+        except (
+            LLMMalformedActionError,
+            LLMNoActionError,
+            LLMResponseError,
+            FunctionCallValidationError,
+        ) as e:
             self.event_stream.add_event(
                 ErrorObservation(
                     content=str(e),
@@ -485,6 +501,15 @@ class AgentController:
                 EventSource.AGENT,
             )
             return
+        except ContextWindowExceededError:
+            # When context window is exceeded, keep roughly half of agent interactions
+            self.state.history = self._apply_conversation_window(self.state.history)
+
+            # Save the ID of the first event in our truncated history for future reloading
+            if self.state.history:
+                self.state.start_id = self.state.history[0].id
+            # Don't add error event - let the agent retry with reduced context
+            return
 
         if action.runnable:
             if self.state.confirmation_mode and (
@@ -659,6 +684,12 @@ class AgentController:
         - For delegate events (between AgentDelegateAction and AgentDelegateObservation):
             - Excludes all events between the action and observation
             - Includes the delegate action and observation themselves
+
+        The history is loaded in two parts if truncation_id is set:
+        1. First user message from start_id onwards
+        2. Rest of history from truncation_id to the end
+
+        Otherwise loads normally from start_id.
         """
 
         # define range of events to fetch
@@ -680,8 +711,33 @@ class AgentController:
             self.state.history = []
             return
 
-        # Get all events, filtering out backend events and hidden events
-        events = list(
+        events: list[Event] = []
+
+        # If we have a truncation point, get first user message and then rest of history
+        if hasattr(self.state, 'truncation_id') and self.state.truncation_id > 0:
+            # Find first user message from stream
+            first_user_msg = next(
+                (
+                    e
+                    for e in self.event_stream.get_events(
+                        start_id=start_id,
+                        end_id=end_id,
+                        reverse=False,
+                        filter_out_type=self.filter_out,
+                        filter_hidden=True,
+                    )
+                    if isinstance(e, MessageAction) and e.source == EventSource.USER
+                ),
+                None,
+            )
+            if first_user_msg:
+                events.append(first_user_msg)
+
+            # the rest of the events are from the truncation point
+            start_id = self.state.truncation_id
+
+        # Get rest of history
+        events_to_add = list(
             self.event_stream.get_events(
                 start_id=start_id,
                 end_id=end_id,
@@ -690,6 +746,7 @@ class AgentController:
                 filter_hidden=True,
             )
         )
+        events.extend(events_to_add)
 
         # Find all delegate action/observation pairs
         delegate_ranges: list[tuple[int, int]] = []
@@ -744,6 +801,92 @@ class AgentController:
         # make sure history is in sync
         self.state.start_id = start_id
 
+    def _apply_conversation_window(self, events: list[Event]) -> list[Event]:
+        """Cuts history roughly in half when context window is exceeded, preserving action-observation pairs
+        and ensuring the first user message is always included.
+
+        The algorithm:
+        1. Cut history in half
+        2. Check first event in new history:
+           - If Observation: find and include its Action
+           - If MessageAction: ensure its related Action-Observation pair isn't split
+        3. Always include the first user message
+
+        Args:
+            events: List of events to filter
+
+        Returns:
+            Filtered list of events keeping newest half while preserving pairs
+        """
+        if not events:
+            return events
+
+        # Find first user message - we'll need to ensure it's included
+        first_user_msg = next(
+            (
+                e
+                for e in events
+                if isinstance(e, MessageAction) and e.source == EventSource.USER
+            ),
+            None,
+        )
+
+        # cut in half
+        mid_point = max(1, len(events) // 2)
+        kept_events = events[mid_point:]
+
+        # Handle first event in truncated history
+        if kept_events:
+            i = 0
+            while i < len(kept_events):
+                first_event = kept_events[i]
+                if isinstance(first_event, Observation) and first_event.cause:
+                    # Find its action and include it
+                    matching_action = next(
+                        (
+                            e
+                            for e in reversed(events[:mid_point])
+                            if isinstance(e, Action) and e.id == first_event.cause
+                        ),
+                        None,
+                    )
+                    if matching_action:
+                        kept_events = [matching_action] + kept_events
+                    else:
+                        self.log(
+                            'warning',
+                            f'Found Observation without matching Action at id={first_event.id}',
+                        )
+                        # drop this observation
+                        kept_events = kept_events[1:]
+                    break
+
+                elif isinstance(first_event, MessageAction) or (
+                    isinstance(first_event, Action)
+                    and first_event.source == EventSource.USER
+                ):
+                    # if it's a message action or a user action, keep it and continue to find the next event
+                    i += 1
+                    continue
+
+                else:
+                    # if it's an action with source == EventSource.AGENT, we're good
+                    break
+
+        # Save where to continue from in next reload
+        if kept_events:
+            self.state.truncation_id = kept_events[0].id
+
+        # Ensure first user message is included
+        if first_user_msg and first_user_msg not in kept_events:
+            kept_events = [first_user_msg] + kept_events
+
+        # start_id points to first user message
+        if first_user_msg:
+            self.state.start_id = first_user_msg.id
+
+        return kept_events
+
     def _is_stuck(self):
         """Checks if the agent or its delegate is stuck in a loop.
 

openhands/controller/state/state.py

@@ -92,6 +92,8 @@ class State:
     # start_id and end_id track the range of events in history
     start_id: int = -1
     end_id: int = -1
+    # truncation_id tracks where to load history after context window truncation
+    truncation_id: int = -1
     almost_stuck: int = 0
     delegates: dict[tuple[int, int], tuple[str, str]] = field(default_factory=dict)
     # NOTE: This will never be used by the controller, but it can be used by different
@@ -149,7 +151,7 @@ class State:
         for event in reversed(self.history):
             if isinstance(event, MessageAction) and event.source == 'user':
                 last_user_message = event.content
-                last_user_message_image_urls = event.images_urls
+                last_user_message_image_urls = event.image_urls
             elif isinstance(event, AgentFinishAction):
                 if last_user_message is not None:
                     return last_user_message, None

openhands/core/cli.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import os
 import sys
 from typing import Type
 from uuid import uuid4
@@ -38,6 +39,8 @@ from openhands.storage import get_file_store
 
 
 def display_message(message: str):
+    if not message:
+        return
     print(colored('🤖 ' + message + '\n', 'yellow'))
 
 
@@ -56,7 +59,8 @@ def display_command_output(output: str):
 
 
 def display_file_edit(event: FileEditAction | FileEditObservation):
-    print(colored(str(event), 'green'))
+    # print(colored(str(event), 'green'))
+    pass
 
 
 def display_event(event: Event):
@@ -66,14 +70,24 @@ def display_event(event: Event):
     if isinstance(event, MessageAction):
         if event.source == EventSource.AGENT:
             display_message(event.content)
-    if isinstance(event, CmdRunAction):
+    elif isinstance(event, CmdRunAction):
         display_command(event.command)
-    if isinstance(event, CmdOutputObservation):
+    elif isinstance(event, CmdOutputObservation):
         display_command_output(event.content)
-    if isinstance(event, FileEditAction):
-        display_file_edit(event)
-    if isinstance(event, FileEditObservation):
+    elif isinstance(event, FileEditAction):
         display_file_edit(event)
+    elif isinstance(event, FileEditObservation):
+        if event.source == EventSource.ENVIRONMENT:
+            # For file watcher events, use a different color and format
+            if not event.prev_exist:
+                print(colored(f'📝 File created: {event.path}', 'cyan'))
+            elif event.new_content == '':
+                print(colored(f'🗑️  File deleted: {event.path}', 'red'))
+            else:
+                print(colored(f'✏️  File modified: {event.path}', 'yellow'))
+        else:
+            # For regular file edits, use the standard display
+            display_file_edit(event)
 
 
 async def main():
@@ -89,6 +103,15 @@ async def main():
         help='Show the version number and exit',
         default=None,
     )
+    # Add the watch directory argument
+    parser.add_argument(
+        '-w',
+        '--watch',
+        type=str,
+        help='Directory to watch for changes',
+        metavar='DIR',
+        default=None,
+    )
     args = parser.parse_args()
 
     if args.version:
@@ -110,12 +133,26 @@ async def main():
     file_store = get_file_store(config.file_store, config.file_store_path)
     event_stream = EventStream(sid, file_store)
 
+    if args.watch:
+        from openhands.intent.watch import FileWatcher
+
+        watch_dir = os.path.abspath(args.watch)
+        if not os.path.isdir(watch_dir):
+            print(
+                f"Error: Watch directory '{args.watch}' does not exist or is not a directory"
+            )
+            return
+        print(f'Starting file watcher for directory: {watch_dir}')
+        file_watcher = FileWatcher(directory=watch_dir, event_stream=event_stream)
+        file_watcher.start()
+
     runtime_cls = get_runtime_cls(config.runtime)
     runtime: Runtime = runtime_cls(  # noqa: F841
         config=config,
         event_stream=event_stream,
         sid=sid,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=True,
     )
 
     controller = AgentController(
@@ -123,11 +160,12 @@ async def main():
         max_iterations=config.max_iterations,
         max_budget_per_task=config.max_budget_per_task,
         agent_to_llm_config=config.get_agent_to_llm_config_map(),
+        agent_configs=config.get_agent_configs(),
         event_stream=event_stream,
     )
 
     async def prompt_for_next_task():
-        # Run input() in a thread pool to avoid blocking the event loop
+        await controller.set_agent_state_to(AgentState.AWAITING_USER_INPUT)
         loop = asyncio.get_event_loop()
         next_message = await loop.run_in_executor(
             None, lambda: input('How can I help? >> ')
@@ -161,6 +199,11 @@ async def main():
         controller, runtime, [AgentState.STOPPED, AgentState.ERROR]
     )
 
+    # Stop file watcher if it was started
+    if args.watch and 'file_watcher' in locals():
+        print('Stopping file watcher...')
+        file_watcher.stop()
+
 
 if __name__ == '__main__':
     loop = asyncio.new_event_loop()

openhands/core/config/agent_config.py

@@ -16,9 +16,10 @@ class AgentConfig:
         memory_enabled: Whether long-term memory (embeddings) is enabled.
         memory_max_threads: The maximum number of threads indexing at the same time for embeddings.
         llm_config: The name of the llm config to use. If specified, this will override global llm config.
+        use_microagents: Whether to use microagents at all. Default is True.
+        disabled_microagents: A list of microagents to disable. Default is None.
     """
 
-    function_calling: bool = True
     codeact_enable_browsing: bool = True
     codeact_enable_llm_editor: bool = False
     codeact_enable_jupyter: bool = True
@@ -26,6 +27,8 @@ class AgentConfig:
     memory_enabled: bool = False
     memory_max_threads: int = 3
     llm_config: str | None = None
+    use_microagents: bool = True
+    disabled_microagents: list[str] | None = None
 
     def defaults_to_dict(self) -> dict:
         """Serialize fields to a dict for the frontend, including type hints, defaults, and whether it's optional."""

openhands/core/config/sandbox_config.py

@@ -36,7 +36,7 @@ class SandboxConfig:
 
     remote_runtime_api_url: str = 'http://localhost:8000'
     local_runtime_url: str = 'http://localhost'
-    keep_remote_runtime_alive: bool = True
+    keep_runtime_alive: bool = True
     api_key: str | None = None
     base_container_image: str = 'nikolaik/python-nodejs:python3.12-nodejs22'  # default to nikolaik/python-nodejs:python3.12-nodejs22 for eventstream runtime
     runtime_container_image: str | None = None

openhands/core/exceptions.py

@@ -94,3 +94,23 @@ class CloudFlareBlockageError(Exception):
     """Exception raised when a request is blocked by CloudFlare."""
 
     pass
+
+
+class FunctionCallConversionError(Exception):
+    """Exception raised when FunctionCallingConverter failed to convert a non-function call message to a function call message.
+
+    This typically happens when there's a malformed message (e.g., missing <function=...> tags). But not due to LLM output.
+    """
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class FunctionCallValidationError(Exception):
+    """Exception raised when FunctionCallingConverter failed to validate a function call message.
+
+    This typically happens when the LLM outputs unrecognized function call / parameter names / values.
+    """
+
+    def __init__(self, message):
+        super().__init__(message)

openhands/core/logger.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 import os
 import re
@@ -45,6 +46,29 @@ LOG_COLORS: Mapping[str, ColorType] = {
 }
 
 
+class NoColorFormatter(logging.Formatter):
+    """Formatter for non-colored logging in files."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        # Create a deep copy of the record to avoid modifying the original
+        new_record: logging.LogRecord = copy.deepcopy(record)
+        # Strip ANSI color codes from the message
+        new_record.msg = strip_ansi(new_record.msg)
+
+        return super().format(new_record)
+
+
+def strip_ansi(s: str) -> str:
+    """
+    Removes ANSI escape sequences from str, as defined by ECMA-048 in
+    http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf
+    # https://github.com/ewen-lbh/python-strip-ansi/blob/master/strip_ansi/__init__.py
+    """
+    pattern = re.compile(r'\x1B\[\d+(;\d+){0,2}m')
+    stripped = pattern.sub('', s)
+    return stripped
+
+
 class ColoredFormatter(logging.Formatter):
     def format(self, record):
         msg_type = record.__dict__.get('msg_type')
@@ -70,7 +94,7 @@ class ColoredFormatter(logging.Formatter):
         return super().format(record)
 
 
-file_formatter = logging.Formatter(
+file_formatter = NoColorFormatter(
     '%(asctime)s - %(name)s:%(levelname)s: %(filename)s:%(lineno)s - %(message)s',
     datefmt='%H:%M:%S',
 )

openhands/core/main.py

@@ -35,8 +35,8 @@ class FakeUserResponseFunc(Protocol):
     def __call__(
         self,
         state: State,
-        encapsulate_solution: bool = ...,
-        try_parse: Callable[[Action], str] = ...,
+        encapsulate_solution: bool = False,
+        try_parse: Callable[[Action | None], str] | None = None,
     ) -> str: ...
 
 
@@ -54,11 +54,14 @@ def read_task_from_stdin() -> str:
 def create_runtime(
     config: AppConfig,
     sid: str | None = None,
+    headless_mode: bool = True,
 ) -> Runtime:
     """Create a runtime for the agent to run on.
 
     config: The app config.
     sid: The session id.
+    headless_mode: Whether the agent is run in headless mode. `create_runtime` is typically called within evaluation scripts,
+        where we don't want to have the VSCode UI open, so it defaults to True.
     """
     # if sid is provided on the command line, use it as the name of the event stream
     # otherwise generate it on the basis of the configured jwt_secret
@@ -80,6 +83,7 @@ def create_runtime(
         event_stream=event_stream,
         sid=session_id,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=headless_mode,
     )
 
     return runtime
@@ -122,7 +126,7 @@ async def run_controller(
     sid = sid or generate_sid(config)
 
     if runtime is None:
-        runtime = create_runtime(config, sid=sid)
+        runtime = create_runtime(config, sid=sid, headless_mode=headless_mode)
         await runtime.connect()
 
     event_stream = runtime.event_stream

openhands/core/message.py

@@ -72,7 +72,12 @@ class Message(BaseModel):
         # - into a single string: for providers that don't support list of content items (e.g. no vision, no tool calls)
         # - into a list of content items: the new APIs of providers with vision/prompt caching/tool calls
         # NOTE: remove this when litellm or providers support the new API
-        if self.cache_enabled or self.vision_enabled or self.tool_call_id is not None:
+        if (
+            self.cache_enabled
+            or self.vision_enabled
+            or self.tool_call_id is not None
+            or self.tool_calls is not None
+        ):
             return self._list_serializer()
         return self._string_serializer()
 
@@ -98,6 +103,13 @@ class Message(BaseModel):
                 content.extend(d)
 
         ret: dict = {'content': content, 'role': self.role}
+        # pop content if it's empty
+        if not content or (
+            len(content) == 1
+            and content[0]['type'] == 'text'
+            and content[0]['text'] == ''
+        ):
+            ret.pop('content')
 
         if role_tool_with_prompt_caching:
             ret['cache_control'] = {'type': 'ephemeral'}

openhands/events/action/message.py

@@ -7,7 +7,7 @@ from openhands.events.action.action import Action, ActionSecurityRisk
 @dataclass
 class MessageAction(Action):
     content: str
-    images_urls: list[str] | None = None
+    image_urls: list[str] | None = None
     wait_for_response: bool = False
     action: str = ActionType.MESSAGE
     security_risk: ActionSecurityRisk | None = None
@@ -16,10 +16,18 @@ class MessageAction(Action):
     def message(self) -> str:
         return self.content
 
+    @property
+    def images_urls(self):
+        # Deprecated alias for backward compatibility
+        return self.image_urls
+
+    @images_urls.setter
+    def images_urls(self, value):
+        self.image_urls = value
     def __str__(self) -> str:
         ret = f'**MessageAction** (source={self.source})\n'
         ret += f'CONTENT: {self.content}'
-        if self.images_urls:
-            for url in self.images_urls:
+        if self.image_urls:
+            for url in self.image_urls:
                 ret += f'\nIMAGE_URL: {url}'
         return ret

openhands/events/serialization/action.py

@@ -66,6 +66,10 @@ def action_from_dict(action: dict) -> Action:
     if is_confirmed is not None:
         args['confirmation_state'] = is_confirmed
 
+    # images_urls has been renamed to image_urls
+    if 'images_urls' in args:
+        args['image_urls'] = args.pop('images_urls')
+        
     try:
         decoded_action = action_class(**args)
         if 'timeout' in action:

openhands/events/serialization/event.py

@@ -101,7 +101,7 @@ def event_to_memory(event: 'Event', max_message_chars: int) -> dict:
     d.pop('cause', None)
     d.pop('timestamp', None)
     d.pop('message', None)
-    d.pop('images_urls', None)
+    d.pop('image_urls', None)
 
     # runnable actions have some extra fields used in the BE/FE, which should not be sent to the LLM
     if 'args' in d:

openhands/events/stream.py

@@ -9,9 +9,9 @@ from openhands.core.logger import openhands_logger as logger
 from openhands.core.utils import json
 from openhands.events.event import Event, EventSource
 from openhands.events.serialization.event import event_from_dict, event_to_dict
-from openhands.runtime.utils.shutdown_listener import should_continue
 from openhands.storage import FileStore
 from openhands.utils.async_utils import call_sync_from_async
+from openhands.utils.shutdown_listener import should_continue
 
 
 class EventStreamSubscriber(str, Enum):

openhands/intent/__init__.py

@@ -0,0 +1 @@
+"""Intent detection and processing for OpenHands."""