reduce the info we put into a webpage

Co-authored-by: adrianamorenogt <adrianamorenogutierrez@gmail.com>

.github/ISSUE_TEMPLATE/bug_template.yml

@@ -31,6 +31,8 @@ body:
       options:
         - Docker command in README
         - Development workflow
+        - app.all-hands.dev
+        - Other
       default: 0
 
   - type: input

.github/workflows/ghcr-build.yml

@@ -399,3 +399,49 @@ jobs:
         run: |
           echo "Some runtime tests failed or were cancelled"
           exit 1
+  update_pr_description:
+    name: Update PR Description
+    if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'
+    needs: [ghcr_build_runtime]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: short_sha
+        run: echo "SHORT_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
+
+      - name: Update PR Description
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+          SHORT_SHA: ${{ steps.short_sha.outputs.SHORT_SHA }}
+        run: |
+          echo "updating PR description"
+          DOCKER_RUN_COMMAND="docker run -it --rm \
+            -p 3000:3000 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            --add-host host.docker.internal:host-gateway \
+            -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:$SHORT_SHA-nikolaik \
+            --name openhands-app-$SHORT_SHA \
+            docker.all-hands.dev/all-hands-ai/openhands:$SHORT_SHA"
+
+          PR_BODY=$(gh pr view $PR_NUMBER --json body --jq .body)
+
+          if echo "$PR_BODY" | grep -q "To run this PR locally, use the following command:"; then
+            UPDATED_PR_BODY=$(echo "${PR_BODY}" | sed -E "s|docker run -it --rm.*|$DOCKER_RUN_COMMAND|")
+          else
+            UPDATED_PR_BODY="${PR_BODY}
+
+          ---
+
+          To run this PR locally, use the following command:
+          \`\`\`
+          $DOCKER_RUN_COMMAND
+          \`\`\`"
+          fi
+
+          echo "updated body: $UPDATED_PR_BODY"
+          gh pr edit $PR_NUMBER --body "$UPDATED_PR_BODY"

.github/workflows/openhands-resolver.yml

@@ -3,11 +3,13 @@ name: Resolve Issues with OpenHands
 on:
   issues:
     types: [labeled]
+  pull_request:
+    types: [labeled]
 
 jobs:
   call-openhands-resolver:
     uses: All-Hands-AI/openhands-resolver/.github/workflows/openhands-resolver.yml@main
     if: github.event.label.name == 'fix-me'
     with:
-      issue_number: ${{ github.event.issue.number }}
+      max_iterations: 50
     secrets: inherit

.gitignore

@@ -174,6 +174,7 @@ evaluation/bird/data
 evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
+evaluation/scienceagentbench/benchmark
 
 # frontend
 

Development.md

@@ -100,7 +100,7 @@ poetry run pytest ./tests/unit/test_*.py
 ### 9. Use existing Docker image
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image. Follow these steps:
 1. Set the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
-2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.9-nikolaik
+2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.12-nikolaik
 
 ## Develop inside Docker container
 

README.md

@@ -12,7 +12,7 @@
   <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
   <br/>
-  <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
+  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
   <br/>
@@ -33,35 +33,34 @@ Learn more at [docs.all-hands.dev](https://docs.all-hands.dev), or jump to the [
 
 ## ⚡ Quick Start
 
-The easiest way to run OpenHands is in Docker. You can change `WORKSPACE_BASE` below to
-point OpenHands to existing code that you'd like to modify.
-
+The easiest way to run OpenHands is in Docker.
 See the [Installation](https://docs.all-hands.dev/modules/usage/installation) guide for
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik
 
-docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11
+    docker.all-hands.dev/all-hands-ai/openhands:0.12
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
 
-You'll need a model provider and API key.
-[Anthropic's Claude 3.5 Sonnet (`anthropic/claude-3-5-sonnet-20241022`)](https://www.anthropic.com/api)
+Finally, you'll need a model provider and API key.
+[Anthropic's Claude 3.5 Sonnet](https://www.anthropic.com/api) (`anthropic/claude-3-5-sonnet-20241022`)
 works best, but you have [many options](https://docs.all-hands.dev/modules/usage/llms).
 
 ---
 
 You can also [connect OpenHands to your local filesystem](https://docs.all-hands.dev/modules/usage/runtimes),
 run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
-or interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode).
+interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode),
+or run it on tagged issues with [a github action](https://github.com/All-Hands-AI/OpenHands-resolver).
 
 Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
@@ -94,7 +93,7 @@ For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
 Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
 Let's make software engineering better together!
 
-- [Slack workspace](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) - Here we talk about research, architecture, and future development.
+- [Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
 - [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
 
 ## 📈 Progress

compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.12-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

config.template.toml

@@ -32,7 +32,8 @@ workspace_base = "./workspace"
 # Enable saving and restoring the session when run from CLI
 #enable_cli_session = false
 
-# Path to store trajectories
+# Path to store trajectories, can be a folder or a file
+# If it's a folder, the session id will be used as the file name
 #trajectories_path="./trajectories"
 
 # File store path

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.12-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -161,7 +161,7 @@ Pour créer un workflow d'évaluation pour votre benchmark, suivez ces étapes :
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -260,7 +260,7 @@ def codeact_user_response(state: State | None) -> str:
         # vérifier si l'agent a essayé de parler à l'utilisateur 3 fois, si oui, faire savoir à l'agent qu'il peut abandonner
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:
@@ -279,4 +279,3 @@ Cette fonction fait ce qui suit :
 3. Si l'agent a fait plusieurs tentatives, il lui donne la possibilité d'abandonner
 
 En utilisant cette fonction, vous pouvez garantir un comportement cohérent sur plusieurs exécutions d'évaluation et empêcher l'agent de rester bloqué en attendant une entrée humaine.
-

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -158,7 +158,7 @@ OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -257,7 +257,7 @@ def codeact_user_response(state: State | None) -> str:
         # 检查代理是否已尝试与用户对话 3 次，如果是，让代理知道它可以放弃
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -58,4 +58,3 @@ docker run -it \
     ghcr.io/all-hands-ai/openhands:0.11 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
-

docs/modules/usage/how-to/cli-mode.md

@@ -50,6 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -58,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.12 \
     python -m openhands.core.cli
 ```
 
@@ -107,4 +108,3 @@ Expected Output:
 ```bash
 🤖 An error occurred. Please try again.
 ```
-

docs/modules/usage/how-to/evaluation-harness.md

@@ -158,7 +158,7 @@ To create an evaluation workflow for your benchmark, follow these steps:
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -257,7 +257,7 @@ def codeact_user_response(state: State | None) -> str:
         # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:

docs/modules/usage/how-to/gui-mode.md

@@ -19,6 +19,15 @@ OpenHands provides a user-friendly Graphical User Interface (GUI) mode for inter
 3. Enter the corresponding `API Key` for your chosen provider.
 4. Click "Save" to apply the settings.
 
+### GitHub Token Setup
+
+OpenHands automatically exports a `GITHUB_TOKEN` to the shell environment if it is available. This can happen in two ways:
+
+1. Locally (OSS): The user directly inputs their GitHub token.
+2. Online (SaaS): The token is obtained through GitHub OAuth authentication.
+
+When you reach the `/app` route, the app checks if a token is present. If it finds one, it sets it in the environment for the agent to use.
+
 ### Advanced Settings
 
 1. Toggle `Advanced Options` to access additional settings.

docs/modules/usage/how-to/headless-mode.md

@@ -44,6 +44,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -52,7 +53,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.12 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
-

docs/modules/usage/installation.mdx

@@ -11,15 +11,15 @@
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11
+    docker.all-hands.dev/all-hands-ai/openhands:0.12
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/modules/usage/runtimes.md

@@ -60,7 +60,6 @@ docker run # ...
     -e SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.app.all-hands.dev" \
     -e SANDBOX_API_KEY="your-all-hands-api-key" \
     -e SANDBOX_KEEP_REMOTE_RUNTIME_ALIVE="true" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
     # ...
 ```
 
@@ -75,5 +74,4 @@ docker run # ...
     -e RUNTIME=modal \
     -e MODAL_API_TOKEN_ID="your-id" \
     -e MODAL_API_TOKEN_SECRET="your-secret" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
 ```

docs/package.json

@@ -15,10 +15,10 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.5.2",
-    "@docusaurus/plugin-content-pages": "^3.5.2",
-    "@docusaurus/preset-classic": "^3.5.2",
-    "@docusaurus/theme-mermaid": "^3.5.2",
+    "@docusaurus/core": "^3.6.0",
+    "@docusaurus/plugin-content-pages": "^3.6.0",
+    "@docusaurus/preset-classic": "^3.6.0",
+    "@docusaurus/theme-mermaid": "^3.6.0",
     "@mdx-js/react": "^3.1.0",
     "clsx": "^2.0.0",
     "prism-react-renderer": "^2.4.0",
@@ -29,7 +29,7 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.1",
-    "@docusaurus/tsconfig": "^3.5.2",
+    "@docusaurus/tsconfig": "^3.6.0",
     "@docusaurus/types": "^3.5.1",
     "typescript": "~5.6.3"
   },

evaluation/EDA/run_infer.py

@@ -8,6 +8,7 @@ from evaluation.EDA.game import Q20Game, Q20GameCelebrity
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -34,7 +35,7 @@ def codeact_user_response_eda(state: State) -> str:
 
     # retrieve the latest model message from history
     if state.history:
-        model_guess = state.history.get_last_agent_message()
+        model_guess = state.get_last_agent_message()
 
     assert game is not None, 'Game is not initialized.'
     msg = game.generate_user_response(model_guess)
@@ -139,7 +140,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    final_message = state.history.get_last_agent_message()
+    final_message = state.get_last_agent_message()
 
     logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
     test_result = game.reward()
@@ -148,7 +149,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/README.md

@@ -84,4 +84,3 @@ all the preprocessing/evaluation/analysis scripts.
 - Raw data and experimental records should not be stored within this repo.
 - For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.
 - Important data files of manageable size and analysis scripts (e.g., jupyter notebooks) can be directly uploaded to this repo.
-

evaluation/agent_bench/run_infer.py

@@ -16,6 +16,7 @@ from evaluation.agent_bench.helper import (
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -242,7 +243,7 @@ def process_instance(
         raw_ans = ''
 
         # retrieve the last agent message or thought
-        for event in state.history.get_events(reverse=True):
+        for event in reversed(state.history):
             if event.source == 'agent':
                 if isinstance(event, AgentFinishAction):
                     raw_ans = event.thought
@@ -271,7 +272,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     metrics = state.metrics.get() if state.metrics else None
 

evaluation/aider_bench/run_infer.py

@@ -15,6 +15,7 @@ from evaluation.aider_bench.helper import (
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -250,7 +251,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
     metrics = state.metrics.get() if state.metrics else None
 
     # Save the output

evaluation/biocoder/run_infer.py

@@ -13,6 +13,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -299,7 +300,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     test_result['generated'] = test_result['metadata']['1_copy_change_code']
 

evaluation/bird/run_infer.py

@@ -16,6 +16,7 @@ from tqdm import tqdm
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -46,7 +47,7 @@ def codeact_user_response(state: State) -> str:
         # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) > 2:
@@ -431,7 +432,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/browsing_delegation/run_infer.py

@@ -9,6 +9,7 @@ from datasets import load_dataset
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -89,7 +90,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # find the last delegate action
     last_delegate_action = None

evaluation/discoverybench/README.md

@@ -0,0 +1,37 @@
+# DiscoveryBench with OpenHands
+
+[DiscoveryBench](https://github.com/allenai/discoverybench/) [(Paper)](https://arxiv.org/abs/2407.01725v1) contains 264 tasks collected across 6 diverse domains, such as biology, economics, and sociology. It incorporates discovery workflows from published papers to approximate the real-world challenges faced by researchers.
+
+<p align="center">
+  <a href="[https://github.com/allenai/discoverybench](https://github.com/allenai/discoverybench)">
+    <img src="https://raw.githubusercontent.com/allenai/discoverybench/refs/heads/main/assets/discoverybench-openhands-teaser.png" width="100%" alt="DiscoveryBench Background" />
+  </a>
+</p>
+
+
+## Setup Environment and LLM Configuration
+
+1. Please follow instructions mentioned [here](https://github.com/openlocus/OpenHands/blob/discoverybench-openhands-integration/evaluation/README.md#setup) to setup OpenHands development environment and LLMs locally
+
+2. Execute the bash script to start DiscoveryBench Evaluation
+
+```
+./evaluation/discoverybench/scripts/run_infer.sh [YOUR MODEL CONFIG]
+```
+Replace `[YOUR MODEL CONFIG]` with any model the model that you have set up in `config.toml`
+
+
+## Run Inference on DiscoveryBench Instances
+
+When the `run_infer.sh` script is started, it will automatically pull the latest DiscoveryBench instances & set up the agent environment. The OpenHands agent is invoked to process the task within this environment, producing a hypothesis. We then evaluate it against the “gold” hypothesis provided by DiscoveryBench. The evaluation result, along with the agent chat history is logged to `output.jsonl` under `evaluation_outputs`.
+
+
+```
+./evaluation/discoverybench/scripts/run_infer.sh [MODEL_CONFIG] [GIT_COMMIT] [AGENT] [EVAL_LIMIT] [NUM_WORKERS]
+```
+
+- `MODEL_CONFIG`: Name of the model you want to evaluate with
+- `GIT_COMMIT`: This should be the git commit hash or release tag for OpenHands, e.g., HEAD or a specific tag like 0.6.2.
+- `AGENT`: Use CoderActAgent, right now it only supports that.
+- `EVAL_LIMIT`: Number of samples to evaluate.
+- `NUM_WORKERS`: Number of workers to parallelize the evaluation process.

evaluation/discoverybench/eval_utils/README.md

@@ -0,0 +1,7 @@
+## DiscoveryBench Evaluation Utils
+
+- **`eval_w_subhypo_gen.py`**: Implements the DiscoveryBench logic for evaluating agent-generated hypotheses.
+- **`lm_utils.py`**: Provides utility functions necessary for the evaluation process.
+- **`openai_helpers.py`**: Includes helper functions for OpenAI-related tasks.
+- **`openai_semantic_gen_prompts.py`**: Contains prompts used for semantic generation.
+- **`response_parser.py`**: Handles the parsing of agent-generated hypotheses.

evaluation/discoverybench/eval_utils/eval_w_subhypo_gen.py

@@ -0,0 +1,538 @@
+import json
+import logging
+
+from openai import OpenAI
+
+from .lm_utils import run_chatgpt_query_multi_turn
+from .openai_helpers import get_response
+
+logging.basicConfig(
+    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+    datefmt='%m/%d/%Y %H:%M:%S',
+    level=logging.INFO,
+)
+logger = logging.getLogger(__name__)
+
+
+def get_score_from_answer(type, answer):
+    if type == 'context':
+        answer = answer.replace('Answer:', '').strip()
+        if answer.startswith('A)'):
+            return 1.0
+        elif answer.startswith('B)'):
+            return 0.0
+        return -1.0
+
+    elif type == 'var':
+        try:
+            var_json = json.loads(answer)
+            # print(f"var_json:{var_json}")
+            p = 0.0
+            r = 0.0
+            f1 = 0.0
+            if var_json['sizeB']:
+                p = var_json['intersection'] / var_json['sizeB']
+            if var_json['sizeA']:
+                r = var_json['intersection'] / var_json['sizeA']
+            if p > 0.0 and r > 0.0:
+                f1 = (2 * p * r) / (p + r)
+            else:
+                f1 = 0.0
+            eval_rec = {
+                'p': p,
+                'r': r,
+                'f1': f1,
+                'sizeA': var_json['sizeA'],
+                'sizeB': var_json['sizeB'],
+                'intersection': var_json['intersection'],
+                'explanation': var_json['explanation'],
+            }
+            print(f'var_eval: {eval_rec}')
+            return eval_rec
+        except Exception:  # COMMENT: added Exception
+            return {'p': -1.0, 'r': -1.0, 'f1': -1.0}
+    elif type == 'rel':
+        print(answer)
+        rel_json = json.loads(answer)
+        answer_str = rel_json['answer'].strip()
+        if answer_str.startswith('A') or 'very similar' in answer_str:
+            return 1.0
+        elif (
+            answer_str.startswith('B') or 'similar but general than HypoA' in answer_str
+        ):
+            return 0.5
+        elif answer_str.startswith('C') or 'different' in answer_str:
+            return 0.0
+        return -1.0
+    return -1.0
+
+
+def ask_dimension_question(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    dimension,
+    dataset_type,
+    use_column_metadata=True,
+):
+    dimension_question = ''
+    answer = ''
+    score = 0.0
+    if dimension == 'var':
+        score = {'p': -1.0, 'r': -1.0, 'f1': -1.0}
+    num_tokens = 256
+    num_retries = 1
+    json_response = False
+
+    messages = [
+        {
+            'role': 'system',
+            'content': 'You are an AI assistant that helps evaluate a data-driven hypothesis. You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+    ]
+    if dimension == 'context':
+        dimension_question = """\
+        Question: Is HypoB defined in the same context as HypoA?
+        (Context refers to assumptions/stratification under which the hypotheses are defined.)
+        Options: A) same   B) different
+        What is your answer?"""
+    elif dimension == 'var':
+        dimension_question = """\
+        Question: For both HypoA and HypoB, what are the different variables found in the hypotheses? \
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+        "sizeA": num of variables used in HypoA
+        "sizeB": num of variables used in HypoB
+        "intersection": num of variables common in HypoA and HypoB. Use *fuzzy matching* to determine intersection, accounting for paraphrases or slightly different surface forms
+        "explanation": a short text explanation about the variables
+        }}```
+        Answer:"""
+        num_tokens = 512
+        num_retries = 1
+        json_response = True
+    elif dimension == 'rel':
+        dimension_question = """\
+        Question: Does HypoB exhibit the same relation as HypoA?
+        Compare using following example hierarchy of relationships (based on specificity): \
+        "there exists a relationship" > "positive relationship" > "positive AND (linear OR quadratic)" > "positive AND linear".
+        Options: A) very similar B) similar but general than HypoA C) different
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+        "answer": one of the options from A) very similar B) similar but general than HypoA C) different
+        "explanation": a short text explanation about the relationship comparison
+        }}```
+        Answer:"""
+        num_tokens = 512
+        num_retries = 1
+        json_response = True
+
+    datasets_json = prepare_dataset_metadata_json(
+        dataset_meta, dataset_type=dataset_type, use_column_metadata=use_column_metadata
+    )
+
+    dimension_question_str = f"""\
+        You are going to compare two natural-language hypotheses HypoA and HypoB accompanied with optional workflows: WorkflowA for HypoA and WorkflowB for HypoB. \
+        Both the hypotheses answer the natural language query "QUERY" over the dataset(s) described by dataset description(s) and column description(s) below. \
+        Compare HypoA and HypoB in terms of three aspects: Contexts, Variables, and Relations. \
+        E.g., for the hypothesis "From 1995 to 2009, the number of sandhill cranes around the tundra (Indigilka River) surged by an astounding ~10X":
+        * Contexts refer to stratification of the data under which the given hypothesis is True. E.g., "For all women", "From 1995 to 2009".
+        * Variables refer to the set of variables (either dependent or independent) that are mentioned in the hypothesis. E.g., number of sandhill cranes, location.
+        * Relations refer to the form of relation between the variables. E.g., "surged by ~10x".
+
+        Answer following questions for a given pair of hypotheses, HypoA and HypoB, along with an explanation grounded on the QUERY and the DATASET(S).
+
+        Here is the metadata for the task:
+        ```json
+        {{
+        "datasets": {datasets_json},
+        "query": {query},
+        "HypoA": {gold_hypo},
+        "WorkflowA": {gold_workflow},
+        "HypoB": {gen_hypo},
+        "WorkflowB": {gen_workflow}
+        }}
+        ```
+
+        {dimension_question}"""
+
+    messages.append({'role': 'user', 'content': dimension_question_str})
+    for retry in range(num_retries):
+        response = run_chatgpt_query_multi_turn(
+            messages=messages,
+            model_name=llm_used,
+            max_tokens=num_tokens,
+            temperature=0,  # 0 for greedy best decoding
+            json_response=json_response,
+        )
+        if response is not None:  # COMMENT: changed from != to is not
+            break
+
+    if response is not None:  # COMMENT: changed from != to is not
+        answer = response.choices[0].message.content.strip()
+        score = get_score_from_answer(type=dimension, answer=answer)
+
+    return dimension_question, answer, score
+
+
+def prepare_dataset_metadata_json(dataset_meta, dataset_type, use_column_metadata=True):
+    if dataset_meta is None:  # COMMENT: changed from == to is None
+        return [
+            {
+                'dataset_description': '',
+                'columns': [],
+            }
+        ]
+    datasets_json = []
+    if dataset_type == 'real':
+        for d in dataset_meta['datasets']:
+            datasets_json.append(
+                {
+                    'dataset_description': d['description'],
+                    'columns': [
+                        {'name': col['name'], 'description': col['description']}
+                        for col in d['columns']['raw']
+                    ]
+                    if use_column_metadata
+                    else [],
+                }
+            )
+    else:
+        for d in dataset_meta['datasets']:
+            datasets_json.append(
+                {
+                    'dataset_description': d['description'],
+                    'columns': [
+                        {'name': col['name'], 'description': col['description']}
+                        for col in d['columns']
+                    ]
+                    if use_column_metadata
+                    else [],
+                }
+            )
+    return datasets_json
+
+
+def get_sub_hypotheses(
+    query,
+    hypo,
+    workflow,
+    dataset_meta,
+    llm_used,
+    dataset_type,
+    use_column_metadata=True,
+):
+    client = OpenAI()
+    extraction_prompt = """\
+        Given a set of dataset columns, a ground-truth hypothesis, and the analysis workflow used, your task is to extract three dimensions that define the hypothesis: Context, Variables, and Relations. \
+        Here are the definitions for these dimensions:
+        - Contexts: Boundary conditions that limit the scope of a hypothesis. E.g., “for men over \
+        the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then extract the context from the dataset_descrption.
+        - Variables: Known concepts that interact in a meaningful way under a given context to \
+        produce the hypothesis. E.g., gender, age, income, or "None" if there is no interacting variable.
+        - Relations: Interactions between a given set of variables under a given context to produce \
+        the hypothesis. E.g., “quadratic relationship”, “inversely proportional”, piecewise conditionals, \
+        or "None" if there is no interacting relationship.
+        Make sure to only use the information present in the hypothesis and the workflow. Do not add any new information. \
+        For each dimension, be specific, and do not omit any important details.
+
+        Here is the metadata for the task:
+        ```json
+        {
+        "datasets": %s,
+        "hypothesis": "%s",
+        "workflow": "%s"
+        }
+        ```
+
+        Return your answer as a JSON object in the following format:
+        ```json
+        {
+        "sub_hypo": [
+            {
+                "text": the hypothesis in natural language,
+                "context": a short text description of the context of the hypothesis,
+                "variables": a list of columns involved in the hypothesis,
+                "relations": a short text description of the relationship between the variables of the hypothesis
+            },
+            ...
+        ]
+        }```
+        """
+    datasets_json = prepare_dataset_metadata_json(
+        dataset_meta, dataset_type, use_column_metadata=use_column_metadata
+    )
+    _prompt = extraction_prompt % (datasets_json, hypo, workflow)
+    sub_hypo_json = get_response(client, _prompt, model=llm_used, max_retry=1)
+
+    if sub_hypo_json is not None:  # COMMENT: changed from != to is not
+        # print(f"full hypothesis: {hypo}")
+        print(f'sub_hypo_json: {sub_hypo_json}')
+    else:
+        sub_hypo_json = {
+            'sub_hypo': [],
+        }
+
+    sub_hypo_json['full_hypo'] = hypo
+
+    return sub_hypo_json
+
+
+def match_context_with_gpt(
+    gold_hyp, gold_context, pred_hyp, pred_context, model='gpt-3.5-turbo'
+):
+    prompt = f"""\
+        Given a gold hypothesis, a gold context, a predicted hypothesis, and a predicted context, your task is \
+        to determine if the predicted context semantically matches the ground-truth context. \
+        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
+        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
+        If the predicted context matches the gold context, return true, otherwise return false.
+        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
+        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
+
+        Here is the metadata for the task:
+        ```json
+        {{
+            "gold_hypothesis": "{gold_hyp}",
+            "gold_context": "{gold_context}",
+            "predicted_hypothesis": "{pred_hyp}",
+            "predicted_context": "{pred_context}"
+        }}
+        ```
+
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+            "match": true or false
+        }}
+        ```"""
+
+    client = OpenAI()
+    output = get_response(client, prompt, model=model)
+    return output.get('match', False)
+
+
+def is_matching_context(gold_hyp, gold_context, pred_hyp, pred_context, llm_used):
+    if gold_context == pred_context:
+        return True
+    if 'None' in [gold_context, pred_context]:
+        return False
+    return match_context_with_gpt(
+        gold_hyp, gold_context, pred_hyp, pred_context, model=llm_used
+    )
+
+
+def run_eval_gold_vs_gen_NL_subhypo(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    context_score,
+    dataset_type,
+    use_column_metadata=True,
+):
+    # GPT-4 based evaluation to evaluate generated hypothesis in terms of context, variables, relation
+
+    eval_rec = {
+        'query': query,
+        'HypoA': gold_hypo,
+        'WorkflowA': gold_workflow,
+        'HypoB': gen_hypo,
+        'WorkflowB': gen_workflow,
+    }
+
+    for dimension in ['var', 'rel']:
+        question, answer, score = ask_dimension_question(
+            query,
+            gold_hypo,
+            gold_workflow,
+            gen_hypo,
+            gen_workflow,
+            dataset_meta,
+            llm_used,
+            dimension=dimension,
+            dataset_type=dataset_type,
+            use_column_metadata=use_column_metadata,
+        )
+
+        eval_rec[dimension] = {'question': question, 'answer': answer, 'score': score}
+
+    eval_rec['context'] = context_score
+    eval_rec['accuracy_score'] = (
+        1.0
+        * eval_rec['context']['score']
+        * eval_rec['var']['score']['f1']
+        * eval_rec['rel']['score']
+    )
+
+    return eval_rec
+
+
+def run_eval_gold_vs_gen_NL_hypo_workflow(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    dataset_type,
+    use_column_metadata=True,
+):
+    # Input: Dataset Metadata, Query, Gold {Hg, Wg}, Predicted {Hp, Wp}
+    # Output: eval_rec json includes final_score
+
+    # Procedure:
+    # Dataset Metadata, Query, Gold {Hg, Wg}, Pred {Hg, Wg}
+    # Gold: [Hg1, Hg2] (compute on the fly) Hg1 is a NL form of subhypothesis
+    # Predicted: [Hp1, Hp2] (compute on the fly)
+
+    # Compute Intersection: [(Hg_i, Hp_j), …]  # tuples of (gold,pred) that matched with context (do this w/o explicit extraction)
+    # # filter so that a gold context and a predicted context are only attached to one tuple
+    # Compute recall_context (programmatically)
+
+    # r_v_list = []
+    # For (Hg_i, Hp_j) in the intersection:
+    #             With Hg_i, Hp_j in NL, ask GPT4 → #variables and #intersection and a paragraph explanation and programmatically calculate f1_v
+    # Hg_i, Hp_j in NL, ask GPT4 → matching score (0, 0.5 or 1) : A) very similar B) similar but general than HypoA C) different + explanation
+    # 	r_v_list ← f1_v * score_r
+    # accuracy_score = mean(r_v_list)
+    # score =   [ recall_context * mean over predicted context(context_score * var_score *rel_score )]
+
+    # recall_context = 1.0  # COMMENT: never used
+    eval_rec = {
+        'query': query,
+        'HypoA': gold_hypo,
+        'WorkflowA': gold_workflow,
+        'HypoB': gen_hypo,
+        'WorkflowB': gen_workflow,
+    }
+
+    gold_sub_hypo_json = get_sub_hypotheses(
+        query=query,
+        hypo=gold_hypo,
+        workflow=gold_workflow,
+        dataset_meta=dataset_meta,
+        llm_used=llm_used,
+        dataset_type=dataset_type,
+        use_column_metadata=use_column_metadata,
+    )
+    if len(gold_sub_hypo_json['sub_hypo']) == 0:
+        gold_sub_hypo_json['sub_hypo'] = [
+            {
+                'text': gold_hypo,
+                'context': 'None',
+                'variables': [],
+                'relations': '',
+                'explanation': 'unable to segment',
+            }
+        ]
+    print(f'gold_sub_hypo_json: {gold_sub_hypo_json}')
+
+    gen_sub_hypo_json = get_sub_hypotheses(
+        query=query,
+        hypo=gen_hypo,
+        workflow=gen_workflow,
+        dataset_meta=dataset_meta,
+        llm_used=llm_used,
+        dataset_type=dataset_type,
+        use_column_metadata=use_column_metadata,
+    )
+    if len(gen_sub_hypo_json['sub_hypo']) == 0:
+        gen_sub_hypo_json['sub_hypo'] = [
+            {
+                'text': gen_hypo,
+                'context': 'None',
+                'variables': [],
+                'relations': '',
+                'explanation': 'unable to segment',
+            }
+        ]
+    print(f'gen_sub_hypo_json: {gen_sub_hypo_json}')
+
+    eval_rec['gold_sub_hypo'] = gold_sub_hypo_json
+    eval_rec['gen_sub_hypo'] = gen_sub_hypo_json
+
+    gold_subh_covered = []
+    gen_subh_to_gold_subh = dict()
+    gen_gold_subh_to_context = dict()
+
+    for p_id, gen_subh in enumerate(gen_sub_hypo_json['sub_hypo']):
+        gen_subh_to_gold_subh[p_id] = -1
+
+        for g_id, gold_subh in enumerate(gold_sub_hypo_json['sub_hypo']):
+            if g_id in gold_subh_covered:
+                continue
+
+            # match context
+            context_bool = is_matching_context(
+                gold_subh['text'],
+                gold_subh.get('context', ''),
+                gen_subh['text'],
+                gen_subh.get('context', ''),
+                llm_used,
+            )
+            if context_bool:
+                context_score = 1.0
+            else:
+                context_score = 0.0
+
+            if context_score == 1.0:  # match only when context_score = 1.0
+                gen_subh_to_gold_subh[p_id] = g_id
+                gold_subh_covered.append(g_id)
+                gen_gold_subh_to_context[f'P{p_id}||G{g_id}'] = {
+                    'question': f"""Comapring: GoldH: {gold_subh["text"]}, GoldC: {gold_subh['context']}\nGenH: {gen_subh['text']}, GenC: {gen_subh['context']}""",
+                    'answer': context_bool,
+                    'score': context_score,
+                }
+                break
+
+    print(f'gen_subh_to_gold_subh: {gen_subh_to_gold_subh}')
+    eval_rec['gen_subh_to_gold_subh'] = gen_subh_to_gold_subh
+    eval_rec['gold_subh_covered'] = gold_subh_covered
+    matched_gold_gen_subh_evals = dict()
+    sum_accuracy_score = 0.0
+    for p_id, g_id in gen_subh_to_gold_subh.items():
+        if g_id >= 0:
+            key = f'P{p_id}||G{g_id}'
+            context_score = gen_gold_subh_to_context[key]
+            subh_eval_rec = run_eval_gold_vs_gen_NL_subhypo(
+                query,
+                gold_hypo,
+                gold_workflow,
+                gen_hypo,
+                gen_workflow,
+                dataset_meta,
+                llm_used,
+                context_score,
+                dataset_type=dataset_type,
+                use_column_metadata=use_column_metadata,
+            )
+            sum_accuracy_score += subh_eval_rec['accuracy_score']
+            matched_gold_gen_subh_evals[key] = subh_eval_rec
+
+    eval_rec['matched_gold_gen_subh_evals'] = matched_gold_gen_subh_evals
+    eval_rec['recall_context'] = (
+        len(gold_subh_covered) / len(gold_sub_hypo_json['sub_hypo'])
+        if len(gold_sub_hypo_json['sub_hypo'])
+        else 0.0
+    )
+    mean_accuracy_score = (
+        sum_accuracy_score / len(gen_subh_to_gold_subh)
+        if len(gen_subh_to_gold_subh)
+        else 0.0
+    )
+    eval_rec['mean_accuracy_score'] = mean_accuracy_score
+    final_score = eval_rec['recall_context'] * mean_accuracy_score
+    eval_rec['final_score'] = final_score
+    print(f'eval_rec: {json.dumps(eval_rec, indent=2)}')
+
+    return eval_rec

evaluation/discoverybench/eval_utils/lm_utils.py

@@ -0,0 +1,64 @@
+import os
+import sys
+import time
+
+from openai import OpenAI
+from tenacity import (
+    retry,
+    stop_after_attempt,  # type: ignore
+    wait_random_exponential,  # type: ignore
+)
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+
+Model = Literal['gpt-4', 'gpt-3.5-turbo', 'text-davinci-003']
+
+OpenAI.api_key = os.getenv('OPENAI_API_KEY')
+OPENAI_GEN_HYP = {
+    'temperature': 0,
+    'max_tokens': 250,
+    'top_p': 1.0,
+    'frequency_penalty': 0,
+    'presence_penalty': 0,
+}
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+def run_chatgpt_query_multi_turn(
+    messages,
+    model_name='gpt-4-turbo',  # pass "gpt4" for more recent model output
+    max_tokens=256,
+    temperature=0.0,
+    json_response=False,
+):
+    response = None
+    num_retries = 3
+    retry = 0
+    while retry < num_retries:
+        retry += 1
+        try:
+            client = OpenAI()
+
+            if json_response:
+                response = client.chat.completions.create(
+                    model=model_name,
+                    response_format={'type': 'json_object'},
+                    messages=messages,
+                    **OPENAI_GEN_HYP,
+                )
+            else:
+                response = client.chat.completions.create(
+                    model=model_name, messages=messages, **OPENAI_GEN_HYP
+                )
+            break
+
+        except Exception as e:
+            print(e)
+            print('GPT error. Retrying in 2 seconds...')
+            time.sleep(2)
+
+    return response

evaluation/discoverybench/eval_utils/openai_helpers.py

@@ -0,0 +1,190 @@
+import json
+
+
+def OPENAI_TOPIC_GEN_MESSAGES(n=10):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given `n`, come up with a list of `n` distinct topics and their descriptions. The topics can be absolutely anything. Be as creative as possible. Return your answer as a JSON object. \n\nFor example, for `n`=3, a valid answer might be:\n```json\n{{"topics": [\n  {{"id": 1, "topic": "cooking", "description": "Related to recipes, ingredients, chefs, etc."}},\n  {{"id": 2, "topic": "sports", "description": "Related to players, stadiums, trophies, etc."}},\n  {{"id": 3, "topic": "antiquing", "description": "Related to unique items, history, etc."}}\n]}}```\n\nNow, give me a list for `n`={n}. Remember, pick diverse topics from everything possible. No consecutive topics should be broadly similar. Directly respond with the answer JSON object.',
+        },
+    ]
+
+
+OPENAI_GEN_HYP = {
+    'temperature': 1.0,
+    'max_tokens': 4096,
+    'top_p': 1.0,
+    'frequency_penalty': 0,
+    'presence_penalty': 0,
+}
+
+
+def OPENAI_SEMANTICS_GEN_MESSAGES(dependent, relationship, domain, domain_desc):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given the true relationship in a dataset and a given domain, your task is to come up with an interpretation of some real-world concepts that the relationship could be modeling from the provided domain. It\'s okay to be wrong, but suggest something reasonable. Try as much as possible to make sure that the TARGET is actually derivable from the other variables. Give your answer as a JSON object. Here\'s an example:\n\nRelationship for x2 = "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)"\nDomain="Sales"\nDomain description="Related to product distribution, revenues, marketing, etc."\n\nBased on this, the following real-world concepts might be applicable:\n```json\n{{\n  "dependent": "x2",\n  "relationship": "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)",\n  "domain": "Sales",\n  "trends": {{\n    "x1": "Positive, cubic factor",\n    "x2": "TARGET",\n    "x3": "Positive, linear factor",\n    "x4": "No relation",\n    "x5": "Positive quadratic factor",\n    "x6": "Positive, inverse quadratic factor"\n  }},\n  "interpretation": {{\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area", "is_target": true}},\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n}}```\n\nHere\'s a new test question:\nRelationship for {dependent} = "{relationship}"\nDomain = "{domain}"\nDomain description="{domain_desc}"\n\nRespond only with the answer JSON. Make sure that you do not forget to include the TARGET variable in the interpretation object.',
+        },
+    ]
+
+
+def OPENAI_SEMANTICS_GEN_W_MAP_MESSAGES(
+    dependent, relationship, domain, domain_desc, mapping
+):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given a partial mapping from variables to real-world concepts and a true relationship in a dataset, your task is to come up with an interpretation of real-world concepts for the variables without any assigned mapping (those starting with x). Suggest something reasonable. The dependent variable must be derivable only from the other variables in the dependent relationship. Give your answer as a JSON object. Here\'s an example:\n\nExample partial mapping and relationship:\n```json\n{{\n  "domain": "Sales",\n  "domain_description": "Related to product distribution, revenues, marketing, etc.",\n  "variable_mapping": {{\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n  }},\n  "dependent_variable": "sales_area",\n  "dependent_relationship": "(96.4 * pop_area ** 3) + (88.72 * x5 ** 2) + (81.96 * dist_to_distr_ctr ** -2) + (28.13 * x3)  + (97.0)"\n}}```\nBased on this, an example answer would be:\n```json\n{{\n  "dependent_variable": "sales_area",\n  "missing_mapping": ["x3", "x5"],\n  "trends": {{\n    "x3": "Positive, linear factor",\n    "x5": "Positive quadratic factor"\n  }},\n  "interpretation": {{\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }}\n}}```\n\nHere\'s a new test question:\n```json\n{{\n  "domain": "{domain}",\n  "domain_description": "{domain_desc}",\n  "variable_mapping": {json.dumps(mapping, indent=2)},\n  "dependent_variable": "{dependent}",\n  "dependent_relationship": "{relationship}"\n}}```\nRespond only with the answer JSON.',
+        },
+    ]
+
+
+def OPENAI_SEMANTICS_GEN_SUMMARY_MESSAGES(dataset):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given the following descriptions of the columns of a dataset, your task is to come up with a natural language overview of the dataset, which should include (1) what the dataset is about, (2) how the data was collected, (3) when the data was collected, and (3) for what purpose the data was collected. Be specific and creative.\n\nExample dataset:\n```json\n{{  \n  "dataset": {{                                                                                                                                                                                       \n    "x6": {{"description": "Ancient artifact significance score", "name": "artifact_significance_score", "is_target": true}},\n    "x1": {{"description": "Distance to ancient city center", "name": "dist_to_ancient_city_ctr"}},\n    "x2": {{"description": "Quantity of discovered relics", "name": "relic_discovery_qty"}},\n    "x3": {{"description": "Years since last archaeological expedition", "name": "years_since_exp"}},\n    "x4": {{"description": "Number of artifacts in excavation site", "name": "artifact_qty"}},\n    "x5": {{"description": "Soil fertility coefficient", "name": "soil_fertility_coef"}},\n    "x7": {{"description": "Distance to ancient burial grounds", "name": "dist_to_burial_grounds"}},\n    "x8": {{"description": "Population estimate of ancient civilization", "name": "ancient_civilization_pop_estimate"}},\n    "x9": {{"description": "Temperature variation in excavation region", "name": "temp_variation"}}\n  }}\n}}```\nExample description:\nThis dataset is about archaeological explorations and findings linked to ancient civilizations. The data was collected in the form of field metrics during various archaeological expeditions during the late mid-20th century. The purpose of the data collection is to evaluate the significance of ancient artifacts discovered during excavations.\n\nHere is a new test dataset.\n{json.dumps(dataset, indent=2)}\nProvide only the description.',
+        },
+    ]
+
+
+def OPENAI_GEN_HYPO_MESSAGES(dataset):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given a dataset with its descriptions and the true functional relationship between its variables, your task is to generate 3 levels of hypotheses for the stated relationship in plain English. The three levels are "broad", "medium" and "narrow". Make sure that the hypotheses sound natural. *Only include concepts for variables that are present in the provided functional relationship.* Give your answer as a JSON.\n\nFor example, an example dataset might be the following:\n```json\n{{\n  "domain": "cybersecurity",\n  "summary": "This dataset is about measuring cybersecurity threats in a system. The data was collected by monitoring various cybersecurity metrics in a network environment. The purpose of the data collection is to assess and predict potential cybersecurity risks and vulnerabilities.",\n  "variables": [\n    {{\n      "description": "Level of cybersecurity threat",\n      "name": "cybersecurity_threat",\n      "is_target": true\n    }},\n    {{\n      "description": "Number of failed login attempts",\n      "name": "failed_login_attempts"\n    }},\n    {{\n      "description": "Amount of encrypted data",\n      "name": "encrypted_data"\n    }},\n    {{\n      "description": "Frequency of software updates",\n      "name": "software_updates"\n    }},\n    {{\n      "description": "Number of antivirus software installed",\n      "name": "antivirus_software"\n    }},\n    {{\n      "description": "Quality of firewall protection",\n      "name": "firewall_quality"\n    }}\n  ],\n  "relationship": {{\n    "dependent": "cybersecurity_threat",\n    "relation": "-53.5*encrypted_data**2 - 53.85*failed_login_attempts**2 + 67.75*firewall_quality - 92.16 - 36.68/software_updates**3"\n  }}\n}}```\nGiven this dataset, the following is a valid answer:\n```json\n{{\n  "broad": {{\n    "instruction": "Be vague. Only indicate which concepts might be related but not how they are related",\n    "hypothesis": "Threat to cybersecurity is influenced by several factors including the amount of encrypted data, the number of failed login attempts, the quality of the firewall, as well as how often the software is updated."\n  }},\n  "medium": {{\n    "instruction": "Be slightly more specific. For each factor, indicate carefully whether it positively or negatively affects the relationship, but do not indicate what the exponent is.",\n    "hypothesis": "Cybersecurity threat tends to decrease with the amount of data encryption, the number of failed login attempts, as well as the frequency of software updates to some extent, while improvement in the firewall quality has a positive effect."\n  }},\n  "narrow": {{\n    "instruction": "Be specific. Communicate the concepts, whether there is a positive or negative effect (be careful), and the meaning of the exponent",\n    "hypothesis": "The threat to cybersecurity interacts in a complex manner with various factors. As the amount of encrypted data increases, there is a quadratic decrease in threat. Similarly for the number of failed login attempts, there is a negative quadratic relationship. The quality of the firewall protection on the other hand demonstrates a positive and linear relationship. Finally, the frequency of software updates has an inverse cubic relationship to the threat."\n  }},\n}}\n```\n\nBased on this, provide an answer for the following test dataset:\n```json\n{dataset}```\nRespond only with a JSON.',
+        },
+    ]
+
+
+def create_prompt(usr_msg):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {'role': 'user', 'content': usr_msg},
+    ]
+
+
+def get_response(client, prompt, max_retry=5, model='gpt-3.5-turbo', verbose=False):
+    n_try = 0
+    while n_try < max_retry:
+        response = client.chat.completions.create(
+            model=model, messages=create_prompt(prompt), **OPENAI_GEN_HYP
+        )
+
+        # COMMENT: changed from
+        # response.choices[0].message.content.strip().strip('```json').strip('```')
+        content = response.choices[0].message.content
+        cleaned_content = content.split('```json')[1].split('```')[0].strip()
+        output = cleaned_content
+        try:
+            response_json = json.loads(output)
+            return response_json
+        except ValueError:
+            if verbose:
+                print(f'Bad JSON output:\n\n{output}')
+            n_try += 1
+            if n_try < max_retry:
+                if verbose:
+                    print('Retrying...')
+            else:
+                if verbose:
+                    print('Retry limit reached')
+    return None
+
+
+def get_code_fix(
+    client, code, error, max_retry=5, model='gpt-3.5-turbo', verbose=False
+):
+    prompt = f"""\
+Given the following code snippet and error message, provide a single-line fix for the error. \
+Note that the code is going to be executed using python `eval`. \
+The code should be executable and should not produce the error message. Be as specific as possible.
+
+Here's the code and the error:
+{{
+    "code": "{code}",
+    "error": "{error}"
+}}
+
+Return only a JSON object with the fixed code in the following format:
+```json
+{{
+    "fixed_code": "..."
+}}"""
+    response = get_response(
+        client, prompt, max_retry=max_retry, model=model, verbose=verbose
+    )
+    return response
+
+
+def get_new_hypothesis(
+    client, target, old, expr, cols, model='gpt-3.5-turbo', verbose=False
+):
+    prompt = f"""\
+Given a target column from a dataset, a pandas expression to derive the column from existing columns, a list of \
+existing columns, and a previously written hypothesis text, carefully check if the hypothesis text is consistent with \
+the pandas expression or not. If it is consistent, simply return the hypothesis as it is. If it is not consistent, \
+provide a new natural language hypothesis that is consistent with the pandas expression using only the provided \
+information. Be specific.
+
+Here's the information:
+```json
+{{
+    "target_column": "{target}",
+    "pandas_expression": "{expr}",
+    "existing_columns": {json.dumps(cols, indent=4)}
+    "old_hypothesis": "{old}",
+}}```
+
+Give your answer as a new JSON with the following format:
+```json
+{{
+    "hypothesis": "..."
+}}"""
+    response = get_response(client, prompt, model=model, verbose=verbose)
+    return response
+
+
+def replace_variable(client, expr, old, new, model='gpt-3.5-turbo', verbose=False):
+    prompt = f"""\
+Given a pandas "expression", replace mentions of the "old" column with its "new" value such that the resultant \
+expression is equivalent to the original expression.
+
+Here's the information:
+```json
+{{
+    "expression": "{expr}",
+    "old": "{old}",
+    "new": "{new}"
+}}```
+
+Give your answer as a new JSON with the following format:
+```json
+{{
+    "new_expression": "..."
+}}"""
+    response = get_response(client, prompt, model=model, verbose=verbose)
+    return response

evaluation/discoverybench/eval_utils/openai_semantic_gen_prompts.py

@@ -0,0 +1,151 @@
+common_hypothesis_features = [
+    '1-2 sentences',
+    'surprising finding',
+    'includes numeric concepts',
+    'includes categorical concepts',
+    'includes binary concepts',
+]
+hypothesis_features = [
+    ['requires within-cluster analysis'],
+    ['requires across-cluster analysis'],
+    ['corresponds to a polynomial relationship of some columns'],
+    ['corresponds to a ratio between some columns'],
+    ['requires temporal analysis'],
+    ['relationship is based on descriptive statistics of some columns'],
+    ['requires concepts based on percentage or percentiles'],
+    ['relationship is only applicable to one cluster in the data and not the others'],
+]
+
+column_features = [
+    [
+        'must have one target column',
+        'must have quantifiable columns',
+        'must have a few categorical columns',
+        'make sure the categorical column values do not contain special characters',
+        'include a few distractor columns',
+    ]
+]
+
+common_pandas_features = [
+    'must be executable using python `eval` to create the target column in variable `df` (pandas dataframe)',
+    "for e.g., df['A']**2 + 3*df['B'] + 9, np.where(df['A'] > 3, 'Yes', 'No'), etc.",
+    'variables in pandas_expression must be from the existing columns listed above',
+    'variables in pandas_expression must NOT contain the target column itself',
+]
+pandas_features = [
+    ['expression is a quadratic polynomial'],
+    ['expression is a cubic polynomial'],
+    ['expression is a ratio of existing columns'],
+    ['expression is derived through logical combination of existing columns'],
+    # workflow
+]
+pandas_features = [common_pandas_features + p for p in pandas_features]
+
+common_derived_features = [
+    '1-2 sentences',
+    'includes numeric concepts',
+    'includes categorical concepts',
+    'includes binary concepts',
+]
+derived_features = [common_derived_features + h for h in hypothesis_features]
+hypothesis_features = [common_hypothesis_features + h for h in hypothesis_features]
+
+PROMPT_HYP = """\
+Given a dataset topic and description, generate an interesting hypothesis based on \
+the provided instructions. Be creative and come up with an unusual finding.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis_features": %s,
+    "hypothesis": "..."
+}```
+
+Give your answer as a new JSON with the following format:
+```json
+{
+    "hypothesis": "..."
+}
+```"""
+
+PROMPT_COL = """\
+Given a dataset topic, its description, and a true hypothesis that can be determined from it, \
+generate a list of valid columns based on the provided instructions.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis": "%s",
+    "column_instructions": %s,
+    "columns": [
+        {
+            "col_name": "...",  # should be an "_"-separated string
+            "description": "...",
+            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
+            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
+            "is_distractor": true/false,  # boolean indicating whether this is a distractor that could cause confusion during data analysis
+            "is_target": true/false  # boolean indicating whether this is the target variable for the hypothesis; at least one column should be the target
+        },
+        ...
+    ],
+    "pandas_instructions": %s,
+    "pandas_equation_for_hypothesis": {
+        "target_col": "...",
+        "target_col_type": "...",
+        "target_col_range": {...},
+        "independent_cols_in_pandas_expression": [],  # list of column names that will be used to derive the target column
+        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
+    }
+}```
+
+Give your answer as a new JSON with the "columns" and "pandas_equation_for_hypothesis" keys filled using the following format:
+```json
+{
+    "columns": [...],
+    "pandas_equation_for_hypothesis": {...}
+}
+```"""
+
+PROMPT_DER = """\
+Given a dataset topic, description, a true hypothesis that can be determined from the data, \
+and a target column from the dataset, generate a hypothesis for the target column using new independent columns not present in the existing columns.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis": "%s",
+    "existing_columns": %s,
+    "target_column": "%s",
+    "new_to_target_instructions": %s,
+    "new_to_target_hypothesis": "...",  # describe a relationship between new columns that explains the target column
+    "new_columns_for_target": [  # do not repeat any of the existing columns in the dataset
+        {
+            "col_name": "...",  # should be an "_"-separated string
+            "description": "...",
+            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
+            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
+        },
+        ...
+    ],
+    "pandas_instructions": %s,
+    "pandas_equation_for_new_to_target_hypothesis": {
+        "target_col": "...",
+        "target_col_type": "...",
+        "target_col_range": {...},
+        "independent_cols_in_pandas_expression": [],  # list of column names from new_columns_for_target that will be used to derive target_col
+        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
+    }
+}```
+
+Give your answer as a new JSON with the "new_to_target_hypothesis", "new_columns_for_target", and \
+"pandas_equation_for_new_to_target_hypothesis" keys filled using the following format:
+```json
+{
+    "new_to_target_hypothesis": "...",
+    "new_columns_for_target": [...],
+    "pandas_equation_for_new_to_target_hypothesis": {...}
+}
+```"""

evaluation/discoverybench/eval_utils/response_parser.py

@@ -0,0 +1,52 @@
+workflow_summary_markers = [
+    'WORKFLOW SUMMARY',
+    'WORKFLOW_SUMMARY',
+    'WORKFLOW-SUMMARY',
+    'Workflow Summary',
+]
+
+final_answer_markers = [
+    'FINAL ANSWER',
+    'FINAL_ANSWER',
+    'FINAL-ANSWER',
+    'Final Answer',
+    'Scientific Hypothesis',
+    'Hypothesis',
+]
+
+next_agent_markers = [
+    'NEXT AGENT',
+    'NEXT-AGENT',
+    'NEXT_AGENT',
+    'FEEDBACK',
+]
+
+
+def extract_between(content, start_markers, end_markers=None):
+    for marker in start_markers:
+        if marker in content:
+            result = content.split(marker, 1)[1]
+            if end_markers:
+                for end_marker in end_markers:
+                    if end_marker in result:
+                        result = result.split(end_marker, 1)[0]
+            return result
+    return ''
+
+
+def extract_gen_hypo_from_logs(content: str):
+    error = ''
+
+    gen_workflow = extract_between(
+        content, workflow_summary_markers, final_answer_markers
+    )
+
+    if not gen_workflow:
+        error += 'No Workflow Summary found in the line. | '
+
+    gen_hypothesis = extract_between(content, final_answer_markers, next_agent_markers)
+
+    if not gen_hypothesis:
+        error += 'No Final Answer in the line.'
+
+    return gen_hypothesis, gen_workflow, error

evaluation/discoverybench/run_infer.py

@@ -0,0 +1,492 @@
+import asyncio
+import json
+import os
+
+import git
+import pandas as pd
+
+from evaluation.discoverybench.eval_utils.eval_w_subhypo_gen import (
+    run_eval_gold_vs_gen_NL_hypo_workflow,
+)
+from evaluation.discoverybench.eval_utils.response_parser import (
+    extract_gen_hypo_from_logs,
+)
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    codeact_user_response,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    parse_arguments,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+EVALUATION_LLM = 'gpt-4-1106-preview'
+
+DATA_FILES = {}
+
+LIBRARIES = [
+    'pandas',
+    'numpy',
+    'scipy',
+    'matplotlib',
+    'seaborn',
+    'scikit-learn',
+    'statsmodels',
+]
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
+
+AGENT_CLS_TO_INST_SUFFIX = {
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+}
+
+
+def get_config(
+    metadata: EvalMetadata,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='eventstream',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=True,
+            use_host_network=False,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    agent_config = AgentConfig(
+        function_calling=False,
+        codeact_enable_jupyter=True,
+        codeact_enable_browsing_delegate=True,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def get_dv_query_for_real(
+    datasets, question, domain_knowledge=None, workflow_tags=None
+):
+    """
+    Prepare a structured query for the agent to execute on the specified datasets.
+
+    This function constructs a query by compiling metadata from the provided datasets, along with any relevant domain knowledge and workflow tags.
+
+    Args:
+        datasets: List of datasets
+        question: Query to be answered
+        domain_knowledge: Domain knowledge if any
+        workflow_tags: Workflow tags if any
+
+    Returns:
+        query_to_dv: Query to be run on the dataset
+        dataset_meta: Metadata of the dataset
+    """
+
+    dataset_meta = ''
+    for dataset_metadata in datasets:
+        dataset_meta += 'Dataset name: ' + dataset_metadata['name']
+        dataset_meta += 'Dataset description: ' + dataset_metadata['description']
+        dataset_meta += '\nBrief description of columns: '
+        for col in dataset_metadata['columns']['raw']:
+            dataset_meta += col['name'] + ': ' + col['description'] + ', '
+
+    query_to_dv = dataset_meta
+
+    query_to_dv += f'\nQuery: {question}'
+
+    if domain_knowledge:
+        query_to_dv += (
+            '\nAdditionally, we provide some hints that might be useful to solve the task. Domain Knowledge: \n'
+            + domain_knowledge
+            + '.\n'
+        )
+
+    if workflow_tags:
+        query_to_dv += 'The meta tags are: ' + workflow_tags + '.\n'
+
+    query_to_dv += (
+        'In the final answer, please write down a scientific hypothesis in '
+        'natural language, derived from the provided dataset, clearly stating the '
+        'context of hypothesis (if any), variables chosen (if any) and '
+        'relationship between those variables (if any) including any statistical significance.'
+        'Also generate a summary of the full workflow starting from data loading that led to the final answer as WORKFLOW SUMMARY:'
+    )
+
+    # Run the NL query through datavoyager
+    return query_to_dv, dataset_meta
+
+
+def initialize_runtime(runtime: Runtime, data_files: list[str]):
+    """
+    Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    action = CmdRunAction(command='mkdir -p /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(command='cd /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    for file in data_files:
+        runtime.copy_to(
+            file,
+            '/workspace',
+        )
+
+    for lib in LIBRARIES:
+        action = CmdRunAction(command=f'pip install {lib}')
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        assert obs.exit_code == 0
+
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+
+
+def get_last_agent_finish_action(state: State) -> AgentFinishAction:
+    for event in reversed(state.history):
+        if isinstance(event, AgentFinishAction):
+            return event
+    return None
+
+
+def get_last_message_action(state: State) -> MessageAction:
+    for event in reversed(state.history):
+        if isinstance(event, MessageAction):
+            return event
+    return None
+
+
+def complete_runtime(state: State):
+    last_agent_finish_action = get_last_agent_finish_action(state)
+    last_agent_message_action = get_last_message_action(state)
+
+    if last_agent_finish_action is not None:
+        final_message_1 = last_agent_finish_action.thought
+        gen_hypo_1, gen_workflow_1, error_1 = extract_gen_hypo_from_logs(
+            final_message_1
+        )
+    else:
+        gen_hypo_1, gen_workflow_1, error_1 = '', '', ''
+
+    if last_agent_message_action is not None:
+        final_message_2 = last_agent_message_action.content
+        gen_hypo_2, gen_workflow_2, error_2 = extract_gen_hypo_from_logs(
+            final_message_2
+        )
+    else:
+        gen_hypo_2, gen_workflow_2, error_2 = '', '', ''
+
+    if gen_hypo_1 and gen_hypo_2:
+        test_result = {
+            'gen_hypo': last_agent_finish_action.thought
+            if last_agent_finish_action
+            else last_agent_message_action.content,
+            'gen_workflow': '',
+            'error': '',
+        }
+        return test_result
+
+    test_result = {
+        'gen_hypo': gen_hypo_1 if gen_hypo_1 else gen_hypo_2,
+        'gen_workflow': gen_workflow_1 if gen_workflow_1 else gen_workflow_2,
+        'error': error_1 if error_1 else error_2,
+    }
+
+    return test_result
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+):
+    """
+    Process and evaluate a single instance of the dataset.
+
+    This function executes the OpenHands agent
+    for a specific instance of the dataset. It retrieves
+    the agent's results and evaluates them against the gold
+    hypothesis.
+
+    Args:
+        instance: A single row of the dataset
+        metadata: Metadata for the evaluation
+        reset_logger: Whether to reset the logger
+
+    Returns:
+        output: EvalOutput object
+    """
+
+    config = get_config(metadata)
+
+    # use a session id for concurrent evaluation
+    sid = 'ID_' + str(instance.instance_id)
+
+    # Setup the logger properly, so you can run
+    # multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    problem_statement, dataset_metadata = get_dv_query_for_real(
+        datasets=instance.datasets,
+        question=instance.query,
+        domain_knowledge=instance.domain_knowledge,
+        workflow_tags=instance.workflow_tags,
+    )
+
+    # Prepare instruction
+    instruction = (
+        f'You are a discovery agent who can execute a python code only once to answer a query based on one or more datasets. The datasets will be present in the current directory.\n\n'
+        'Environment has been set up for you to start working. You may assume all necessary tools and datasets are installed.\n\n'
+        '# Problem Statement\n'
+        f'{problem_statement}\n\n'
+    )
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
+        'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
+        'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
+    )
+    # NOTE: You can actually set slightly different instruction for different agents
+    instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
+
+    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    runtime = create_runtime(config, sid=sid)
+    call_async_from_sync(runtime.connect)
+    initialize_runtime(runtime, instance.data_files)
+
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=MessageAction(content=instruction),
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
+    )
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    metrics = state.metrics.get() if state.metrics else None
+    test_result = complete_runtime(state)
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    # DiscoveryBench Evaluation
+    eval_rec = run_eval_gold_vs_gen_NL_hypo_workflow(
+        query=instance.query,
+        gold_hypo=instance.gold_hypo,
+        gold_workflow='',
+        gen_hypo=test_result['gen_hypo'],
+        gen_workflow='',
+        dataset_meta=instance.dataset_metadata,
+        llm_used=EVALUATION_LLM,
+        dataset_type='real',
+    )
+
+    test_result['eval_rec'] = eval_rec
+
+    output = EvalOutput(
+        instance_id=str(instance.instance_id),
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result=test_result,
+    )
+
+    return output
+
+
+def update_csv_name(name):
+    name = name.replace('-', '_')
+
+    if 'meta_regression' in name:
+        name = name.replace('meta_regression', 'meta-regression')
+    if 'ML_enabled' in name:
+        name = name.replace('ML_enabled', 'ML-enabled')
+
+    return name
+
+
+def list_csv_files(list_of_datasets):
+    res = []
+    for ele in list_of_datasets:
+        for key, value in ele.items():
+            if key == 'name':
+                csv_file_name = update_csv_name(value)
+                res.append(DATA_FILES[csv_file_name])
+    return res
+
+
+def create_dataset(repo_location: str, split: str = 'test'):
+    """
+    Create a dataset from the discoverybench repository
+    by walking through the repository and extracting metadata
+    from the metadata_{}.json files
+
+    Args:
+        repo_location: Location of the repository
+        split: Split of the dataset to use
+
+    Returns:
+        df: DataFrame containing the dataset instances
+    """
+
+    data_dict = {}
+
+    data_location = os.path.join(repo_location, 'discoverybench', 'real', split)
+    answer_key_location = os.path.join(repo_location, 'eval', 'answer_key_real.csv')
+
+    idx = 0
+
+    for root, dirs, files in os.walk(data_location):
+        for file in files:
+            if file.endswith('.json'):
+                if 'metadata' in file:
+                    metadata = json.load(open(os.path.join(root, file)))
+
+                    dataset = root.split('/')[-1]
+                    metadata_id = file.split('_')[-1].split('.')[0]
+                    domain = metadata.get('domain', '')
+                    domain_knowledge = metadata.get('domain_knowledge', '')
+                    workflow_tags = metadata.get('workflow_tags', '')
+                    datasets = metadata.get('datasets', [])
+                    queries = metadata.get('queries', [])
+                    gold_workflow = metadata.get('workflow')
+
+                    # loop through queries list to get queries
+                    # and each query has qid; add that to dictionary
+                    for query in queries[0]:
+                        qid = query.get('qid', '')
+
+                        data = {
+                            'dataset': dataset,
+                            'metadata_id': metadata_id,
+                            'qid': qid,
+                            'domain': domain,
+                            'domain_knowledge': domain_knowledge,
+                            'workflow_tags': workflow_tags,
+                            'datasets': datasets,
+                            'question_type': query['question_type'],
+                            'query': query['question'],
+                            'gold_workflow': gold_workflow,
+                            'dataset_metadata': metadata,
+                        }
+
+                        data_dict[idx] = data
+                        idx += 1
+
+            if file.endswith('.csv'):
+                DATA_FILES[file] = os.path.join(root, file)
+            if file.endswith('.txt'):
+                DATA_FILES[file] = os.path.join(root, file)
+
+    df = pd.DataFrame.from_dict(data_dict, orient='index')
+
+    df['instance_id'] = df.index
+
+    df['data_files'] = df['datasets'].apply(lambda x: list_csv_files(x))
+
+    answer_key = pd.read_csv(answer_key_location)
+
+    answer_key = answer_key.rename(
+        columns={
+            'metadataid': 'metadata_id',
+            'query_id': 'qid',
+            'gold_hypothesis': 'gold_hypothesis',
+        }
+    )
+
+    df['qid'] = df['qid'].astype(int)
+    df['metadata_id'] = df['metadata_id'].astype(int)
+
+    answer_key['qid'] = answer_key['qid'].astype(int)
+    answer_key['metadata_id'] = answer_key['metadata_id'].astype(int)
+
+    df = pd.merge(df, answer_key, on=['dataset', 'metadata_id', 'qid'], how='left')
+
+    return df
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+
+    # clone git repositor for csv files
+    repo_url = 'https://github.com/allenai/discoverybench.git'
+    repo_location = 'git-discoverybench-allenai'
+
+    try:
+        git.Repo.clone_from(repo_url, repo_location)
+    except git.exc.GitCommandError:
+        print('Repository already exists')
+
+    dataset = create_dataset(repo_location)
+
+    # check if there is any empty csv_file
+    if dataset['data_files'].isnull().any():
+        raise ValueError('Some csv files are missing.')
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        'discoverybench-python',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+    )

evaluation/discoverybench/scripts/run_infer.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=$5
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+
+# ################################################################################
+
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+get_agent_version
+
+echo "AGENT: $AGENT"
+echo "AGENT_VERSION: $AGENT_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+COMMAND="poetry run python evaluation/discoverybench/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 10 \
+  --max-chars 10000000 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $AGENT_VERSION"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/gaia/run_infer.py

@@ -12,6 +12,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -166,7 +167,7 @@ def process_instance(
 
     model_answer_raw = ''
     # get the last message or thought from the agent
-    for event in state.history.get_events(reverse=True):
+    for event in reversed(state.history):
         if event.source == 'agent':
             if isinstance(event, AgentFinishAction):
                 model_answer_raw = event.thought
@@ -203,7 +204,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/gorilla/run_infer.py

@@ -10,6 +10,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -101,7 +102,7 @@ def process_instance(
         raise ValueError('State should not be None.')
 
     # retrieve the last message from the agent
-    model_answer_raw = state.history.get_last_agent_message()
+    model_answer_raw = state.get_last_agent_message()
 
     # attempt to parse model_answer
     ast_eval_fn = instance['ast_eval']
@@ -114,7 +115,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     output = EvalOutput(
         instance_id=instance_id,

evaluation/gpqa/run_infer.py

@@ -28,6 +28,7 @@ from datasets import load_dataset
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -244,7 +245,7 @@ Ok now its time to start solving the question. Good luck!
         'C': False,
         'D': False,
     }
-    for event in state.history.get_events(reverse=True):
+    for event in reversed(state.history):
         if (
             isinstance(event, AgentFinishAction)
             and event.source != 'user'
@@ -300,7 +301,7 @@ Ok now its time to start solving the question. Good luck!
         instance_id=str(instance.instance_id),
         instruction=instruction,
         metadata=metadata,
-        history=state.history.compatibility_for_eval_history_pairs(),
+        history=compatibility_for_eval_history_pairs(state.history),
         metrics=metrics,
         error=state.last_error if state and state.last_error else None,
         test_result={

evaluation/humanevalfix/run_infer.py

@@ -21,6 +21,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -255,7 +256,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/integration_tests/run_infer.py

@@ -13,6 +13,7 @@ from evaluation.utils.shared import (
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
+    update_llm_config_for_completions_logging,
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
@@ -55,18 +56,14 @@ def get_config(
         workspace_base=None,
         workspace_mount_path=None,
     )
-    if metadata.llm_config.log_completions:
-        metadata.llm_config.log_completions_folder = os.path.join(
-            metadata.eval_output_dir, 'llm_completions', instance_id
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance_id
         )
-        logger.info(
-            f'Logging LLM completions for instance {instance_id} to '
-            f'{metadata.llm_config.log_completions_folder}'
-        )
-    config.set_llm_config(metadata.llm_config)
+    )
     agent_config = AgentConfig(
         codeact_enable_jupyter=True,
-        codeact_enable_browsing_delegate=True,
+        codeact_enable_browsing=True,
         codeact_enable_llm_editor=False,
     )
     config.set_agent_config(agent_config)
@@ -132,7 +129,7 @@ def process_instance(
     # # result evaluation
     # # =============================================
 
-    histories = [event_to_dict(event) for event in state.history.get_events()]
+    histories = [event_to_dict(event) for event in state.history]
     test_result: TestResult = test_class.verify_result(runtime, histories)
     metrics = state.metrics.get() if state.metrics else None
 

evaluation/integration_tests/tests/t06_github_pr_browsing.py

@@ -0,0 +1,44 @@
+from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult
+from openhands.events.action import AgentFinishAction, MessageAction
+from openhands.events.event import Event
+from openhands.events.observation import AgentDelegateObservation
+from openhands.runtime.base import Runtime
+
+
+class Test(BaseIntegrationTest):
+    INSTRUCTION = 'Look at https://github.com/All-Hands-AI/OpenHands/pull/8, and tell me what is happening there and what did @asadm suggest.'
+
+    @classmethod
+    def initialize_runtime(cls, runtime: Runtime) -> None:
+        pass
+
+    @classmethod
+    def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult:
+        # check if the "The answer is OpenHands is all you need!" is in any message
+        message_actions = [
+            event
+            for event in histories
+            if isinstance(
+                event, (MessageAction, AgentFinishAction, AgentDelegateObservation)
+            )
+        ]
+        for event in message_actions:
+            if isinstance(event, AgentDelegateObservation):
+                content = event.content
+            elif isinstance(event, AgentFinishAction):
+                content = event.outputs.get('content', '')
+            elif isinstance(event, MessageAction):
+                content = event.content
+            else:
+                raise ValueError(f'Unknown event type: {type(event)}')
+
+            if (
+                'non-commercial' in content
+                or 'MIT' in content
+                or 'Apache 2.0' in content
+            ):
+                return TestResult(success=True)
+        return TestResult(
+            success=False,
+            reason=f'The answer is not found in any message. Total messages: {len(message_actions)}. Messages: {message_actions}',
+        )

evaluation/logic_reasoning/run_infer.py

@@ -8,6 +8,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -225,7 +226,7 @@ def process_instance(
         raise ValueError('State should not be None.')
 
     final_message = ''
-    for event in state.history.get_events(reverse=True):
+    for event in reversed(state.history):
         if isinstance(event, AgentFinishAction):
             final_message = event.thought
             break
@@ -247,7 +248,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/miniwob/run_infer.py

@@ -10,10 +10,13 @@ import pandas as pd
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
+    update_llm_config_for_completions_logging,
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
@@ -29,7 +32,10 @@ from openhands.events.action import (
     CmdRunAction,
     MessageAction,
 )
-from openhands.events.observation import CmdOutputObservation
+from openhands.events.observation import (
+    BrowserOutputObservation,
+    CmdOutputObservation,
+)
 from openhands.runtime.base import Runtime
 from openhands.runtime.browser.browser_env import (
     BROWSER_EVAL_GET_GOAL_ACTION,
@@ -37,7 +43,11 @@ from openhands.runtime.browser.browser_env import (
 )
 from openhands.utils.async_utils import call_async_from_sync
 
-SUPPORTED_AGENT_CLS = {'BrowsingAgent'}
+SUPPORTED_AGENT_CLS = {'BrowsingAgent', 'CodeActAgent'}
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
 
 
 def get_config(
@@ -47,25 +57,32 @@ def get_config(
     config = AppConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
-        runtime='eventstream',
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
         max_iterations=metadata.max_iterations,
         sandbox=SandboxConfig(
             base_container_image='xingyaoww/od-eval-miniwob:v1.0',
             enable_auto_lint=True,
             use_host_network=False,
             browsergym_eval_env=env_id,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_remote_runtime_alive=False,
         ),
         # do not mount workspace
         workspace_base=None,
         workspace_mount_path=None,
     )
-    config.set_llm_config(metadata.llm_config)
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, env_id
+        )
+    )
     return config
 
 
 def initialize_runtime(
     runtime: Runtime,
-) -> str:
+) -> tuple[str, BrowserOutputObservation]:
     """Initialize the runtime for the agent.
 
     This function is called before the runtime is used to run the agent.
@@ -85,8 +102,14 @@ def initialize_runtime(
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     goal = obs.content
 
+    # Run noop to get the initial browser observation (e.g., the page URL & content)
+    action = BrowseInteractiveAction(browser_actions='noop(1000)')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
-    return goal
+    return goal, obs
 
 
 def complete_runtime(
@@ -117,7 +140,7 @@ def process_instance(
     metadata: EvalMetadata,
     reset_logger: bool = True,
 ) -> EvalOutput:
-    env_id = instance.id
+    env_id = instance.instance_id
     config = get_config(metadata, env_id)
 
     # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
@@ -129,7 +152,12 @@ def process_instance(
 
     runtime = create_runtime(config)
     call_async_from_sync(runtime.connect)
-    task_str = initialize_runtime(runtime)
+    task_str, obs = initialize_runtime(runtime)
+
+    task_str += (
+        f'\nInitial browser state (output of `noop(1000)`):\n{obs.get_agent_obs_text()}'
+    )
+
     state: State | None = asyncio.run(
         run_controller(
             config=config,
@@ -137,6 +165,9 @@ def process_instance(
                 content=task_str
             ),  # take output from initialize_runtime
             runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
         )
     )
 
@@ -152,19 +183,19 @@ def process_instance(
 
     # Instruction is the first message from the USER
     instruction = ''
-    for event in state.history.get_events():
+    for event in state.history:
         if isinstance(event, MessageAction):
             instruction = event.content
             break
 
     return_val = complete_runtime(runtime)
     logger.info(f'Return value from complete_runtime: {return_val}')
-    reward = max(return_val['rewards'])
+    reward = max(return_val['rewards'], default=0)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/mint/run_infer.py

@@ -13,6 +13,7 @@ from evaluation.mint.tasks import Task
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -28,6 +29,7 @@ from openhands.core.config import (
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
 from openhands.events.action import (
+    Action,
     CmdRunAction,
     MessageAction,
 )
@@ -45,7 +47,10 @@ def codeact_user_response_mint(state: State, task: Task, task_config: dict[str,
         task=task,
         task_config=task_config,
     )
-    last_action = state.history.get_last_action()
+    last_action = next(
+        (event for event in reversed(state.history) if isinstance(event, Action)),
+        None,
+    )
     result_state: TaskState = env.step(last_action.message or '')
 
     state.extra_data['task_state'] = result_state
@@ -202,7 +207,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/ml_bench/run_infer.py

@@ -24,6 +24,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -256,7 +257,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/scienceagentbench/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-bookworm
+
+
+# For OpenHands agents to explore the dataset directories, please download the full benchmark [here](https://buckeyemailosu-my.sharepoint.com/:u:/g/personal/chen_8336_buckeyemail_osu_edu/EQuA6uJ3CtRHvRfZ2GiN1tYBRVJE4DSUD10MW61fr7HuSQ?e=sCBegG) and unzip it with password `scienceagentbench`.
+# **Please DO NOT redistribute the unzipped data files online.**
+# It will download a benchmark.zip file to the current directory.
+# unzip it and put the benchmark folder under evaluation/scienceagentbench/
+
+RUN mkdir -p /benchmark
+COPY benchmark /benchmark
+
+RUN mkdir -p /workspace
+WORKDIR /workspace
+
+# pushd evaluation/scienceagentbench
+# docker build -t xingyaoww/openhands-eval-scienceagentbench .
+# popd

evaluation/scienceagentbench/Dockerfile.evaluator

@@ -0,0 +1,25 @@
+FROM mambaorg/micromamba:debian12
+
+USER root
+# For https://github.com/OSU-NLP-Group/ScienceAgentBench/tree/main?tab=readme-ov-file#code-generation-with-agents
+
+RUN micromamba create -n sci-agent-eval python=3.10 pip setuptools wheel
+RUN micromamba run -n sci-agent-eval pip install pip-tools
+
+RUN mkdir -p /workspace
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y git
+
+RUN git clone https://github.com/OSU-NLP-Group/ScienceAgentBench.git /workspace/
+RUN git checkout 4eddc7db6449a5ade3e37285747c8b208cd54ce7
+
+RUN micromamba create -n sci-agent python=3.10 pip setuptools wheel
+RUN micromamba run -n sci-agent pip install -r requirements.txt
+
+# Replace all occurence of conda with micromamba under the /workspace
+RUN find ./ -type f -exec sed -i 's/conda/micromamba/g' {} \;
+
+# pushd evaluation/scienceagentbench
+# docker build -t xingyaoww/openhands-eval-scienceagentbench-evaluator -f Dockerfile.evaluator .
+# popd

evaluation/scienceagentbench/README.md

@@ -0,0 +1,54 @@
+# ScienceAgentBench Evaluation with OpenHands
+
+This folder contains the evaluation harness for [ScienceAgentBench](https://osu-nlp-group.github.io/ScienceAgentBench/) (paper: https://arxiv.org/abs/2410.05080).
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+
+## Setup ScienceAgentBench
+
+To prevent benchmark data contamination, we only provide the annotation sheet on [Huggingface](https://huggingface.co/datasets/osunlp/ScienceAgentBench), which includes all necessary *inputs* to run an agent.
+
+## Run Inference on ScienceAgentBench
+
+```bash
+./evaluation/scienceagentbench/scripts/run_infer.sh [model_config] [git-version] [use_knowledge] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
+
+# Example
+./evaluation/scienceagentbench/scripts/run_infer.sh llm.eval_gpt4o 0.9.3
+```
+
+where `model_config` is mandatory, and the rest are optional.
+
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
+LLM settings, as defined in your `config.toml`.
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
+like to evaluate. It could also be a release tag like `0.6.2`.
+- `use_knowledge`, e.g. `true`, specifies whether allowing the agent to use expert-provided knowledge as additional input or not. By default, it is set to `false`.
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
+to `CodeActAgent`.
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
+default, the script evaluates the entire SWE-bench_Lite test set (300 issues). Note:
+in order to use `eval_limit`, you must also set `agent`.
+- `max_iter`, e.g. `20`, is the maximum number of iterations for the agent to run. By
+default, it is set to 30.
+- `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
+default, it is set to 1.
+
+## Evaluate Generated Programs
+
+### Extract Necessary Information from OpenHands Log
+
+After the inference is completed, you may use the following command to extract necessary information from the output log for evaluation:
+
+```bash
+python post_proc.py [log_fname]
+```
+- `log_fname`, e.g. `evaluation/.../output.jsonl`, is the automatically saved trajectory log of an OpenHands agent.
+
+Output will be write to e.g. `evaluation/.../output.converted.jsonl`
+
+### Run evaluation
+
+Please follow the steps [here](https://github.com/OSU-NLP-Group/ScienceAgentBench/tree/main?tab=readme-ov-file#evaluation-of-generated-code) to evaluate the generated programs.

evaluation/scienceagentbench/post_proc.py

@@ -0,0 +1,30 @@
+import json
+from argparse import ArgumentParser
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument(
+        'log_fname',
+        type=str,
+    )
+    args = parser.parse_args()
+
+    fname = args.log_fname
+    out_fname = args.log_fname.replace('.jsonl', '.converted.jsonl')
+
+    log = [json.loads(line) for line in open(fname)]
+
+    simple_log = [
+        json.dumps(
+            {
+                'instance_id': ex['instance_id'],
+                'instruction': ex['instruction'],
+                'test_result': ex['test_result'],
+                'cost': ex['metrics']['accumulated_cost'],
+            }
+        )
+        for ex in log
+    ]
+
+    with open(out_fname, 'w+', encoding='utf-8') as f:
+        f.write('\n'.join(simple_log))

evaluation/scienceagentbench/run_infer.py

@@ -0,0 +1,292 @@
+import asyncio
+import os
+from typing import Any
+
+import pandas as pd
+from datasets import load_dataset
+from tqdm import tqdm
+
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    codeact_user_response,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    get_parser,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
+
+LOCAL_DATASET_PATH = os.path.join(os.path.dirname(__file__), 'benchmark')
+
+
+def format_task_dict(example, use_knowledge):
+    task = {
+        'instance_id': example['instance_id'],
+        'task_inst': example['task_inst'],
+        'dataset_path': '/benchmark/datasets/'
+        + example['dataset_folder_tree'].split('\n')[0][4:],
+        'dataset_folder_tree': example['dataset_folder_tree'],
+        'dataset_preview': example['dataset_preview'],
+        'pred_program_name': 'pred_' + example['gold_program_name'],
+    }
+
+    if use_knowledge:
+        task['task_inst'] += '\n' + str(example['domain_knowledge'])
+
+    return task
+
+
+def get_config(
+    metadata: EvalMetadata,
+    instance_id: str,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
+        max_budget_per_task=4,
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='docker.io/xingyaoww/openhands-eval-scienceagentbench',
+            enable_auto_lint=True,
+            use_host_network=False,
+            timeout=300,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_remote_runtime_alive=False,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config,
+            metadata.eval_output_dir,
+            instance_id,
+        )
+    )
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    # Set up workspace directories
+    action = CmdRunAction(command='mkdir -p /workspace/pred_programs')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(command='mkdir -p /workspace/pred_results')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    dataset_name = instance['dataset_folder_tree'].split('\n')[0][4:].rstrip('/')
+
+    # Copy the dataset to the workspace
+    dataset_dir = os.path.join(
+        LOCAL_DATASET_PATH,
+        'datasets',
+        dataset_name,
+    )
+    runtime.copy_to(dataset_dir, '/workspace/benchmark/datasets', recursive=True)
+
+    # Check the dataset exists
+    action = CmdRunAction(
+        command='cd /workspace/benchmark/datasets && ls',
+        keep_prompt=False,
+    )
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert obs.exit_code == 0
+    assert dataset_name in obs.content
+
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    test_result = {}
+
+    action = CmdRunAction(command='cd /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(
+        command=f'cat pred_programs/{instance.pred_program_name}',
+        keep_prompt=False,
+    )
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+
+    if obs.exit_code == 0:
+        test_result = {'program': obs.content}
+    else:
+        test_result = {'program': 'ERROR'}
+
+    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+    return test_result
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    instance_id = instance.instance_id.replace('/', '__')
+    config = get_config(metadata, instance_id)
+
+    # Set up the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance_id}.')
+
+    instruction = f"""You are an expert Python programming assistant that helps scientist users to write high-quality code to solve their tasks.
+Given a user request, you are expected to write a complete program that accomplishes the requested task and save any outputs to `/workspace/pred_results/` in the correct format.
+
+Here's the user request you need to work on:
+{instance.task_inst}
+
+You can access the dataset at `{instance.dataset_path}`. Here is the directory structure of the dataset:
+```
+{instance.dataset_folder_tree}
+```
+Here are some helpful previews for the dataset file(s):
+{instance.dataset_preview}
+
+Please save your program as `/workspace/pred_programs/{instance.pred_program_name}`.
+Then, please run the program to check and fix any errors.
+Please do NOT run the program in the background.
+If the program uses some packages that are incompatible, please figure out alternative implementations and do NOT restart the environment.
+
+"""
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+    initialize_runtime(runtime, instance)
+
+    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=MessageAction(content=instruction),
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
+    )
+
+    # ======= Attempt to evaluate the agent's edits =======
+    test_result = complete_runtime(runtime, instance)
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+    if state is None:
+        raise ValueError('State should not be None.')
+    metrics = state.metrics.get() if state.metrics else None
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result=test_result,
+    )
+    return output
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--use_knowledge',
+        type=str,
+        default='false',
+        choices=['true', 'false'],
+        help='use expert-provided knowledge or not',
+    )
+    args, _ = parser.parse_known_args()
+
+    sab_dataset = load_dataset('osunlp/ScienceAgentBench', split='validation')
+
+    dataset_processed = []
+    for example in tqdm(sab_dataset):
+        dataset_processed.append(
+            format_task_dict(example, args.use_knowledge == 'true')
+        )
+
+    dataset = pd.DataFrame(dataset_processed)
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        'ScienceAgentBench',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    dataset['instance_id'] = dataset['instance_id'].apply(str)
+    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
+    )

evaluation/scienceagentbench/scripts/run_infer.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+USE_KNOWLEDGE=$3
+AGENT=$4
+EVAL_LIMIT=$5
+NUM_WORKERS=$6
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$USE_KNOWLEDGE" ]; then
+  echo "Use knowledge not specified, use default False"
+  USE_KNOWLEDGE=false
+fi
+
+get_agent_version
+
+echo "AGENT: $AGENT"
+echo "AGENT_VERSION: $AGENT_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+COMMAND="poetry run python evaluation/scienceagentbench/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --use_knowledge $USE_KNOWLEDGE \
+  --max-iterations 30 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $AGENT_VERSION" \
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/swe_bench/eval_infer.py

@@ -239,7 +239,7 @@ def process_instance(
                         # Create a directory structure that matches the expected format
                         # NOTE: this is a hack to make the eval report format consistent
                         # with the original SWE-Bench eval script
-                        log_dir = os.path.join(temp_dir, 'logs', instance_id)
+                        log_dir = os.path.join(temp_dir, 'logs', instance_id.lower())
                         os.makedirs(log_dir, exist_ok=True)
                         test_output_path = os.path.join(log_dir, 'test_output.txt')
                         with open(test_output_path, 'w') as f:

evaluation/swe_bench/run_infer.py

@@ -20,6 +20,7 @@ from evaluation.utils.shared import (
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
+    update_llm_config_for_completions_logging,
 )
 from openhands.controller.state.state import State
 from openhands.core.config import (
@@ -40,6 +41,7 @@ from openhands.utils.async_utils import call_async_from_sync
 
 USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
 USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
 
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
     'CodeActAgent': codeact_user_response,
@@ -79,7 +81,7 @@ def get_instruction(instance: pd.Series, metadata: EvalMetadata):
             '</pr_description>\n\n'
             'Can you help me implement the necessary changes to the repository so that the requirements specified in the <pr_description> are met?\n'
             "I've already taken care of all changes to any of the test files described in the <pr_description>. This means you DON'T have to modify the testing logic or any of the tests in any way!\n"
-            'Your task is to make the minimal changes to non-tests files in the /repo directory to ensure the <pr_description> is satisfied.\n'
+            'Your task is to make the minimal changes to non-tests files in the /workspace directory to ensure the <pr_description> is satisfied.\n'
             'Follow these steps to resolve the issue:\n'
             '1. As a first step, it might be a good idea to explore the repo to familiarize yourself with its structure.\n'
             '2. Create a script to reproduce the error and execute it with `python <filename.py>` using the BashTool, to confirm the error\n'
@@ -88,6 +90,13 @@ def get_instruction(instance: pd.Series, metadata: EvalMetadata):
             '5. Think about edgecases and make sure your fix handles them as well\n'
             "Your thinking should be thorough and so it's fine if it's very long.\n"
         )
+
+    if RUN_WITH_BROWSING:
+        instruction += (
+            '<IMPORTANT!>\n'
+            'You SHOULD NEVER attempt to browse the web. '
+            '</IMPORTANT!>\n'
+        )
     return instruction
 
 
@@ -101,7 +110,7 @@ def get_instance_docker_image(instance_id: str) -> str:
     image_name = image_name.replace(
         '__', '_s_'
     )  # to comply with docker image naming convention
-    return DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name
+    return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name).lower()
 
 
 def get_config(
@@ -142,18 +151,14 @@ def get_config(
         workspace_base=None,
         workspace_mount_path=None,
     )
-    if metadata.llm_config.log_completions:
-        metadata.llm_config.log_completions_folder = os.path.join(
-            metadata.eval_output_dir, 'llm_completions', instance['instance_id']
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
         )
-        logger.info(
-            f'Logging LLM completions for instance {instance["instance_id"]} to '
-            f'{metadata.llm_config.log_completions_folder}'
-        )
-    config.set_llm_config(metadata.llm_config)
+    )
     agent_config = AgentConfig(
         codeact_enable_jupyter=False,
-        codeact_enable_browsing_delegate=False,
+        codeact_enable_browsing=RUN_WITH_BROWSING,
         codeact_enable_llm_editor=False,
     )
     config.set_agent_config(agent_config)
@@ -438,7 +443,8 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    histories = [event_to_dict(event) for event in state.history.get_events()]
+    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
+    histories = [event_to_dict(event) for event in state.history]
     metrics = state.metrics.get() if state.metrics else None
 
     # Save the output

evaluation/swe_bench/scripts/run_infer.sh

@@ -34,6 +34,11 @@ if [ -z "$USE_INSTANCE_IMAGE" ]; then
   USE_INSTANCE_IMAGE=true
 fi
 
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
 
 if [ -z "$DATASET" ]; then
   echo "DATASET not specified, use default princeton-nlp/SWE-bench_Lite"
@@ -47,6 +52,8 @@ fi
 
 export USE_INSTANCE_IMAGE=$USE_INSTANCE_IMAGE
 echo "USE_INSTANCE_IMAGE: $USE_INSTANCE_IMAGE"
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
 
 get_agent_version
 
@@ -67,6 +74,10 @@ if [ "$USE_HINT_TEXT" = false ]; then
   EVAL_NOTE="$EVAL_NOTE-no-hint"
 fi
 
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
 if [ -n "$EXP_NAME" ]; then
   EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
 fi

evaluation/toolqa/run_infer.py

@@ -9,6 +9,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -126,7 +127,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
         raise ValueError('State should not be None.')
 
     # retrieve the last message from the agent
-    model_answer_raw = state.history.get_last_agent_message()
+    model_answer_raw = state.get_last_agent_message()
 
     # attempt to parse model_answer
     correct = eval_answer(str(model_answer_raw), str(answer))
@@ -137,7 +138,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/utils/shared.py

@@ -18,6 +18,9 @@ from openhands.core.logger import get_console_handler
 from openhands.core.logger import openhands_logger as logger
 from openhands.events.action import Action
 from openhands.events.action.message import MessageAction
+from openhands.events.event import Event
+from openhands.events.serialization.event import event_to_dict
+from openhands.events.utils import get_pairs_from_events
 
 
 class EvalMetadata(BaseModel):
@@ -112,7 +115,14 @@ def codeact_user_response(
     if state.history:
         # check if the last action has an answer, if so, early exit
         if try_parse is not None:
-            last_action = state.history.get_last_action()
+            last_action = next(
+                (
+                    event
+                    for event in reversed(state.history)
+                    if isinstance(event, Action)
+                ),
+                None,
+            )
             ans = try_parse(last_action)
             if ans is not None:
                 return '/exit'
@@ -120,7 +130,7 @@ def codeact_user_response(
         # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:
@@ -411,3 +421,35 @@ def reset_logger_for_multiprocessing(
     )
     file_handler.setLevel(logging.INFO)
     logger.addHandler(file_handler)
+
+
+def update_llm_config_for_completions_logging(
+    llm_config: LLMConfig,
+    eval_output_dir: str,
+    instance_id: str,
+) -> LLMConfig:
+    """Update the LLM config for logging completions."""
+    if llm_config.log_completions:
+        llm_config.log_completions_folder = os.path.join(
+            eval_output_dir, 'llm_completions', instance_id
+        )
+        logger.info(
+            f'Logging LLM completions for instance {instance_id} to '
+            f'{llm_config.log_completions_folder}'
+        )
+    return llm_config
+
+
+# history is now available as a filtered stream of events, rather than list of pairs of (Action, Observation)
+# we rebuild the pairs here
+# for compatibility with the existing output format in evaluations
+# remove this when it's no longer necessary
+def compatibility_for_eval_history_pairs(
+    history: list[Event],
+) -> list[tuple[dict, dict]]:
+    history_pairs = []
+
+    for action, observation in get_pairs_from_events(history):
+        history_pairs.append((event_to_dict(action), event_to_dict(observation)))
+
+    return history_pairs

evaluation/webarena/run_infer.py

@@ -10,6 +10,7 @@ import pandas as pd
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -166,7 +167,7 @@ def process_instance(
 
     # Instruction is the first message from the USER
     instruction = ''
-    for event in state.history.get_events():
+    for event in state.history:
         if isinstance(event, MessageAction):
             instruction = event.content
             break
@@ -178,7 +179,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

frontend/.eslintrc

@@ -84,4 +84,4 @@
       }
     }
   ]
-}
+}

frontend/.gitignore

@@ -1,4 +1,9 @@
 # i18n translation files make by script using `make build`
 public/locales/**/*
 src/i18n/declaration.ts
-.env
+.env
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

frontend/__tests__/components/chat/chat-input.test.tsx

@@ -1,5 +1,5 @@
 import userEvent from "@testing-library/user-event";
-import { render, screen } from "@testing-library/react";
+import { fireEvent, render, screen } from "@testing-library/react";
 import { describe, afterEach, vi, it, expect } from "vitest";
 import { ChatInput } from "#/components/chat-input";
 
@@ -158,4 +158,46 @@ describe("ChatInput", () => {
     await user.tab();
     expect(onBlurMock).toHaveBeenCalledOnce();
   });
+
+  it("should handle text paste correctly", () => {
+    const onSubmit = vi.fn();
+    const onChange = vi.fn();
+
+    render(<ChatInput onSubmit={onSubmit} onChange={onChange} />);
+
+    const input = screen.getByTestId("chat-input").querySelector("textarea");
+    expect(input).toBeTruthy();
+
+    // Fire paste event with text data
+    fireEvent.paste(input!, {
+      clipboardData: {
+        getData: (type: string) => type === 'text/plain' ? 'test paste' : '',
+        files: []
+      }
+    });
+  });
+
+  it("should handle image paste correctly", () => {
+    const onSubmit = vi.fn();
+    const onImagePaste = vi.fn();
+
+    render(<ChatInput onSubmit={onSubmit} onImagePaste={onImagePaste} />);
+
+    const input = screen.getByTestId("chat-input").querySelector("textarea");
+    expect(input).toBeTruthy();
+
+    // Create a paste event with an image file
+    const file = new File(["dummy content"], "image.png", { type: "image/png" });
+
+    // Fire paste event with image data
+    fireEvent.paste(input!, {
+      clipboardData: {
+        getData: () => '',
+        files: [file]
+      }
+    });
+
+    // Verify image paste was handled
+    expect(onImagePaste).toHaveBeenCalledWith([file]);
+  });
 });

frontend/__tests__/components/chat/chat-interface.test.tsx

@@ -1,19 +1,156 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
-import { render, screen, within } from "@testing-library/react";
+import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { act, screen, waitFor, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
+import { renderWithProviders } from "test-utils";
 import { ChatInterface } from "#/components/chat-interface";
-import { SocketProvider } from "#/context/socket";
+import { addUserMessage } from "#/state/chatSlice";
+import { SUGGESTIONS } from "#/utils/suggestions";
+import * as ChatSlice from "#/state/chatSlice";
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const renderChatInterface = (messages: (Message | ErrorMessage)[]) =>
-  render(<ChatInterface />, { wrapper: SocketProvider });
+  renderWithProviders(<ChatInterface />);
+
+describe("Empty state", () => {
+  const { send: sendMock } = vi.hoisted(() => ({
+    send: vi.fn(),
+  }));
+
+  const { useSocket: useSocketMock } = vi.hoisted(() => ({
+    useSocket: vi.fn(() => ({ send: sendMock, runtimeActive: true })),
+  }));
+
+  beforeAll(() => {
+    vi.mock("#/context/socket", async (importActual) => ({
+      ...(await importActual<typeof import("#/context/socket")>()),
+      useSocket: useSocketMock,
+    }));
+  });
 
-describe.skip("ChatInterface", () => {
   afterEach(() => {
     vi.clearAllMocks();
   });
 
-  it.todo("should render suggestions if empty");
+  it("should render suggestions if empty", () => {
+    const { store } = renderWithProviders(<ChatInterface />, {
+      preloadedState: {
+        chat: { messages: [] },
+      },
+    });
+
+    expect(screen.getByTestId("suggestions")).toBeInTheDocument();
+
+    act(() => {
+      store.dispatch(
+        addUserMessage({
+          content: "Hello",
+          imageUrls: [],
+          timestamp: new Date().toISOString(),
+        }),
+      );
+    });
+
+    expect(screen.queryByTestId("suggestions")).not.toBeInTheDocument();
+  });
+
+  it("should render the default suggestions", () => {
+    renderWithProviders(<ChatInterface />, {
+      preloadedState: {
+        chat: { messages: [] },
+      },
+    });
+
+    const suggestions = screen.getByTestId("suggestions");
+    const repoSuggestions = Object.keys(SUGGESTIONS.repo);
+
+    // check that there are at most 4 suggestions displayed
+    const displayedSuggestions = within(suggestions).getAllByRole("button");
+    expect(displayedSuggestions.length).toBeLessThanOrEqual(4);
+
+    // Check that each displayed suggestion is one of the repo suggestions
+    displayedSuggestions.forEach((suggestion) => {
+      expect(repoSuggestions).toContain(suggestion.textContent);
+    });
+  });
+
+  it.fails(
+    "should load the a user message to the input when selecting",
+    async () => {
+      // this is to test that the message is in the UI before the socket is called
+      useSocketMock.mockImplementation(() => ({
+        send: sendMock,
+        runtimeActive: false, // mock an inactive runtime setup
+      }));
+      const addUserMessageSpy = vi.spyOn(ChatSlice, "addUserMessage");
+      const user = userEvent.setup();
+      const { store } = renderWithProviders(<ChatInterface />, {
+        preloadedState: {
+          chat: { messages: [] },
+        },
+      });
+
+      const suggestions = screen.getByTestId("suggestions");
+      const displayedSuggestions = within(suggestions).getAllByRole("button");
+      const input = screen.getByTestId("chat-input");
+
+      await user.click(displayedSuggestions[0]);
+
+      // user message loaded to input
+      expect(addUserMessageSpy).not.toHaveBeenCalled();
+      expect(screen.queryByTestId("suggestions")).toBeInTheDocument();
+      expect(store.getState().chat.messages).toHaveLength(0);
+      expect(input).toHaveValue(displayedSuggestions[0].textContent);
+    },
+  );
+
+  it.fails(
+    "should send the message to the socket only if the runtime is active",
+    async () => {
+      useSocketMock.mockImplementation(() => ({
+        send: sendMock,
+        runtimeActive: false, // mock an inactive runtime setup
+      }));
+      const user = userEvent.setup();
+      const { rerender } = renderWithProviders(<ChatInterface />, {
+        preloadedState: {
+          chat: { messages: [] },
+        },
+      });
+
+      const suggestions = screen.getByTestId("suggestions");
+      const displayedSuggestions = within(suggestions).getAllByRole("button");
+
+      await user.click(displayedSuggestions[0]);
+      expect(sendMock).not.toHaveBeenCalled();
+
+      useSocketMock.mockImplementation(() => ({
+        send: sendMock,
+        runtimeActive: true, // mock an active runtime setup
+      }));
+      rerender(<ChatInterface />);
+
+      await waitFor(() =>
+        expect(sendMock).toHaveBeenCalledWith(expect.any(String)),
+      );
+    },
+  );
+});
+
+describe.skip("ChatInterface", () => {
+  beforeAll(() => {
+    // mock useScrollToBottom hook
+    vi.mock("#/hooks/useScrollToBottom", () => ({
+      useScrollToBottom: vi.fn(() => ({
+        scrollDomToBottom: vi.fn(),
+        onChatBodyScroll: vi.fn(),
+        hitBottom: vi.fn(),
+      })),
+    }));
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
 
   it("should render messages", () => {
     const messages: Message[] = [
@@ -128,14 +265,14 @@ describe.skip("ChatInterface", () => {
         timestamp: new Date().toISOString(),
       },
       {
-        error: "Woops!",
+        error: true,
+        id: "",
         message: "Something went wrong",
       },
     ];
     renderChatInterface(messages);
 
     const error = screen.getByTestId("error-message");
-    expect(within(error).getByText("Woops!")).toBeInTheDocument();
     expect(within(error).getByText("Something went wrong")).toBeInTheDocument();
   });
 

frontend/__tests__/components/feedback-form.test.tsx

@@ -5,7 +5,6 @@ import { FeedbackForm } from "#/components/feedback-form";
 
 describe("FeedbackForm", () => {
   const user = userEvent.setup();
-  const onSubmitMock = vi.fn();
   const onCloseMock = vi.fn();
 
   afterEach(() => {
@@ -13,7 +12,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should render correctly", () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
 
     screen.getByLabelText("Email");
     screen.getByLabelText("Private");
@@ -24,7 +23,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should switch between private and public permissions", async () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
     const privateRadio = screen.getByLabelText("Private");
     const publicRadio = screen.getByLabelText("Public");
 
@@ -40,69 +39,11 @@ describe("FeedbackForm", () => {
     expect(publicRadio).not.toBeChecked();
   });
 
-  it("should call onSubmit when the form is submitted", async () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
-    const email = screen.getByLabelText("Email");
-
-    await user.type(email, "test@test.test");
-    await user.click(screen.getByRole("button", { name: "Submit" }));
-
-    expect(onSubmitMock).toHaveBeenCalledWith("private", "test@test.test"); // private is the default value
-  });
-
-  it("should not call onSubmit when the email is invalid", async () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
-    const email = screen.getByLabelText("Email");
-    const submitButton = screen.getByRole("button", { name: "Submit" });
-
-    await user.click(submitButton);
-
-    expect(onSubmitMock).not.toHaveBeenCalled();
-
-    await user.type(email, "test");
-    await user.click(submitButton);
-
-    expect(onSubmitMock).not.toHaveBeenCalled();
-  });
-
-  it("should submit public permissions when the public radio is checked", async () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
-    const email = screen.getByLabelText("Email");
-    const publicRadio = screen.getByLabelText("Public");
-
-    await user.type(email, "test@test.test");
-    await user.click(publicRadio);
-    await user.click(screen.getByRole("button", { name: "Submit" }));
-
-    expect(onSubmitMock).toHaveBeenCalledWith("public", "test@test.test");
-  });
-
   it("should call onClose when the close button is clicked", async () => {
-    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
     await user.click(screen.getByRole("button", { name: "Cancel" }));
 
-    expect(onSubmitMock).not.toHaveBeenCalled();
     expect(onCloseMock).toHaveBeenCalled();
   });
 
-  it("should disable the buttons if isSubmitting is true", () => {
-    const { rerender } = render(
-      <FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />,
-    );
-    const submitButton = screen.getByRole("button", { name: "Submit" });
-    const cancelButton = screen.getByRole("button", { name: "Cancel" });
-
-    expect(submitButton).not.toBeDisabled();
-    expect(cancelButton).not.toBeDisabled();
-
-    rerender(
-      <FeedbackForm
-        onSubmit={onSubmitMock}
-        onClose={onCloseMock}
-        isSubmitting
-      />,
-    );
-    expect(submitButton).toBeDisabled();
-    expect(cancelButton).toBeDisabled();
-  });
 });

frontend/__tests__/components/file-explorer/FileExplorer.test.tsx

@@ -16,13 +16,16 @@ vi.mock("../../services/fileService", async () => ({
 }));
 
 const renderFileExplorerWithRunningAgentState = () =>
-  renderWithProviders(<FileExplorer error={null} />, {
-    preloadedState: {
-      agent: {
-        curAgentState: AgentState.RUNNING,
+  renderWithProviders(
+    <FileExplorer error={null} isOpen onToggle={() => {}} />,
+    {
+      preloadedState: {
+        agent: {
+          curAgentState: AgentState.RUNNING,
+        },
       },
     },
-  });
+  );
 
 describe.skip("FileExplorer", () => {
   afterEach(() => {

frontend/__tests__/components/interactive-chat-box.test.tsx

@@ -25,6 +25,21 @@ describe("InteractiveChatBox", () => {
     within(chatBox).getByTestId("upload-image-input");
   });
 
+  it.fails("should set custom values", () => {
+    render(
+      <InteractiveChatBox
+        onSubmit={onSubmitMock}
+        onStop={onStopMock}
+        value="Hello, world!"
+      />,
+    );
+
+    const chatBox = screen.getByTestId("interactive-chat-box");
+    const chatInput = within(chatBox).getByTestId("chat-input");
+
+    expect(chatInput).toHaveValue("Hello, world!");
+  });
+
   it("should display the image previews when images are uploaded", async () => {
     const user = userEvent.setup();
     render(<InteractiveChatBox onSubmit={onSubmitMock} onStop={onStopMock} />);

frontend/__tests__/components/suggestion-item.test.tsx

@@ -0,0 +1,30 @@
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { SuggestionItem } from "#/components/suggestion-item";
+
+describe("SuggestionItem", () => {
+  const suggestionItem = { label: "suggestion1", value: "a long text value" };
+  const onClick = vi.fn();
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should render a suggestion", () => {
+    render(<SuggestionItem suggestion={suggestionItem} onClick={onClick} />);
+
+    expect(screen.getByTestId("suggestion")).toBeInTheDocument();
+    expect(screen.getByText(/suggestion1/i)).toBeInTheDocument();
+  });
+
+  it("should call onClick when clicking a suggestion", async () => {
+    const user = userEvent.setup();
+    render(<SuggestionItem suggestion={suggestionItem} onClick={onClick} />);
+
+    const suggestion = screen.getByTestId("suggestion");
+    await user.click(suggestion);
+
+    expect(onClick).toHaveBeenCalledWith("a long text value");
+  });
+});

frontend/__tests__/components/suggestions.test.tsx

@@ -0,0 +1,60 @@
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Suggestions } from "#/components/suggestions";
+
+describe("Suggestions", () => {
+  const firstSuggestion = {
+    label: "first-suggestion",
+    value: "value-of-first-suggestion",
+  };
+  const secondSuggestion = {
+    label: "second-suggestion",
+    value: "value-of-second-suggestion",
+  };
+  const suggestions = [firstSuggestion, secondSuggestion];
+
+  const onSuggestionClickMock = vi.fn();
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should render suggestions", () => {
+    render(
+      <Suggestions
+        suggestions={suggestions}
+        onSuggestionClick={onSuggestionClickMock}
+      />,
+    );
+
+    expect(screen.getByTestId("suggestions")).toBeInTheDocument();
+    const suggestionElements = screen.getAllByTestId("suggestion");
+
+    expect(suggestionElements).toHaveLength(2);
+    expect(suggestionElements[0]).toHaveTextContent("first-suggestion");
+    expect(suggestionElements[1]).toHaveTextContent("second-suggestion");
+  });
+
+  it("should call onSuggestionClick when clicking a suggestion", async () => {
+    const user = userEvent.setup();
+    render(
+      <Suggestions
+        suggestions={suggestions}
+        onSuggestionClick={onSuggestionClickMock}
+      />,
+    );
+
+    const suggestionElements = screen.getAllByTestId("suggestion");
+
+    await user.click(suggestionElements[0]);
+    expect(onSuggestionClickMock).toHaveBeenCalledWith(
+      "value-of-first-suggestion",
+    );
+
+    await user.click(suggestionElements[1]);
+    expect(onSuggestionClickMock).toHaveBeenCalledWith(
+      "value-of-second-suggestion",
+    );
+  });
+});

frontend/__tests__/initial-query.test.tsx

@@ -0,0 +1,17 @@
+import { describe, it, expect } from "vitest";
+import store from "../src/store";
+import { setInitialQuery, clearInitialQuery } from "../src/state/initial-query-slice";
+
+describe("Initial Query Behavior", () => {
+  it("should clear initial query when clearInitialQuery is dispatched", () => {
+    // Set up initial query in the store
+    store.dispatch(setInitialQuery("test query"));
+    expect(store.getState().initalQuery.initialQuery).toBe("test query");
+
+    // Clear the initial query
+    store.dispatch(clearInitialQuery());
+
+    // Verify initial query is cleared
+    expect(store.getState().initalQuery.initialQuery).toBeNull();
+  });
+});

frontend/__tests__/routes/_oh.app.test.tsx

@@ -0,0 +1,5 @@
+import { describe, it } from "vitest";
+
+describe("App", () => {
+  it.todo("should render");
+});

frontend/__tests__/utils/cache.test.ts

@@ -0,0 +1,74 @@
+import { afterEach } from "node:test";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { cache } from "#/utils/cache";
+
+describe("Cache", () => {
+  const testKey = "key";
+  const testData = { message: "Hello, world!" };
+  const testTTL = 1000; // 1 second
+
+  beforeEach(() => {
+    localStorage.clear();
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("sets data in localStorage with expiration", () => {
+    cache.set(testKey, testData, testTTL);
+    const cachedEntry = JSON.parse(
+      localStorage.getItem(`app_cache_${testKey}`) || "",
+    );
+
+    expect(cachedEntry.data).toEqual(testData);
+    expect(cachedEntry.expiration).toBeGreaterThan(Date.now());
+  });
+
+  it("gets data from localStorage if not expired", () => {
+    cache.set(testKey, testData, testTTL);
+
+    expect(cache.get(testKey)).toEqual(testData);
+  });
+
+  it("should expire after 5 minutes by default", () => {
+    cache.set(testKey, testData);
+    expect(cache.get(testKey)).not.toBeNull();
+
+    vi.advanceTimersByTime(5 * 60 * 1000 + 1);
+
+    expect(cache.get(testKey)).toBeNull();
+    expect(localStorage.getItem(`app_cache_${testKey}`)).toBeNull();
+  });
+
+  it("returns null if cached data is expired", () => {
+    cache.set(testKey, testData, testTTL);
+
+    vi.advanceTimersByTime(testTTL + 1);
+    expect(cache.get(testKey)).toBeNull();
+    expect(localStorage.getItem(`app_cache_${testKey}`)).toBeNull();
+  });
+
+  it("deletes data from localStorage", () => {
+    cache.set(testKey, testData, testTTL);
+    cache.delete(testKey);
+
+    expect(localStorage.getItem(`app_cache_${testKey}`)).toBeNull();
+  });
+
+  it("clears all data with the app prefix from localStorage", () => {
+    cache.set(testKey, testData, testTTL);
+    cache.set("anotherKey", { data: "More data" }, testTTL);
+    cache.clearAll();
+
+    expect(localStorage.length).toBe(0);
+  });
+
+  it("does not retrieve non-prefixed data from localStorage when clearing", () => {
+    localStorage.setItem("nonPrefixedKey", "should remain");
+    cache.set(testKey, testData, testTTL);
+    cache.clearAll();
+    expect(localStorage.getItem("nonPrefixedKey")).toBe("should remain");
+  });
+});

frontend/__tests__/utils/extractModelAndProvider.test.ts

@@ -78,4 +78,3 @@ describe("extractModelAndProvider", () => {
     });
   });
 });
-

frontend/__tests__/utils/organizeModelsAndProviders.test.ts

@@ -63,4 +63,3 @@ test("organizeModelsAndProviders", () => {
     },
   });
 });
-

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "openhands-frontend",
-  "version": "0.11.0",
+  "version": "0.12.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "openhands-frontend",
-      "version": "0.11.0",
+      "version": "0.12.3",
       "dependencies": {
         "@monaco-editor/react": "^4.6.0",
         "@nextui-org/react": "^2.4.8",
@@ -26,6 +26,7 @@
         "isbot": "^5.1.17",
         "jose": "^5.9.4",
         "monaco-editor": "^0.52.0",
+        "posthog-js": "^1.176.0",
         "react": "^18.3.1",
         "react-dom": "^18.3.1",
         "react-highlight": "^0.15.0",
@@ -45,6 +46,7 @@
         "ws": "^8.18.0"
       },
       "devDependencies": {
+        "@playwright/test": "^1.48.2",
         "@remix-run/dev": "^2.11.2",
         "@remix-run/testing": "^2.11.2",
         "@tailwindcss/typography": "^0.5.15",
@@ -61,6 +63,7 @@
         "@typescript-eslint/parser": "^7.18.0",
         "@vitest/coverage-v8": "^1.6.0",
         "autoprefixer": "^10.4.20",
+        "cross-env": "^7.0.3",
         "eslint": "^8.57.0",
         "eslint-config-airbnb": "^19.0.4",
         "eslint-config-airbnb-typescript": "^18.0.0",
@@ -3378,6 +3381,21 @@
         "url": "https://opencollective.com/unts"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.48.2",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.48.2.tgz",
+      "integrity": "sha512-54w1xCWfXuax7dz4W2M9uw0gDyh+ti/0K/MxcCUxChFh37kkdxPdfZDw5QBbuPUJHr1CiHJ1hXgSs+GgeQc5Zw==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.48.2"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@polka/url": {
       "version": "1.0.0-next.28",
       "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz",
@@ -7864,6 +7882,16 @@
         "node": ">=6.6.0"
       }
     },
+    "node_modules/core-js": {
+      "version": "3.38.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.38.1.tgz",
+      "integrity": "sha512-OP35aUorbU3Zvlx7pjsFdu1rGNnD4pgw/CWoYzRY3t2EzoVT7shKHY1dlAy3f41cGIO7ZDPQimhGFTlEYkG/Hw==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
+    },
     "node_modules/core-util-is": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -7896,6 +7924,24 @@
         }
       }
     },
+    "node_modules/cross-env": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
+      "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",
+      "dev": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.1"
+      },
+      "bin": {
+        "cross-env": "src/bin/cross-env.js",
+        "cross-env-shell": "src/bin/cross-env-shell.js"
+      },
+      "engines": {
+        "node": ">=10.14",
+        "npm": ">=6",
+        "yarn": ">=1"
+      }
+    },
     "node_modules/cross-fetch": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz",
@@ -9666,6 +9712,11 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/fflate": {
+      "version": "0.4.8",
+      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz",
+      "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA=="
+    },
     "node_modules/file-entry-cache": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@@ -19406,6 +19457,50 @@
         "pathe": "^1.1.2"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.48.2",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.48.2.tgz",
+      "integrity": "sha512-NjYvYgp4BPmiwfe31j4gHLa3J7bD2WiBz8Lk2RoSsmX38SVIARZ18VYjxLjAcDsAhA+F4iSEXTSGgjua0rrlgQ==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.48.2"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.48.2",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.48.2.tgz",
+      "integrity": "sha512-sjjw+qrLFlriJo64du+EK0kJgZzoQPsabGF4lBvsid+3CNIZIYLgnMj9V6JY5VhM2Peh20DJWIVpVljLLnlawA==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/possible-typed-array-names": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz",
@@ -19653,6 +19748,31 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
+    "node_modules/posthog-js": {
+      "version": "1.176.0",
+      "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.176.0.tgz",
+      "integrity": "sha512-T5XKNtRzp7q6CGb7Vc7wAI76rWap9fiuDUPxPsyPBPDkreKya91x9RIsSapAVFafwD1AEin1QMczCmt9Le9BWw==",
+      "dependencies": {
+        "core-js": "^3.38.1",
+        "fflate": "^0.4.8",
+        "preact": "^10.19.3",
+        "web-vitals": "^4.2.0"
+      }
+    },
+    "node_modules/posthog-js/node_modules/web-vitals": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-4.2.4.tgz",
+      "integrity": "sha512-r4DIlprAGwJ7YM11VZp4R884m0Vmgr6EAKe3P+kO0PPj3Unqyvv59rczf6UiGcb9Z8QxZVcqKNwv/g0WNdWwsw=="
+    },
+    "node_modules/preact": {
+      "version": "10.24.3",
+      "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.3.tgz",
+      "integrity": "sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/preact"
+      }
+    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.11.0",
+  "version": "0.12.3",
   "private": true,
   "type": "module",
   "engines": {
@@ -25,6 +25,7 @@
     "isbot": "^5.1.17",
     "jose": "^5.9.4",
     "monaco-editor": "^0.52.0",
+    "posthog-js": "^1.176.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-highlight": "^0.15.0",
@@ -44,11 +45,12 @@
     "ws": "^8.18.0"
   },
   "scripts": {
-    "dev": "npm run make-i18n && VITE_MOCK_API=false remix vite:dev",
-    "dev:mock": "npm run make-i18n && VITE_MOCK_API=true remix vite:dev",
+    "dev": "npm run make-i18n && cross-env VITE_MOCK_API=false remix vite:dev",
+    "dev:mock": "npm run make-i18n && cross-env VITE_MOCK_API=true remix vite:dev",
     "build": "npm run make-i18n && tsc && remix vite:build",
     "start": "npx sirv-cli build/ --single",
     "test": "vitest run",
+    "test:e2e": "playwright test",
     "test:coverage": "npm run make-i18n && vitest run --coverage",
     "dev_wsl": "VITE_WATCH_USE_POLLING=true vite",
     "preview": "vite preview",
@@ -70,6 +72,7 @@
     ]
   },
   "devDependencies": {
+    "@playwright/test": "^1.48.2",
     "@remix-run/dev": "^2.11.2",
     "@remix-run/testing": "^2.11.2",
     "@tailwindcss/typography": "^0.5.15",
@@ -86,6 +89,7 @@
     "@typescript-eslint/parser": "^7.18.0",
     "@vitest/coverage-v8": "^1.6.0",
     "autoprefixer": "^10.4.20",
+    "cross-env": "^7.0.3",
     "eslint": "^8.57.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^18.0.0",
@@ -116,4 +120,4 @@
       "public"
     ]
   }
-}
+}

frontend/playwright.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./tests",
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://127.0.0.1:3000",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"] },
+    },
+
+    {
+      name: "webkit",
+      use: { ...devices["Desktop Safari"] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: "npm run dev:mock -- --port 3000",
+    url: "http://127.0.0.1:3000",
+    reuseExistingServer: !process.env.CI,
+  },
+});

frontend/src/api/github.ts

@@ -122,6 +122,9 @@ export const retrieveGitHubUser = async (
       id: data.id,
       login: data.login,
       avatar_url: data.avatar_url,
+      company: data.company,
+      name: data.name,
+      email: data.email,
     };
 
     return user;
@@ -136,33 +139,6 @@ export const retrieveGitHubUser = async (
   return error;
 };
 
-/**
- * Given a GitHub token and a repository name, creates a repository for the authenticated user
- * @param token The GitHub token
- * @param repositoryName Name of the repository to create
- * @param description Description of the repository
- * @param isPrivate Boolean indicating if the repository should be private
- * @returns The created repository or an error response
- */
-export const createGitHubRepository = async (
-  token: string,
-  repositoryName: string,
-  description?: string,
-  isPrivate = true,
-): Promise<GitHubRepository | GitHubErrorReponse> => {
-  const response = await fetch("https://api.github.com/user/repos", {
-    method: "POST",
-    headers: generateGitHubAPIHeaders(token),
-    body: JSON.stringify({
-      name: repositoryName,
-      description,
-      private: isPrivate,
-    }),
-  });
-
-  return response.json();
-};
-
 export const retrieveLatestGitHubCommit = async (
   token: string,
   repository: string,

frontend/src/api/open-hands.ts

@@ -1,4 +1,5 @@
-import { getValidFallbackHost } from "#/utils/get-valid-fallback-host";
+import { request } from "#/services/api";
+import { cache } from "#/utils/cache";
 import {
   SaveFileSuccessResponse,
   FileUploadSuccessResponse,
@@ -9,36 +10,19 @@ import {
   GetConfigResponse,
 } from "./open-hands.types";
 
-/**
- * Generate the base URL of the OpenHands API
- * @returns Base URL of the OpenHands API
- */
-const generateBaseURL = () => {
-  const fallback = getValidFallbackHost();
-  const baseUrl = import.meta.env.VITE_BACKEND_BASE_URL || fallback;
-
-  if (typeof window === "undefined") {
-    return `http://${baseUrl}`;
-  }
-  return `${window.location.protocol}//${baseUrl}`;
-};
-
-/**
- * Class to interact with the OpenHands API
- */
 class OpenHands {
-  /**
-   * Base URL of the OpenHands API
-   */
-  static BASE_URL = generateBaseURL();
-
   /**
    * Retrieve the list of models available
    * @returns List of models available
    */
   static async getModels(): Promise<string[]> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/options/models`);
-    return response.json();
+    const cachedData = cache.get<string[]>("models");
+    if (cachedData) return cachedData;
+
+    const data = await request("/api/options/models");
+    cache.set("models", data);
+
+    return data;
   }
 
   /**
@@ -46,8 +30,13 @@ class OpenHands {
    * @returns List of agents available
    */
   static async getAgents(): Promise<string[]> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/options/agents`);
-    return response.json();
+    const cachedData = cache.get<string[]>("agents");
+    if (cachedData) return cachedData;
+
+    const data = await request(`/api/options/agents`);
+    cache.set("agents", data);
+
+    return data;
   }
 
   /**
@@ -55,178 +44,135 @@ class OpenHands {
    * @returns List of security analyzers available
    */
   static async getSecurityAnalyzers(): Promise<string[]> {
-    const response = await fetch(
-      `${OpenHands.BASE_URL}/api/options/security-analyzers`,
-    );
-    return response.json();
+    const cachedData = cache.get<string[]>("agents");
+    if (cachedData) return cachedData;
+
+    const data = await request(`/api/options/security-analyzers`);
+    cache.set("security-analyzers", data);
+
+    return data;
   }
 
   static async getConfig(): Promise<GetConfigResponse> {
-    const response = await fetch("config.json", {
-      headers: {
-        "Cache-Control": "no-cache",
-      },
-    });
-    return response.json();
+    const cachedData = cache.get<GetConfigResponse>("config");
+    if (cachedData) return cachedData;
+
+    const data = await request("/config.json");
+    cache.set("config", data);
+
+    return data;
   }
 
   /**
    * Retrieve the list of files available in the workspace
-   * @param token User token provided by the server
    * @param path Path to list files from
    * @returns List of files available in the given path. If path is not provided, it lists all the files in the workspace
    */
-  static async getFiles(token: string, path?: string): Promise<string[]> {
-    const url = new URL(`${OpenHands.BASE_URL}/api/list-files`);
-    if (path) url.searchParams.append("path", path);
-
-    const response = await fetch(url.toString(), {
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
-    return response.json();
+  static async getFiles(path?: string): Promise<string[]> {
+    let url = "/api/list-files";
+    if (path) url += `?path=${encodeURIComponent(path)}`;
+    return request(url);
   }
 
   /**
    * Retrieve the content of a file
-   * @param token User token provided by the server
    * @param path Full path of the file to retrieve
    * @returns Content of the file
    */
-  static async getFile(token: string, path: string): Promise<string> {
-    const url = new URL(`${OpenHands.BASE_URL}/api/select-file`);
-    url.searchParams.append("file", path);
-    const response = await fetch(url.toString(), {
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
-    const data = await response.json();
+  static async getFile(path: string): Promise<string> {
+    const url = `/api/select-file?file=${encodeURIComponent(path)}`;
+    const data = await request(url);
     return data.code;
   }
 
   /**
    * Save the content of a file
-   * @param token User token provided by the server
    * @param path Full path of the file to save
    * @param content Content to save in the file
    * @returns Success message or error message
    */
   static async saveFile(
-    token: string,
     path: string,
     content: string,
   ): Promise<SaveFileSuccessResponse | ErrorResponse> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/save-file`, {
+    return request(`/api/save-file`, {
       method: "POST",
       body: JSON.stringify({ filePath: path, content }),
       headers: {
-        Authorization: `Bearer ${token}`,
         "Content-Type": "application/json",
       },
     });
-
-    return response.json();
   }
 
   /**
    * Upload a file to the workspace
-   * @param token User token provided by the server
    * @param file File to upload
    * @returns Success message or error message
    */
   static async uploadFiles(
-    token: string,
     file: File[],
   ): Promise<FileUploadSuccessResponse | ErrorResponse> {
     const formData = new FormData();
     file.forEach((f) => formData.append("files", f));
 
-    const response = await fetch(`${OpenHands.BASE_URL}/api/upload-files`, {
+    return request(`/api/upload-files`, {
       method: "POST",
       body: formData,
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
     });
-
-    return response.json();
   }
 
   /**
    * Get the blob of the workspace zip
-   * @param token User token provided by the server
    * @returns Blob of the workspace zip
    */
-  static async getWorkspaceZip(token: string): Promise<Blob> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/zip-directory`, {
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
+  static async getWorkspaceZip(): Promise<Blob> {
+    const response = await request(`/api/zip-directory`, {}, false, true);
     return response.blob();
   }
 
   /**
    * Send feedback to the server
-   * @param token User token provided by the server
    * @param data Feedback data
    * @returns The stored feedback data
    */
-  static async sendFeedback(
-    token: string,
-    data: Feedback,
-  ): Promise<FeedbackResponse> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/submit-feedback`, {
+  static async submitFeedback(data: Feedback): Promise<FeedbackResponse> {
+    return request(`/api/submit-feedback`, {
       method: "POST",
       body: JSON.stringify(data),
       headers: {
-        Authorization: `Bearer ${token}`,
         "Content-Type": "application/json",
       },
     });
-
-    return response.json();
   }
 
   /**
-   * Get the GitHub access token
    * @param code Code provided by GitHub
    * @returns GitHub access token
    */
   static async getGitHubAccessToken(
     code: string,
   ): Promise<GitHubAccessTokenResponse> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/github/callback`, {
+    return request(`/api/github/callback`, {
       method: "POST",
       body: JSON.stringify({ code }),
       headers: {
         "Content-Type": "application/json",
       },
     });
-
-    return response.json();
   }
 
   /**
-   * Check if the user is authenticated
-   * @param login The user's GitHub login handle
-   * @returns Whether the user is authenticated
+   * Authenticate with GitHub token
+   * @returns Response with authentication status and user info if successful
    */
-  static async isAuthenticated(login: string): Promise<boolean> {
-    const response = await fetch(`${OpenHands.BASE_URL}/api/authenticate`, {
-      method: "POST",
-      body: JSON.stringify({ login }),
-      headers: {
-        "Content-Type": "application/json",
+  static async authenticate(): Promise<Response> {
+    return request(
+      `/api/authenticate`,
+      {
+        method: "POST",
       },
-    });
-
-    return response.status === 200;
+      true,
+    );
   }
 }
 

frontend/src/api/open-hands.types.ts

@@ -27,11 +27,16 @@ export interface GitHubAccessTokenResponse {
   access_token: string;
 }
 
+export interface AuthenticationResponse {
+  message: string;
+  login?: string; // Only present when allow list is enabled
+}
+
 export interface Feedback {
   version: string;
   email: string;
   token: string;
-  feedback: "positive" | "negative";
+  polarity: "positive" | "negative";
   permissions: "public" | "private";
   trajectory: unknown[];
 }

frontend/src/assets/arrow-send.svg

@@ -2,4 +2,4 @@
   <path fill-rule="evenodd" clip-rule="evenodd"
     d="M11.5304 6.46978L10.4697 7.53044L6.75006 3.81077L6.75006 17.0001H5.25006L5.25006 3.81077L1.53039 7.53044L0.469727 6.46978L6.00006 0.939453L11.5304 6.46978Z"
     fill="white" />
-</svg>
+</svg>

frontend/src/assets/branding/all-hands-logo-spark.svg

@@ -32,4 +32,4 @@
       <rect width="69" height="46" fill="white" transform="translate(0.5)" />
     </clipPath>
   </defs>
-</svg>
+</svg>

frontend/src/assets/branding/github-logo.svg

@@ -2,4 +2,4 @@
   <path
     d="M15.359 21V17.319C15.3974 16.8654 15.3314 16.4095 15.1651 15.9814C14.9989 15.5534 14.7363 15.1631 14.3949 14.8364C17.6154 14.5035 21 13.3716 21 8.17826C20.9997 6.85027 20.4489 5.57321 19.4615 4.61139C19.9291 3.44954 19.896 2.16532 19.3692 1.02548C19.3692 1.02548 18.159 0.692576 15.359 2.43321C13.0082 1.84237 10.5302 1.84237 8.17949 2.43321C5.37949 0.692576 4.16923 1.02548 4.16923 1.02548C3.64244 2.16532 3.60938 3.44954 4.07692 4.61139C3.08218 5.58034 2.53079 6.86895 2.53846 8.2068C2.53846 13.3621 5.92308 14.494 9.14359 14.865C8.80615 15.1883 8.54591 15.574 8.3798 15.9968C8.2137 16.4196 8.14544 16.8701 8.17949 17.319V21M8.17949 18.1465C3.05128 19.5732 3.05128 15.7686 1 15.293L8.17949 18.1465Z"
     stroke="white" stroke-linecap="round" stroke-linejoin="round" />
-</svg>
+</svg>

frontend/src/assets/clip.svg

@@ -2,4 +2,4 @@
   <path fill-rule="evenodd" clip-rule="evenodd"
     d="M10.5 3.75C9.25736 3.75 8.25 4.75736 8.25 6V16.5C8.25 18.5711 9.92893 20.25 12 20.25C14.0711 20.25 15.75 18.5711 15.75 16.5V7H17.25V16.5C17.25 19.3995 14.8995 21.75 12 21.75C9.1005 21.75 6.75 19.3995 6.75 16.5V6C6.75 3.92893 8.42893 2.25 10.5 2.25C12.5711 2.25 14.25 3.92893 14.25 6V16C14.25 17.2426 13.2426 18.25 12 18.25C10.7574 18.25 9.75 17.2426 9.75 16V7H11.25V16C11.25 16.4142 11.5858 16.75 12 16.75C12.4142 16.75 12.75 16.4142 12.75 16V6C12.75 4.75736 11.7426 3.75 10.5 3.75Z"
     fill="white" />
-</svg>
+</svg>

frontend/src/assets/clipboard.svg

@@ -32,4 +32,4 @@
       <rect width="54" height="75" fill="white" />
     </clipPath>
   </defs>
-</svg>
+</svg>

frontend/src/assets/default-user.svg

@@ -2,4 +2,4 @@
   <path
     d="M13.0919 10.5917C13.9089 9.94891 14.5052 9.06746 14.7979 8.06997C15.0906 7.07249 15.0652 6.00858 14.7251 5.02625C14.385 4.04391 13.7471 3.19202 12.9003 2.58907C12.0535 1.98612 11.0398 1.66211 10.0002 1.66211C8.9607 1.66211 7.947 1.98612 7.10018 2.58907C6.25336 3.19202 5.61553 4.04391 5.27542 5.02625C4.93531 6.00858 4.90984 7.07249 5.20254 8.06997C5.49525 9.06746 6.09158 9.94891 6.90858 10.5917C5.50864 11.1526 4.28715 12.0828 3.37432 13.2833C2.46149 14.4838 1.89154 15.9094 1.72524 17.4084C1.7132 17.5178 1.72284 17.6285 1.7536 17.7342C1.78435 17.8399 1.83563 17.9386 1.9045 18.0245C2.04359 18.1979 2.24589 18.309 2.46691 18.3334C2.68792 18.3577 2.90954 18.2932 3.08301 18.1541C3.25648 18.015 3.3676 17.8127 3.39191 17.5917C3.5749 15.9627 4.35165 14.4582 5.57376 13.3657C6.79587 12.2732 8.37766 11.6692 10.0169 11.6692C11.6562 11.6692 13.2379 12.2732 14.4601 13.3657C15.6822 14.4582 16.4589 15.9627 16.6419 17.5917C16.6646 17.7965 16.7623 17.9856 16.9162 18.1225C17.0701 18.2595 17.2692 18.3346 17.4752 18.3334H17.5669C17.7854 18.3082 17.985 18.1978 18.1224 18.0261C18.2597 17.8544 18.3237 17.6353 18.3002 17.4167C18.1332 15.9135 17.5601 14.4842 16.6426 13.2819C15.7251 12.0795 14.4977 11.1496 13.0919 10.5917ZM10.0002 10C9.34097 10 8.69651 9.80453 8.14834 9.43825C7.60018 9.07198 7.17294 8.55139 6.92064 7.9423C6.66835 7.33321 6.60234 6.66299 6.73096 6.01639C6.85957 5.36979 7.17704 4.77584 7.64322 4.30967C8.10939 3.84349 8.70334 3.52602 9.34994 3.39741C9.99654 3.26879 10.6668 3.3348 11.2759 3.58709C11.8849 3.83938 12.4055 4.26662 12.7718 4.81479C13.1381 5.36295 13.3336 6.00742 13.3336 6.66669C13.3336 7.55074 12.9824 8.39859 12.3573 9.02371C11.7321 9.64883 10.8843 10 10.0002 10Z"
     fill="#262626" />
-</svg>
+</svg>

frontend/src/assets/docs.svg

@@ -24,4 +24,4 @@
       <rect width="23.33" height="18.67" fill="white" transform="translate(2.33496 4.66504)" />
     </clipPath>
   </defs>
-</svg>
+</svg>

frontend/src/assets/external-link.svg

@@ -4,4 +4,4 @@
   <path
     d="M4.33333 3.66667C3.59695 3.66667 3 4.26362 3 5V11.6667C3 12.403 3.59695 13 4.33333 13H11C11.7364 13 12.3333 12.403 12.3333 11.6667V9.66667H11.3333V11.6667C11.3333 11.8508 11.1841 12 11 12H4.33333C4.14924 12 4 11.8508 4 11.6667V5C4 4.81591 4.14924 4.66667 4.33333 4.66667H6.33333V3.66667H4.33333Z"
     fill="#EEEEEE" />
-</svg>
+</svg>

frontend/src/assets/lightbulb.svg

@@ -2,4 +2,4 @@
   <path
     d="M11.3931 1.88021C10.78 1.37593 10.062 1.01489 9.29157 0.823438C8.52113 0.631981 7.71767 0.614934 6.9398 0.773542C5.90405 0.982756 4.95379 1.49492 4.20959 2.24506C3.46538 2.9952 2.96078 3.9495 2.7598 4.98688C2.61303 5.76469 2.6397 6.56531 2.83791 7.33163C3.03611 8.09795 3.40097 8.8111 3.90647 9.42021C4.37559 9.94959 4.64449 10.6266 4.66647 11.3335V13.3335C4.66647 13.864 4.87718 14.3727 5.25225 14.7478C5.62732 15.1228 6.13603 15.3335 6.66647 15.3335H9.33313C9.86356 15.3335 10.3723 15.1228 10.7473 14.7478C11.1224 14.3727 11.3331 13.864 11.3331 13.3335V11.4602C11.3555 10.6797 11.6423 9.92983 12.1465 9.33354C13.0299 8.24073 13.4463 6.84342 13.3052 5.44529C13.1642 4.04716 12.477 2.7612 11.3931 1.86687V1.88021ZM9.9998 13.3335C9.9998 13.5104 9.92956 13.6799 9.80454 13.8049C9.67951 13.93 9.50994 14.0002 9.33313 14.0002H6.66647C6.48965 14.0002 6.32009 13.93 6.19506 13.8049C6.07004 13.6799 5.9998 13.5104 5.9998 13.3335V12.6669H9.9998V13.3335ZM11.1131 8.50688C10.4428 9.30194 10.0517 10.2949 9.9998 11.3335H8.66647V9.33354C8.66647 9.15673 8.59623 8.98716 8.4712 8.86214C8.34618 8.73711 8.17661 8.66688 7.9998 8.66688C7.82299 8.66688 7.65342 8.73711 7.52839 8.86214C7.40337 8.98716 7.33313 9.15673 7.33313 9.33354V11.3335H5.9998C5.98221 10.3123 5.60443 9.33005 4.93313 8.56021C4.49023 8.02954 4.19239 7.39316 4.06867 6.71311C3.94495 6.03306 3.99957 5.33255 4.22719 4.6799C4.45481 4.02724 4.84768 3.4447 5.36748 2.98909C5.88728 2.53348 6.51627 2.22034 7.19313 2.08021C7.77483 1.96044 8.37591 1.9717 8.95271 2.11319C9.52952 2.25467 10.0676 2.52282 10.5278 2.89818C10.9881 3.27353 11.359 3.74666 11.6136 4.28322C11.8682 4.81978 12.0001 5.4063 11.9998 6.00021C12.0047 6.91345 11.6912 7.79985 11.1131 8.50688Z"
     fill="white" />
-</svg>
+</svg>

frontend/src/assets/loading-outer.svg

@@ -1,4 +1,4 @@
 <svg width="66" height="66" viewBox="0 0 66 66" fill="none" xmlns="http://www.w3.org/2000/svg">
   <path d="M63 33C63 16.4315 49.5685 3 33 3C16.4315 3 3 16.4315 3 33C3 49.5685 16.4315 63 33 63"
     stroke="#007AFF" stroke-width="6" stroke-linecap="round" />
-</svg>
+</svg>

frontend/src/assets/message.svg

@@ -2,4 +2,4 @@
   <path
     d="M19.8335 8.16634H8.16683C7.85741 8.16634 7.56066 8.28926 7.34187 8.50805C7.12308 8.72684 7.00016 9.02359 7.00016 9.33301C7.00016 9.64243 7.12308 9.93917 7.34187 10.158C7.56066 10.3768 7.85741 10.4997 8.16683 10.4997H19.8335C20.1429 10.4997 20.4397 10.3768 20.6585 10.158C20.8772 9.93917 21.0002 9.64243 21.0002 9.33301C21.0002 9.02359 20.8772 8.72684 20.6585 8.50805C20.4397 8.28926 20.1429 8.16634 19.8335 8.16634ZM19.8335 12.833H8.16683C7.85741 12.833 7.56066 12.9559 7.34187 13.1747C7.12308 13.3935 7.00016 13.6903 7.00016 13.9997C7.00016 14.3091 7.12308 14.6058 7.34187 14.8246C7.56066 15.0434 7.85741 15.1663 8.16683 15.1663H19.8335C20.1429 15.1663 20.4397 15.0434 20.6585 14.8246C20.8772 14.6058 21.0002 14.3091 21.0002 13.9997C21.0002 13.6903 20.8772 13.3935 20.6585 13.1747C20.4397 12.9559 20.1429 12.833 19.8335 12.833ZM22.1668 2.33301H5.8335C4.90524 2.33301 4.015 2.70176 3.35862 3.35813C2.70225 4.01451 2.3335 4.90475 2.3335 5.83301V17.4997C2.3335 18.4279 2.70225 19.3182 3.35862 19.9745C4.015 20.6309 4.90524 20.9997 5.8335 20.9997H19.3552L23.6718 25.328C23.7808 25.4361 23.9101 25.5217 24.0523 25.5797C24.1944 25.6378 24.3466 25.6672 24.5002 25.6663C24.6532 25.6703 24.805 25.6383 24.9435 25.573C25.1565 25.4855 25.3389 25.3369 25.4677 25.1458C25.5964 24.9548 25.6657 24.73 25.6668 24.4997V5.83301C25.6668 4.90475 25.2981 4.01451 24.6417 3.35813C23.9853 2.70176 23.0951 2.33301 22.1668 2.33301ZM23.3335 21.688L20.6618 19.0047C20.5528 18.8965 20.4235 18.811 20.2814 18.7529C20.1392 18.6949 19.987 18.6655 19.8335 18.6663H5.8335C5.52408 18.6663 5.22733 18.5434 5.00854 18.3246C4.78975 18.1058 4.66683 17.8091 4.66683 17.4997V5.83301C4.66683 5.52359 4.78975 5.22684 5.00854 5.00805C5.22733 4.78926 5.52408 4.66634 5.8335 4.66634H22.1668C22.4762 4.66634 22.773 4.78926 22.9918 5.00805C23.2106 5.22684 23.3335 5.52359 23.3335 5.83301V21.688Z"
     fill="white" />
-</svg>
+</svg>

frontend/src/assets/new-project.svg

@@ -3,4 +3,4 @@
   <path
     d="M20.4165 13.0837H14.9165V7.58366C14.9165 7.34054 14.8199 7.10739 14.648 6.93548C14.4761 6.76357 14.243 6.66699 13.9998 6.66699C13.7567 6.66699 13.5236 6.76357 13.3517 6.93548C13.1797 7.10739 13.0832 7.34054 13.0832 7.58366V13.0837H7.58317C7.34006 13.0837 7.1069 13.1802 6.93499 13.3521C6.76308 13.5241 6.6665 13.7572 6.6665 14.0003C6.6665 14.2434 6.76308 14.4766 6.93499 14.6485C7.1069 14.8204 7.34006 14.917 7.58317 14.917H13.0832V20.417C13.0832 20.6601 13.1797 20.8933 13.3517 21.0652C13.5236 21.2371 13.7567 21.3337 13.9998 21.3337C14.243 21.3337 14.4761 21.2371 14.648 21.0652C14.8199 20.8933 14.9165 20.6601 14.9165 20.417V14.917H20.4165C20.6596 14.917 20.8928 14.8204 21.0647 14.6485C21.2366 14.4766 21.3332 14.2434 21.3332 14.0003C21.3332 13.7572 21.2366 13.5241 21.0647 13.3521C20.8928 13.1802 20.6596 13.0837 20.4165 13.0837Z"
     fill="black" />
-</svg>
+</svg>

frontend/src/assets/refresh.svg

@@ -2,4 +2,4 @@
   <path
     d="M4.26446 0.763647C4.39463 0.633472 4.60569 0.633472 4.73586 0.763647L5.73586 1.76365C5.86604 1.89382 5.86604 2.10488 5.73586 2.23505L4.73586 3.23505C4.60569 3.36523 4.39463 3.36523 4.26446 3.23505C4.13429 3.10488 4.13429 2.89382 4.26446 2.76365L4.69542 2.33268H4.16683C2.98426 2.33268 2.00016 3.31678 2.00016 4.49935C2.00016 5.68192 2.98426 6.66602 4.16683 6.66602C5.3494 6.66602 6.3335 5.68192 6.3335 4.49935C6.3335 4.31525 6.48273 4.16602 6.66683 4.16602C6.85092 4.16602 7.00016 4.31525 7.00016 4.49935C7.00016 6.05011 5.71759 7.33268 4.16683 7.33268C2.61607 7.33268 1.3335 6.05011 1.3335 4.49935C1.3335 2.94859 2.61607 1.66602 4.16683 1.66602H4.69542L4.26446 1.23505C4.13429 1.10488 4.13429 0.893821 4.26446 0.763647Z"
     fill="white" />
-</svg>
+</svg>

frontend/src/components/AgentStatusBar.tsx

@@ -1,6 +1,7 @@
 import React, { useEffect } from "react";
 import { useTranslation } from "react-i18next";
 import { useSelector } from "react-redux";
+import toast from "react-hot-toast";
 import { I18nKey } from "#/i18n/declaration";
 import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
@@ -16,7 +17,7 @@ enum IndicatorColor {
 }
 
 function AgentStatusBar() {
-  const { t } = useTranslation();
+  const { t, i18n } = useTranslation();
   const { curAgentState } = useSelector((state: RootState) => state.agent);
   const { curStatusMessage } = useSelector((state: RootState) => state.status);
 
@@ -94,15 +95,27 @@ function AgentStatusBar() {
   const [statusMessage, setStatusMessage] = React.useState<string>("");
 
   React.useEffect(() => {
-    if (curAgentState === AgentState.LOADING) {
-      const trimmedCustomMessage = curStatusMessage.status.trim();
-      if (trimmedCustomMessage) {
-        setStatusMessage(t(trimmedCustomMessage));
-        return;
+    let message = curStatusMessage.message || "";
+    if (curStatusMessage?.id) {
+      const id = curStatusMessage.id.trim();
+      if (i18n.exists(id)) {
+        message = t(curStatusMessage.id.trim()) || message;
       }
     }
+    if (curStatusMessage?.type === "error") {
+      toast.error(message);
+      return;
+    }
+    if (curAgentState === AgentState.LOADING && message.trim()) {
+      setStatusMessage(message);
+    } else {
+      setStatusMessage(AgentStatusMap[curAgentState].message);
+    }
+  }, [curStatusMessage.id]);
+
+  React.useEffect(() => {
     setStatusMessage(AgentStatusMap[curAgentState].message);
-  }, [curAgentState, curStatusMessage.status]);
+  }, [curAgentState]);
 
   return (
     <div className="flex flex-col items-center">

frontend/src/components/Jupyter.tsx

@@ -25,7 +25,11 @@ function JupyterCell({ cell }: IJupyterCell): JSX.Element {
           className="scrollbar-custom scrollbar-thumb-gray-500 hover:scrollbar-thumb-gray-400 dark:scrollbar-thumb-white/10 dark:hover:scrollbar-thumb-white/20 overflow-auto px-5"
           style={{ padding: 0, marginBottom: 0, fontSize: "0.75rem" }}
         >
-          <SyntaxHighlighter language="python" style={atomOneDark}>
+          <SyntaxHighlighter
+            language="python"
+            style={atomOneDark}
+            wrapLongLines
+          >
             {code}
           </SyntaxHighlighter>
         </pre>
@@ -78,7 +82,11 @@ function JupyterCell({ cell }: IJupyterCell): JSX.Element {
   );
 }
 
-function JupyterEditor(): JSX.Element {
+interface JupyterEditorProps {
+  maxWidth: number;
+}
+
+function JupyterEditor({ maxWidth }: JupyterEditorProps) {
   const { t } = useTranslation();
 
   const { cells } = useSelector((state: RootState) => state.jupyter);
@@ -88,7 +96,7 @@ function JupyterEditor(): JSX.Element {
     useScrollToBottom(jupyterRef);
 
   return (
-    <div className="flex-1">
+    <div className="flex-1" style={{ maxWidth }}>
       <div
         className="overflow-y-auto h-full"
         ref={jupyterRef}

frontend/src/components/analytics-consent-form-modal.tsx

@@ -0,0 +1,42 @@
+import { useFetcher } from "@remix-run/react";
+import { ModalBackdrop } from "./modals/modal-backdrop";
+import ModalBody from "./modals/ModalBody";
+import ModalButton from "./buttons/ModalButton";
+import {
+  BaseModalTitle,
+  BaseModalDescription,
+} from "./modals/confirmation-modals/BaseModal";
+
+export function AnalyticsConsentFormModal() {
+  const fetcher = useFetcher({ key: "set-consent" });
+
+  return (
+    <ModalBackdrop>
+      <fetcher.Form
+        method="POST"
+        action="/set-consent"
+        className="flex flex-col gap-2"
+      >
+        <ModalBody>
+          <BaseModalTitle title="Your Privacy Preferences" />
+          <BaseModalDescription>
+            We use tools to understand how our application is used to improve
+            your experience. You can enable or disable analytics. Your
+            preferences will be stored and can be updated anytime.
+          </BaseModalDescription>
+
+          <label className="flex gap-2 items-center self-start">
+            <input name="analytics" type="checkbox" defaultChecked />
+            Send anonymous usage data
+          </label>
+
+          <ModalButton
+            type="submit"
+            text="Confirm Preferences"
+            className="bg-primary text-white w-full hover:opacity-80"
+          />
+        </ModalBody>
+      </fetcher.Form>
+    </ModalBackdrop>
+  );
+}

frontend/src/components/buttons/ModalButton.tsx

@@ -2,6 +2,7 @@ import clsx from "clsx";
 import React from "react";
 
 interface ModalButtonProps {
+  testId?: string;
   variant?: "default" | "text-like";
   onClick?: () => void;
   text: string;
@@ -13,6 +14,7 @@ interface ModalButtonProps {
 }
 
 function ModalButton({
+  testId,
   variant = "default",
   onClick,
   text,
@@ -24,6 +26,7 @@ function ModalButton({
 }: ModalButtonProps) {
   return (
     <button
+      data-testid={testId}
       type={type === "submit" ? "submit" : "button"}
       disabled={disabled}
       onClick={onClick}

frontend/src/components/chat-input.tsx

@@ -16,6 +16,7 @@ interface ChatInputProps {
   onChange?: (message: string) => void;
   onFocus?: () => void;
   onBlur?: () => void;
+  onImagePaste?: (files: File[]) => void;
   className?: React.HTMLAttributes<HTMLDivElement>["className"];
 }
 
@@ -32,9 +33,51 @@ export function ChatInput({
   onChange,
   onFocus,
   onBlur,
+  onImagePaste,
   className,
 }: ChatInputProps) {
   const textareaRef = React.useRef<HTMLTextAreaElement>(null);
+  const [isDraggingOver, setIsDraggingOver] = React.useState(false);
+
+  const handlePaste = (event: React.ClipboardEvent<HTMLTextAreaElement>) => {
+    // Only handle paste if we have an image paste handler and there are files
+    if (onImagePaste && event.clipboardData.files.length > 0) {
+      const files = Array.from(event.clipboardData.files).filter((file) =>
+        file.type.startsWith("image/"),
+      );
+      // Only prevent default if we found image files to handle
+      if (files.length > 0) {
+        event.preventDefault();
+        onImagePaste(files);
+      }
+    }
+    // For text paste, let the default behavior handle it
+  };
+
+  const handleDragOver = (event: React.DragEvent<HTMLTextAreaElement>) => {
+    event.preventDefault();
+    if (event.dataTransfer.types.includes("Files")) {
+      setIsDraggingOver(true);
+    }
+  };
+
+  const handleDragLeave = (event: React.DragEvent<HTMLTextAreaElement>) => {
+    event.preventDefault();
+    setIsDraggingOver(false);
+  };
+
+  const handleDrop = (event: React.DragEvent<HTMLTextAreaElement>) => {
+    event.preventDefault();
+    setIsDraggingOver(false);
+    if (onImagePaste && event.dataTransfer.files.length > 0) {
+      const files = Array.from(event.dataTransfer.files).filter((file) =>
+        file.type.startsWith("image/"),
+      );
+      if (files.length > 0) {
+        onImagePaste(files);
+      }
+    }
+  };
 
   const handleSubmitMessage = () => {
     if (textareaRef.current?.value) {
@@ -67,12 +110,20 @@ export function ChatInput({
         onChange={handleChange}
         onFocus={onFocus}
         onBlur={onBlur}
+        onPaste={handlePaste}
+        onDrop={handleDrop}
+        onDragOver={handleDragOver}
+        onDragLeave={handleDragLeave}
         value={value}
         minRows={1}
         maxRows={maxRows}
+        data-dragging-over={isDraggingOver}
         className={cn(
-          "grow text-sm self-center placeholder:text-neutral-400 text-white resize-none bg-transparent outline-none ring-0",
-          "transition-[height] duration-200 ease-in-out",
+          "grow text-sm self-center placeholder:text-neutral-400 text-white resize-none outline-none ring-0",
+          "transition-all duration-200 ease-in-out",
+          isDraggingOver
+            ? "bg-neutral-600/50 rounded-lg px-2"
+            : "bg-transparent",
           className,
         )}
       />

frontend/src/components/chat-interface.tsx

@@ -1,6 +1,6 @@
 import { useDispatch, useSelector } from "react-redux";
 import React from "react";
-import { useFetcher } from "@remix-run/react";
+import posthog from "posthog-js";
 import { useSocket } from "#/context/socket";
 import { convertImageToBase64 } from "#/utils/convert-image-to-base-64";
 import { ChatMessage } from "./chat-message";
@@ -13,27 +13,23 @@ import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import { generateAgentStateChangeEvent } from "#/services/agentStateService";
 import { FeedbackModal } from "./feedback-modal";
-import { Feedback } from "#/api/open-hands.types";
-import { getToken } from "#/services/auth";
-import { removeApiKey, removeUnwantedKeys } from "#/utils/utils";
-import { clientAction } from "#/routes/submit-feedback";
 import { useScrollToBottom } from "#/hooks/useScrollToBottom";
 import TypingIndicator from "./chat/TypingIndicator";
 import ConfirmationButtons from "./chat/ConfirmationButtons";
 import { ErrorMessage } from "./error-message";
 import { ContinueButton } from "./continue-button";
 import { ScrollToBottomButton } from "./scroll-to-bottom-button";
-
-const FEEDBACK_VERSION = "1.0";
+import { Suggestions } from "./suggestions";
+import { SUGGESTIONS } from "#/utils/suggestions";
+import BuildIt from "#/assets/build-it.svg?react";
 
 const isErrorMessage = (
   message: Message | ErrorMessage,
 ): message is ErrorMessage => "error" in message;
 
 export function ChatInterface() {
-  const { send, events } = useSocket();
+  const { send } = useSocket();
   const dispatch = useDispatch();
-  const fetcher = useFetcher<typeof clientAction>({ key: "feedback" });
   const scrollRef = React.useRef<HTMLDivElement>(null);
   const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
     useScrollToBottom(scrollRef);
@@ -44,19 +40,24 @@ export function ChatInterface() {
   const [feedbackPolarity, setFeedbackPolarity] = React.useState<
     "positive" | "negative"
   >("positive");
-  const [feedbackShared, setFeedbackShared] = React.useState(0);
   const [feedbackModalIsOpen, setFeedbackModalIsOpen] = React.useState(false);
+  const [messageToSend, setMessageToSend] = React.useState<string | null>(null);
 
   const handleSendMessage = async (content: string, files: File[]) => {
+    posthog.capture("user_message_sent", {
+      current_message_count: messages.length,
+    });
     const promises = files.map((file) => convertImageToBase64(file));
     const imageUrls = await Promise.all(promises);
 
     const timestamp = new Date().toISOString();
     dispatch(addUserMessage({ content, imageUrls, timestamp }));
     send(createChatMessage(content, imageUrls, timestamp));
+    setMessageToSend(null);
   };
 
   const handleStop = () => {
+    posthog.capture("stop_button_clicked");
     send(generateAgentStateChangeEvent(AgentState.STOPPED));
   };
 
@@ -71,32 +72,30 @@ export function ChatInterface() {
     setFeedbackPolarity(polarity);
   };
 
-  const handleSubmitFeedback = (
-    permissions: "private" | "public",
-    email: string,
-  ) => {
-    const feedback: Feedback = {
-      version: FEEDBACK_VERSION,
-      feedback: feedbackPolarity,
-      email,
-      permissions,
-      token: getToken(),
-      trajectory: removeApiKey(removeUnwantedKeys(events)),
-    };
-
-    const formData = new FormData();
-    formData.append("feedback", JSON.stringify(feedback));
-
-    fetcher.submit(formData, {
-      action: "/submit-feedback",
-      method: "POST",
-    });
-
-    setFeedbackShared(messages.length);
-  };
-
   return (
     <div className="h-full flex flex-col justify-between">
+      {messages.length === 0 && (
+        <div className="flex flex-col gap-6 h-full px-4 items-center justify-center">
+          <div className="flex flex-col items-center p-4 bg-neutral-700 rounded-xl w-full">
+            <BuildIt width={45} height={54} />
+            <span className="font-semibold text-[20px] leading-6 -tracking-[0.01em] gap-1">
+              Let&apos;s start building!
+            </span>
+          </div>
+          <Suggestions
+            suggestions={Object.entries(SUGGESTIONS.repo)
+              .slice(0, 4)
+              .map(([label, value]) => ({
+                label,
+                value,
+              }))}
+            onSuggestionClick={(value) => {
+              setMessageToSend(value);
+            }}
+          />
+        </div>
+      )}
+
       <div
         ref={scrollRef}
         onScroll={(e) => onChatBodyScroll(e.currentTarget)}
@@ -106,7 +105,7 @@ export function ChatInterface() {
           isErrorMessage(message) ? (
             <ErrorMessage
               key={index}
-              error={message.error}
+              id={message.id}
               message={message.message}
             />
           ) : (
@@ -130,16 +129,14 @@ export function ChatInterface() {
 
       <div className="flex flex-col gap-[6px] px-4 pb-4">
         <div className="flex justify-between relative">
-          {feedbackShared !== messages.length && messages.length > 3 && (
-            <FeedbackActions
-              onPositiveFeedback={() =>
-                onClickShareFeedbackActionButton("positive")
-              }
-              onNegativeFeedback={() =>
-                onClickShareFeedbackActionButton("negative")
-              }
-            />
-          )}
+          <FeedbackActions
+            onPositiveFeedback={() =>
+              onClickShareFeedbackActionButton("positive")
+            }
+            onNegativeFeedback={() =>
+              onClickShareFeedbackActionButton("negative")
+            }
+          />
           <div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
             {messages.length > 2 &&
               curAgentState === AgentState.AWAITING_USER_INPUT && (
@@ -158,14 +155,15 @@ export function ChatInterface() {
             curAgentState === AgentState.AWAITING_USER_CONFIRMATION
           }
           mode={curAgentState === AgentState.RUNNING ? "stop" : "submit"}
+          value={messageToSend ?? undefined}
+          onChange={setMessageToSend}
         />
       </div>
 
       <FeedbackModal
         isOpen={feedbackModalIsOpen}
-        isSubmitting={fetcher.state === "submitting"}
         onClose={() => setFeedbackModalIsOpen(false)}
-        onSubmit={handleSubmitFeedback}
+        polarity={feedbackPolarity}
       />
     </div>
   );