Fix issue #4629: '[Bug]: Replace claude-3-5-sonnet-20240620 with claude-3-5-sonnet-20241022'

"b/frontend/__tests__/utils/extractModelAndProvider.test.ts\""

@@ -0,0 +1,5 @@
+expect(extractModelAndProvider("claude-3-5-sonnet-20241022")).toEqual({
+      provider: "anthropic",
+      model: "claude-3-5-sonnet-20241022",
+      separator: "/",
+    });

"b/frontend/__tests__/utils/organizeModelsAndProviders.test.ts\""

@@ -0,0 +1,65 @@
+import { expect, test } from "vitest";
+import { organizeModelsAndProviders } from "../../src/utils/organizeModelsAndProviders";
+
+test("organizeModelsAndProviders", () => {
+  const models = [
+    "azure/ada",
+    "azure/gpt-35-turbo",
+    "azure/gpt-3-turbo",
+    "azure/standard/1024-x-1024/dall-e-2",
+    "vertex_ai_beta/chat-bison",
+    "vertex_ai_beta/chat-bison-32k",
+    "sagemaker/meta-textgeneration-llama-2-13b",
+    "cohere.command-r-v1:0",
+    "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1",
+    "gpt-4o",
+    "together-ai-21.1b-41b",
+    "gpt-4o-mini",
+    "claude-3-5-sonnet-20241022",
+    "claude-3-haiku-20240307",
+    "claude-2",
+    "claude-2.1",
+    "anthropic.unsafe-claude-2.1",
+  ];
+
+  const object = organizeModelsAndProviders(models);
+
+  expect(object).toEqual({
+    azure: {
+      separator: "/",
+      models: [
+        "ada",
+        "gpt-35-turbo",
+        "gpt-3-turbo",
+        "standard/1024-x-1024/dall-e-2",
+      ],
+    },
+    vertex_ai_beta: {
+      separator: "/",
+      models: ["chat-bison", "chat-bison-32k"],
+    },
+    sagemaker: { separator: "/", models: ["meta-textgeneration-llama-2-13b"] },
+    cohere: { separator: ".", models: ["command-r-v1:0"] },
+    cloudflare: {
+      separator: "/",
+      models: ["@cf/mistral/mistral-7b-instruct-v0.1"],
+    },
+    openai: {
+      separator: "/",
+      models: ["gpt-4o", "gpt-4o-mini"],
+    },
+    anthropic: {
+      separator: "/",
+      models: [
+        "claude-3-5-sonnet-20241022",
+        "claude-3-haiku-20240307",
+        "claude-2",
+        "claude-2.1",
+      ],
+    },
+    other: {
+      separator: "",
+      models: ["together-ai-21.1b-41b"],
+    },
+  });
+});

"b/frontend/src/utils/verified-models.ts\""

@@ -0,0 +1,29 @@
+// Here are the list of verified models and providers that we know work well with OpenHands.
+export const VERIFIED_PROVIDERS = ["openai", "azure", "anthropic"];
+export const VERIFIED_MODELS = ["gpt-4o", "claude-3-5-sonnet-20241022"];
+
+// LiteLLM does not return OpenAI models with the provider, so we list them here to set them ourselves for consistency
+// (e.g., they return `gpt-4o` instead of `openai/gpt-4o`)
+export const VERIFIED_OPENAI_MODELS = [
+  "gpt-4o",
+  "gpt-4o-mini",
+  "gpt-4-turbo",
+  "gpt-4",
+  "gpt-4-32k",
+  "o1-mini",
+  "o1-preview",
+];
+
+// LiteLLM does not return the compatible Anthropic models with the provider, so we list them here to set them ourselves
+// (e.g., they return `claude-3-5-sonnet-20241022` instead of `anthropic/claude-3-5-sonnet-20241022`)
+export const VERIFIED_ANTHROPIC_MODELS = [
+  "claude-2",
+  "claude-2.1",
+  "claude-3-5-sonnet-20241022",
+  "claude-3-5-sonnet-20240620",
+  "claude-3-haiku-20240307",
+  "claude-3-opus-20240229",
+  "claude-3-sonnet-20240229",
+  "claude-instant-1",
+  "claude-instant-1.2",
+];

.github/workflows/ghcr-build.yml

@@ -1,5 +1,5 @@
 # Workflow that builds, tests and then pushes the OpenHands and runtime docker images to the ghcr.io repository
-name: Docker
+name: Build, Test and Publish RT Image
 
 # Always run on "main"
 # Always run on tags
@@ -399,49 +399,3 @@ jobs:
         run: |
           echo "Some runtime tests failed or were cancelled"
           exit 1
-  update_pr_description:
-    name: Update PR Description
-    if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'
-    needs: [ghcr_build_runtime]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Get short SHA
-        id: short_sha
-        run: echo "SHORT_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
-
-      - name: Update PR Description
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REPO: ${{ github.repository }}
-          SHORT_SHA: ${{ steps.short_sha.outputs.SHORT_SHA }}
-        run: |
-          echo "updating PR description"
-          DOCKER_RUN_COMMAND="docker run -it --rm \
-            -p 3000:3000 \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            --add-host host.docker.internal:host-gateway \
-            -e SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:$SHORT_SHA-nikolaik \
-            --name openhands-app-$SHORT_SHA \
-            ghcr.io/all-hands-ai/runtime:$SHORT_SHA"
-
-          PR_BODY=$(gh pr view $PR_NUMBER --json body --jq .body)
-
-          if echo "$PR_BODY" | grep -q "To run this PR locally, use the following command:"; then
-            UPDATED_PR_BODY=$(echo "${PR_BODY}" | sed -E "s|docker run -it --rm.*|$DOCKER_RUN_COMMAND|")
-          else
-            UPDATED_PR_BODY="${PR_BODY}
-
-          ---
-
-          To run this PR locally, use the following command:
-          \`\`\`
-          $DOCKER_RUN_COMMAND
-          \`\`\`"
-          fi
-
-          echo "updated body: $UPDATED_PR_BODY"
-          gh pr edit $PR_NUMBER --body "$UPDATED_PR_BODY"

.github/workflows/openhands-resolver.yml

@@ -3,8 +3,6 @@ name: Resolve Issues with OpenHands
 on:
   issues:
     types: [labeled]
-  pull_request:
-    types: [labeled]
 
 jobs:
   call-openhands-resolver:

.gitignore

@@ -174,7 +174,6 @@ evaluation/bird/data
 evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
-evaluation/scienceagentbench/benchmark
 
 # frontend
 

README.md

@@ -12,7 +12,7 @@
   <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
   <br/>
-  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
+  <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
   <br/>
@@ -33,34 +33,37 @@ Learn more at [docs.all-hands.dev](https://docs.all-hands.dev), or jump to the [
 
 ## ⚡ Quick Start
 
-The easiest way to run OpenHands is in Docker.
+The easiest way to run OpenHands is in Docker. You can change `WORKSPACE_BASE` below to
+point OpenHands to existing code that you'd like to modify.
+
 See the [Installation](https://docs.all-hands.dev/modules/usage/installation) guide for
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik
+export WORKSPACE_BASE=$(pwd)/workspace
 
-docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
+docker pull ghcr.io/all-hands-ai/runtime:0.11-nikolaik
+
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_USER_ID=$(id -u) \
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
-    --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.12
+    --name openhands-app-$(date +%Y%m%d%H%M%S) \
+    ghcr.io/all-hands-ai/openhands:0.11
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
 
-Finally, you'll need a model provider and API key.
-[Anthropic's Claude 3.5 Sonnet](https://www.anthropic.com/api) (`anthropic/claude-3-5-sonnet-20241022`)
-works best, but you have [many options](https://docs.all-hands.dev/modules/usage/llms).
+You'll need a model provider and API key. One option that works well: [Claude 3.5 Sonnet](https://www.anthropic.com/api), but you have [many options](https://docs.all-hands.dev/modules/usage/llms).
 
 ---
 
-You can also [connect OpenHands to your local filesystem](https://docs.all-hands.dev/modules/usage/runtimes),
-run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
-interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode),
-or run it on tagged issues with [a github action](https://github.com/All-Hands-AI/OpenHands-resolver).
+You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
+or as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode).
 
 Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
@@ -93,7 +96,7 @@ For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
 Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
 Let's make software engineering better together!
 
-- [Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
+- [Slack workspace](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) - Here we talk about research, architecture, and future development.
 - [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
 
 ## 📈 Progress

containers/app/Dockerfile

@@ -41,7 +41,6 @@ ENV SANDBOX_LOCAL_RUNTIME_URL=http://host.docker.internal
 ENV USE_HOST_NETWORK=false
 ENV WORKSPACE_BASE=/opt/workspace_base
 ENV OPENHANDS_BUILD_VERSION=$OPENHANDS_BUILD_VERSION
-ENV SANDBOX_USER_ID=0
 RUN mkdir -p $WORKSPACE_BASE
 
 RUN apt-get update -y \

containers/app/entrypoint.sh

@@ -18,11 +18,6 @@ if [ -z "$SANDBOX_USER_ID" ]; then
   exit 1
 fi
 
-if [ -z "$WORKSPACE_MOUNT_PATH" ]; then
-  # This is set to /opt/workspace in the Dockerfile. But if the user isn't mounting, we want to unset it so that OpenHands doesn't mount at all
-  unset WORKSPACE_BASE
-fi
-
 if [[ "$SANDBOX_USER_ID" -eq 0 ]]; then
   echo "Running OpenHands as root"
   export RUN_AS_OPENHANDS=false

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/cli-mode.md

@@ -14,97 +14,4 @@ Pour démarrer une session OpenHands interactive via la ligne de commande, suive
 
 2. Exécutez la commande suivante :
 
-```bash
-poetry run python -m openhands.core.cli
-```
 
-Cette commande démarrera une session interactive où vous pourrez saisir des tâches et recevoir des réponses d'OpenHands.
-
-Vous devrez vous assurer de définir votre modèle, votre clé API et d'autres paramètres via des variables d'environnement
-[ou le fichier `config.toml`](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml).
-
-
-## Avec Docker
-
-Pour exécuter OpenHands en mode CLI avec Docker, suivez ces étapes :
-
-1. Définissez `WORKSPACE_BASE` sur le répertoire que vous souhaitez qu'OpenHands modifie :
-
-```bash
-WORKSPACE_BASE=$(pwd)/workspace
-```
-
-2. Définissez `LLM_MODEL` sur le modèle que vous souhaitez utiliser :
-
-```bash
-LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
-```
-
-3. Définissez `LLM_API_KEY` sur votre clé API :
-
-```bash
-LLM_API_KEY="sk_test_12345"
-```
-
-4. Exécutez la commande Docker suivante :
-
-```bash
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_API_KEY=$LLM_API_KEY \
-    -e LLM_MODEL=$LLM_MODEL \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/all-hands-ai/openhands:0.11 \
-    python -m openhands.core.cli
-```
-
-Cette commande démarrera une session interactive dans Docker où vous pourrez saisir des tâches et recevoir des réponses d'OpenHands.
-
-## Exemples de commandes CLI et de sorties attendues
-
-Voici quelques exemples de commandes CLI et leurs sorties attendues :
-
-### Exemple 1 : Tâche simple
-
-```bash
-Comment puis-je vous aider ? >> Écrivez un script Python qui affiche "Hello, World!"
-```
-
-Sortie attendue :
-
-```bash
-🤖 Bien sûr ! Voici un script Python qui affiche "Hello, World!" :
-
-❯ print("Hello, World!")
-```
-
-### Exemple 2 : Commande Bash
-
-```bash
-Comment puis-je vous aider ? >> Créez un répertoire nommé "test_dir"
-```
-
-Sortie attendue :
-
-```bash
-🤖 Création d'un répertoire nommé "test_dir" :
-
-❯ mkdir test_dir
-```
-
-### Exemple 3 : Gestion des erreurs
-
-```bash
-Comment puis-je vous aider ? >> Supprimez un fichier inexistant
-```
-
-Sortie attendue :
-
-```bash
-🤖 Une erreur s'est produite. Veuillez réessayer.
-```

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -1,5 +1,3 @@
-
-
 # Évaluation
 
 Ce guide fournit un aperçu de la façon d'intégrer votre propre benchmark d'évaluation dans le framework OpenHands.
@@ -11,272 +9,4 @@ OpenHands en mode développement utilise `config.toml` pour garder une trace de
 
 Voici un exemple de fichier de configuration que vous pouvez utiliser pour définir et utiliser plusieurs LLMs :
 
-```toml
-[llm]
-# IMPORTANT : ajoutez votre clé API ici et définissez le modèle que vous souhaitez évaluer
-model = "claude-3-5-sonnet-20241022"
-
-api_key = "sk-XXX"
-
-[llm.eval_gpt4_1106_preview_llm]
-model = "gpt-4-1106-preview"
-api_key = "XXX"
-temperature = 0.0
-
-[llm.eval_some_openai_compatible_model_llm]
-model = "openai/MODEL_NAME"
-base_url = "https://OPENAI_COMPATIBLE_URL/v1"
-api_key = "XXX"
-temperature = 0.0
-```
-
-
-## Comment utiliser OpenHands en ligne de commande
-
-OpenHands peut être exécuté depuis la ligne de commande en utilisant le format suivant :
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i <max_iterations> \
-        -t "<task_description>" \
-        -c <agent_class> \
-        -l <llm_config>
-```
-
-Par exemple :
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i 10 \
-        -t "Écrivez-moi un script bash qui affiche hello world." \
-        -c CodeActAgent \
-        -l llm
-```
-
-Cette commande exécute OpenHands avec :
-- Un maximum de 10 itérations
-- La description de tâche spécifiée
-- En utilisant CodeActAgent
-- Avec la configuration LLM définie dans la section `llm` de votre fichier `config.toml`
-
-## Comment fonctionne OpenHands
-
-Le point d'entrée principal d'OpenHands se trouve dans `openhands/core/main.py`. Voici un flux simplifié de son fonctionnement :
-
-1. Analyse des arguments de ligne de commande et chargement de la configuration
-2. Création d'un environnement d'exécution à l'aide de `create_runtime()`
-3. Initialisation de l'agent spécifié
-4. Exécution du contrôleur à l'aide de `run_controller()`, qui :
-   - Attache l'environnement d'exécution à l'agent
-   - Exécute la tâche de l'agent
-   - Renvoie un état final une fois terminé
-
-La fonction `run_controller()` est le cœur de l'exécution d'OpenHands. Elle gère l'interaction entre l'agent, l'environnement d'exécution et la tâche, en gérant des choses comme la simulation d'entrée utilisateur et le traitement des événements.
-
-
-## Le moyen le plus simple de commencer : Explorer les benchmarks existants
-
-Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) de notre dépôt.
-
-Pour intégrer votre propre benchmark, nous vous suggérons de commencer par celui qui ressemble le plus à vos besoins. Cette approche peut considérablement rationaliser votre processus d'intégration, vous permettant de vous appuyer sur les structures existantes et de les adapter à vos exigences spécifiques.
-
-## Comment créer un workflow d'évaluation
-
-
-Pour créer un workflow d'évaluation pour votre benchmark, suivez ces étapes :
-
-1. Importez les utilitaires OpenHands pertinents :
-   ```python
-    import openhands.agenthub
-    from evaluation.utils.shared import (
-        EvalMetadata,
-        EvalOutput,
-        make_metadata,
-        prepare_dataset,
-        reset_logger_for_multiprocessing,
-        run_evaluation,
-    )
-    from openhands.controller.state.state import State
-    from openhands.core.config import (
-        AppConfig,
-        SandboxConfig,
-        get_llm_config_arg,
-        parse_arguments,
-    )
-    from openhands.core.logger import openhands_logger as logger
-    from openhands.core.main import create_runtime, run_controller
-    from openhands.events.action import CmdRunAction
-    from openhands.events.observation import CmdOutputObservation, ErrorObservation
-    from openhands.runtime.runtime import Runtime
-   ```
-
-2. Créez une configuration :
-   ```python
-   def get_config(instance: pd.Series, metadata: EvalMetadata) -> AppConfig:
-       config = AppConfig(
-           default_agent=metadata.agent_class,
-           runtime='eventstream',
-           max_iterations=metadata.max_iterations,
-           sandbox=SandboxConfig(
-               base_container_image='your_container_image',
-               enable_auto_lint=True,
-               timeout=300,
-           ),
-       )
-       config.set_llm_config(metadata.llm_config)
-       return config
-   ```
-
-3. Initialisez l'environnement d'exécution et configurez l'environnement d'évaluation :
-   ```python
-   def initialize_runtime(runtime: Runtime, instance: pd.Series):
-       # Configurez votre environnement d'évaluation ici
-       # Par exemple, définir des variables d'environnement, préparer des fichiers, etc.
-       pass
-   ```
-
-4. Créez une fonction pour traiter chaque instance :
-   ```python
-   from openhands.utils.async_utils import call_async_from_sync
-   def process_instance(instance: pd.Series, metadata: EvalMetadata) -> EvalOutput:
-       config = get_config(instance, metadata)
-       runtime = create_runtime(config)
-       call_async_from_sync(runtime.connect)
-       initialize_runtime(runtime, instance)
-
-       instruction = get_instruction(instance, metadata)
-
-       state = run_controller(
-           config=config,
-           task_str=instruction,
-           runtime=runtime,
-           fake_user_response_fn=your_user_response_function,
-       )
-
-       # Évaluez les actions de l'agent
-       evaluation_result = await evaluate_agent_actions(runtime, instance)
-
-       return EvalOutput(
-           instance_id=instance.instance_id,
-           instruction=instruction,
-           test_result=evaluation_result,
-           metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
-           metrics=state.metrics.get() if state.metrics else None,
-           error=state.last_error if state and state.last_error else None,
-       )
-   ```
-
-5. Exécutez l'évaluation :
-   ```python
-   metadata = make_metadata(llm_config, dataset_name, agent_class, max_iterations, eval_note, eval_output_dir)
-   output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-   instances = prepare_dataset(your_dataset, output_file, eval_n_limit)
-
-   await run_evaluation(
-       instances,
-       metadata,
-       output_file,
-       num_workers,
-       process_instance
-   )
-   ```
-
-Ce workflow configure la configuration, initialise l'environnement d'exécution, traite chaque instance en exécutant l'agent et en évaluant ses actions, puis collecte les résultats dans un objet `EvalOutput`. La fonction `run_evaluation` gère la parallélisation et le suivi de la progression.
-
-N'oubliez pas de personnaliser les fonctions `get_instruction`, `your_user_response_function` et `evaluate_agent_actions` en fonction des exigences spécifiques de votre benchmark.
-
-En suivant cette structure, vous pouvez créer un workflow d'évaluation robuste pour votre benchmark dans le framework OpenHands.
-
-
-## Comprendre la `user_response_fn`
-
-La `user_response_fn` est un composant crucial dans le workflow d'évaluation d'OpenHands. Elle simule l'interaction de l'utilisateur avec l'agent, permettant des réponses automatisées pendant le processus d'évaluation. Cette fonction est particulièrement utile lorsque vous souhaitez fournir des réponses cohérentes et prédéfinies aux requêtes ou actions de l'agent.
-
-
-### Workflow et interaction
-
-Le workflow correct pour gérer les actions et la `user_response_fn` est le suivant :
-
-1. L'agent reçoit une tâche et commence à la traiter
-2. L'agent émet une Action
-3. Si l'Action est exécutable (par exemple, CmdRunAction, IPythonRunCellAction) :
-   - Le Runtime traite l'Action
-   - Le Runtime renvoie une Observation
-4. Si l'Action n'est pas exécutable (généralement une MessageAction) :
-   - La `user_response_fn` est appelée
-   - Elle renvoie une réponse utilisateur simulée
-5. L'agent reçoit soit l'Observation, soit la réponse simulée
-6. Les étapes 2 à 5 se répètent jusqu'à ce que la tâche soit terminée ou que le nombre maximum d'itérations soit atteint
-
-Voici une représentation visuelle plus précise :
-
-```
-                 [Agent]
-                    |
-                    v
-               [Émettre une Action]
-                    |
-                    v
-            [L'Action est-elle exécutable ?]
-           /                       \
-         Oui                        Non
-          |                          |
-          v                          v
-     [Runtime]               [user_response_fn]
-          |                          |
-          v                          v
-  [Renvoyer une Observation]    [Réponse simulée]
-           \                        /
-            \                      /
-             v                    v
-           [L'agent reçoit le feedback]
-                    |
-                    v
-         [Continuer ou terminer la tâche]
-```
-
-Dans ce workflow :
-
-- Les actions exécutables (comme l'exécution de commandes ou de code) sont gérées directement par le Runtime
-- Les actions non exécutables (généralement lorsque l'agent veut communiquer ou demander des clarifications) sont gérées par la `user_response_fn`
-- L'agent traite ensuite le feedback, qu'il s'agisse d'une Observation du Runtime ou d'une réponse simulée de la `user_response_fn`
-
-Cette approche permet une gestion automatisée des actions concrètes et des interactions utilisateur simulées, ce qui la rend adaptée aux scénarios d'évaluation où vous souhaitez tester la capacité de l'agent à effectuer des tâches avec une intervention humaine minimale.
-
-### Exemple d'implémentation
-
-Voici un exemple de `user_response_fn` utilisée dans l'évaluation SWE-Bench :
-
-```python
-def codeact_user_response(state: State | None) -> str:
-    msg = (
-        'Veuillez continuer à travailler sur la tâche avec l\'approche que vous jugez appropriée.\n'
-        'Si vous pensez avoir résolu la tâche, veuillez d\'abord envoyer votre réponse à l\'utilisateur via un message, puis <execute_bash> exit </execute_bash>.\n'
-        'IMPORTANT : VOUS NE DEVEZ JAMAIS DEMANDER DE L\'AIDE HUMAINE.\n'
-    )
-
-    if state and state.history:
-        # vérifier si l'agent a essayé de parler à l'utilisateur 3 fois, si oui, faire savoir à l'agent qu'il peut abandonner
-        user_msgs = [
-            event
-            for event in state.history.get_events()
-            if isinstance(event, MessageAction) and event.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # faire savoir à l'agent qu'il peut abandonner lorsqu'il a essayé 3 fois
-            return (
-                msg
-                + 'Si vous voulez abandonner, exécutez : <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-```
-
-Cette fonction fait ce qui suit :
-
-1. Fournit un message standard encourageant l'agent à continuer à travailler
-2. Vérifie combien de fois l'agent a tenté de communiquer avec l'utilisateur
-3. Si l'agent a fait plusieurs tentatives, il lui donne la possibilité d'abandonner
-
-En utilisant cette fonction, vous pouvez garantir un comportement cohérent sur plusieurs exécutions d'évaluation et empêcher l'agent de rester bloqué en attendant une entrée humaine.
 

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -1,5 +1,3 @@
-
-
 # Mode sans interface
 
 Vous pouvez exécuter OpenHands avec une seule commande, sans démarrer l'application web.
@@ -13,46 +11,4 @@ Pour exécuter OpenHands en mode sans interface avec Python,
 [suivez les instructions de configuration de développement](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
 puis exécutez :
 
-```bash
-poetry run python -m openhands.core.main -t "write a bash script that prints hi"
-```
 
-Vous devrez vous assurer de définir votre modèle, votre clé API et d'autres paramètres via des variables d'environnement
-[ou le fichier `config.toml`](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml).
-
-## Avec Docker
-
-1. Définissez `WORKSPACE_BASE` sur le répertoire que vous voulez qu'OpenHands modifie :
-
-```bash
-WORKSPACE_BASE=$(pwd)/workspace
-```
-
-2. Définissez `LLM_MODEL` sur le modèle que vous voulez utiliser :
-
-```bash
-LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
-```
-
-3. Définissez `LLM_API_KEY` sur votre clé API :
-
-```bash
-LLM_API_KEY="sk_test_12345"
-```
-
-4. Exécutez la commande Docker suivante :
-
-```bash
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_API_KEY=$LLM_API_KEY \
-    -e LLM_MODEL=$LLM_MODEL \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/all-hands-ai/openhands:0.11 \
-    python -m openhands.core.main -t "write a bash script that prints hi"
-```

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/cli-mode.md

@@ -14,96 +14,4 @@ OpenHands 可以在交互式命令行模式下运行,允许用户通过命令行
 
 2. 运行以下命令:
 
-```bash
-poetry run python -m openhands.core.cli
-```
 
-该命令将启动一个交互式会话,你可以在其中输入任务并接收来自 OpenHands 的响应。
-
-你需要确保通过环境变量[或 `config.toml` 文件](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml)设置你的模型、API 密钥和其他设置。
-
-
-## 使用 Docker
-
-要在 Docker 中以命令行模式运行 OpenHands,请按照以下步骤操作:
-
-1. 将 `WORKSPACE_BASE` 设置为你希望 OpenHands 编辑的目录:
-
-```bash
-WORKSPACE_BASE=$(pwd)/workspace
-```
-
-2. 将 `LLM_MODEL` 设置为你要使用的模型:
-
-```bash
-LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
-```
-
-3. 将 `LLM_API_KEY` 设置为你的 API 密钥:
-
-```bash
-LLM_API_KEY="sk_test_12345"
-```
-
-4. 运行以下 Docker 命令:
-
-```bash
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_API_KEY=$LLM_API_KEY \
-    -e LLM_MODEL=$LLM_MODEL \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/all-hands-ai/openhands:0.11 \
-    python -m openhands.core.cli
-```
-
-该命令将在 Docker 中启动一个交互式会话,你可以在其中输入任务并接收来自 OpenHands 的响应。
-
-## 命令行命令和预期输出示例
-
-以下是一些命令行命令及其预期输出的示例:
-
-### 示例 1: 简单任务
-
-```bash
-How can I help? >> Write a Python script that prints "Hello, World!"
-```
-
-预期输出:
-
-```bash
-🤖 Sure! Here is a Python script that prints "Hello, World!":
-
-❯ print("Hello, World!")
-```
-
-### 示例 2: Bash 命令
-
-```bash
-How can I help? >> Create a directory named "test_dir"
-```
-
-预期输出:
-
-```bash
-🤖 Creating a directory named "test_dir":
-
-❯ mkdir test_dir
-```
-
-### 示例 3: 错误处理
-
-```bash
-How can I help? >> Delete a non-existent file
-```
-
-预期输出:
-
-```bash
-🤖 An error occurred. Please try again.
-```

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -9,270 +9,4 @@
 
 以下是一个示例配置文件，您可以使用它来定义和使用多个 LLM：
 
-```toml
-[llm]
-# 重要：在此处添加您的 API 密钥，并将模型设置为您要评估的模型
-model = "claude-3-5-sonnet-20241022"
-api_key = "sk-XXX"
 
-[llm.eval_gpt4_1106_preview_llm]
-model = "gpt-4-1106-preview"
-api_key = "XXX"
-temperature = 0.0
-
-[llm.eval_some_openai_compatible_model_llm]
-model = "openai/MODEL_NAME"
-base_url = "https://OPENAI_COMPATIBLE_URL/v1"
-api_key = "XXX"
-temperature = 0.0
-```
-
-
-## 如何在命令行中使用 OpenHands
-
-可以使用以下格式从命令行运行 OpenHands：
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i <max_iterations> \
-        -t "<task_description>" \
-        -c <agent_class> \
-        -l <llm_config>
-```
-
-例如：
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i 10 \
-        -t "Write me a bash script that prints hello world." \
-        -c CodeActAgent \
-        -l llm
-```
-
-此命令使用以下参数运行 OpenHands：
-- 最大迭代次数为 10
-- 指定的任务描述
-- 使用 CodeActAgent
-- 使用 `config.toml` 文件的 `llm` 部分中定义的 LLM 配置
-
-## OpenHands 如何工作
-
-OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工作原理的简化流程：
-
-1. 解析命令行参数并加载配置
-2. 使用 `create_runtime()` 创建运行时环境
-3. 初始化指定的代理
-4. 使用 `run_controller()` 运行控制器，它：
-   - 将运行时附加到代理
-   - 执行代理的任务
-   - 完成后返回最终状态
-
-`run_controller()` 函数是 OpenHands 执行的核心。它管理代理、运行时和任务之间的交互，处理用户输入模拟和事件处理等事项。
-
-
-## 入门最简单的方法：探索现有基准
-
-我们鼓励您查看我们仓库的 [`evaluation/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation)中提供的各种评估基准。
-
-要集成您自己的基准，我们建议从最接近您需求的基准开始。这种方法可以显著简化您的集成过程，允许您在现有结构的基础上进行构建并使其适应您的特定要求。
-
-## 如何创建评估工作流
-
-
-要为您的基准创建评估工作流，请按照以下步骤操作：
-
-1. 导入相关的 OpenHands 实用程序：
-   ```python
-    import openhands.agenthub
-    from evaluation.utils.shared import (
-        EvalMetadata,
-        EvalOutput,
-        make_metadata,
-        prepare_dataset,
-        reset_logger_for_multiprocessing,
-        run_evaluation,
-    )
-    from openhands.controller.state.state import State
-    from openhands.core.config import (
-        AppConfig,
-        SandboxConfig,
-        get_llm_config_arg,
-        parse_arguments,
-    )
-    from openhands.core.logger import openhands_logger as logger
-    from openhands.core.main import create_runtime, run_controller
-    from openhands.events.action import CmdRunAction
-    from openhands.events.observation import CmdOutputObservation, ErrorObservation
-    from openhands.runtime.runtime import Runtime
-   ```
-
-2. 创建配置：
-   ```python
-   def get_config(instance: pd.Series, metadata: EvalMetadata) -> AppConfig:
-       config = AppConfig(
-           default_agent=metadata.agent_class,
-           runtime='eventstream',
-           max_iterations=metadata.max_iterations,
-           sandbox=SandboxConfig(
-               base_container_image='your_container_image',
-               enable_auto_lint=True,
-               timeout=300,
-           ),
-       )
-       config.set_llm_config(metadata.llm_config)
-       return config
-   ```
-
-3. 初始化运行时并设置评估环境：
-   ```python
-   def initialize_runtime(runtime: Runtime, instance: pd.Series):
-       # 在此处设置您的评估环境
-       # 例如，设置环境变量、准备文件等
-       pass
-   ```
-
-4. 创建一个函数来处理每个实例：
-   ```python
-   from openhands.utils.async_utils import call_async_from_sync
-   def process_instance(instance: pd.Series, metadata: EvalMetadata) -> EvalOutput:
-       config = get_config(instance, metadata)
-       runtime = create_runtime(config)
-       call_async_from_sync(runtime.connect)
-       initialize_runtime(runtime, instance)
-
-       instruction = get_instruction(instance, metadata)
-
-       state = run_controller(
-           config=config,
-           task_str=instruction,
-           runtime=runtime,
-           fake_user_response_fn=your_user_response_function,
-       )
-
-       # 评估代理的操作
-       evaluation_result = await evaluate_agent_actions(runtime, instance)
-
-       return EvalOutput(
-           instance_id=instance.instance_id,
-           instruction=instruction,
-           test_result=evaluation_result,
-           metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
-           metrics=state.metrics.get() if state.metrics else None,
-           error=state.last_error if state and state.last_error else None,
-       )
-   ```
-
-5. 运行评估：
-   ```python
-   metadata = make_metadata(llm_config, dataset_name, agent_class, max_iterations, eval_note, eval_output_dir)
-   output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-   instances = prepare_dataset(your_dataset, output_file, eval_n_limit)
-
-   await run_evaluation(
-       instances,
-       metadata,
-       output_file,
-       num_workers,
-       process_instance
-   )
-   ```
-
-此工作流设置配置，初始化运行时环境，通过运行代理并评估其操作来处理每个实例，然后将结果收集到 `EvalOutput` 对象中。`run_evaluation` 函数处理并行化和进度跟踪。
-
-请记住根据您特定的基准要求自定义 `get_instruction`、`your_user_response_function` 和 `evaluate_agent_actions` 函数。
-
-通过遵循此结构，您可以在 OpenHands 框架内为您的基准创建强大的评估工作流。
-
-
-## 理解 `user_response_fn`
-
-`user_response_fn` 是 OpenHands 评估工作流中的关键组件。它模拟用户与代理的交互，允许在评估过程中自动响应。当您想要为代理的查询或操作提供一致的、预定义的响应时，此函数特别有用。
-
-
-### 工作流和交互
-
-处理操作和 `user_response_fn` 的正确工作流如下：
-
-1. 代理接收任务并开始处理
-2. 代理发出操作
-3. 如果操作可执行（例如 CmdRunAction、IPythonRunCellAction）：
-   - 运行时处理操作
-   - 运行时返回观察结果
-4. 如果操作不可执行（通常是 MessageAction）：
-   - 调用 `user_response_fn`
-   - 它返回模拟的用户响应
-5. 代理接收观察结果或模拟响应
-6. 重复步骤 2-5，直到任务完成或达到最大迭代次数
-
-以下是更准确的可视化表示：
-
-```
-                 [代理]
-                    |
-                    v
-               [发出操作]
-                    |
-                    v
-            [操作是否可执行？]
-           /                       \
-         是                         否
-          |                          |
-          v                          v
-     [运行时]                 [user_response_fn]
-          |                          |
-          v                          v
-  [返回观察结果]           [模拟响应]
-           \                        /
-            \                      /
-             v                    v
-           [代理接收反馈]
-                    |
-                    v
-         [继续或完成任务]
-```
-
-在此工作流中：
-
-- 可执行的操作（如运行命令或执行代码）由运行时直接处理
-- 不可执行的操作（通常是当代理想要通信或寻求澄清时）由 `user_response_fn` 处理
-- 然后，代理处理反馈，无论是来自运行时的观察结果还是来自 `user_response_fn` 的模拟响应
-
-这种方法允许自动处理具体操作和模拟用户交互，使其适用于您想要测试代理在最少人工干预的情况下完成任务的能力的评估场景。
-
-### 示例实现
-
-以下是 SWE-Bench 评估中使用的 `user_response_fn` 示例：
-
-```python
-def codeact_user_response(state: State | None) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have solved the task, please first send your answer to user through message and then <execute_bash> exit </execute_bash>.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
-    )
-
-    if state and state.history:
-        # 检查代理是否已尝试与用户对话 3 次，如果是，让代理知道它可以放弃
-        user_msgs = [
-            event
-            for event in state.history.get_events()
-            if isinstance(event, MessageAction) and event.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # 让代理知道它在尝试 3 次后可以放弃
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-```
-
-此函数执行以下操作：
-
-1. 提供一条标准消息，鼓励代理继续工作
-2. 检查代理尝试与用户通信的次数
-3. 如果代理已多次尝试，它会提供放弃的选项
-
-通过使用此函数，您可以确保在多次评估运行中保持一致的行为，并防止代理在等待人工输入时陷入困境。

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -13,49 +13,4 @@
 [请按照开发设置说明](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
 然后运行:
 
-```bash
-poetry run python -m openhands.core.main -t "write a bash script that prints hi"
-```
-
-你需要确保通过环境变量
-[或 `config.toml` 文件](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml)
-设置你的模型、API 密钥和其他设置。
-
-## 使用 Docker
-
-1. 将 `WORKSPACE_BASE` 设置为你希望 OpenHands 编辑的目录:
-
-```bash
-WORKSPACE_BASE=$(pwd)/workspace
-```
-
-2. 将 `LLM_MODEL` 设置为你要使用的模型:
-
-```bash
-LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
-
-```
-
-3. 将 `LLM_API_KEY` 设置为你的 API 密钥:
-
-```bash
-LLM_API_KEY="sk_test_12345"
-```
-
-4. 运行以下 Docker 命令:
-
-```bash
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_API_KEY=$LLM_API_KEY \
-    -e LLM_MODEL=$LLM_MODEL \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/all-hands-ai/openhands:0.11 \
-    python -m openhands.core.main -t "write a bash script that prints hi"
-```
 

docs/modules/usage/how-to/cli-mode.md

@@ -50,7 +50,6 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -59,7 +58,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.12 \
+    ghcr.io/all-hands-ai/openhands:0.11 \
     python -m openhands.core.cli
 ```
 
@@ -108,3 +107,4 @@ Expected Output:
 ```bash
 🤖 An error occurred. Please try again.
 ```
+

docs/modules/usage/how-to/evaluation-harness.md

@@ -9,270 +9,4 @@ OpenHands in development mode uses `config.toml` to keep track of most configura
 
 Here's an example configuration file you can use to define and use multiple LLMs:
 
-```toml
-[llm]
-# IMPORTANT: add your API key here, and set the model to the one you want to evaluate
-model = "claude-3-5-sonnet-20241022"
-api_key = "sk-XXX"
 
-[llm.eval_gpt4_1106_preview_llm]
-model = "gpt-4-1106-preview"
-api_key = "XXX"
-temperature = 0.0
-
-[llm.eval_some_openai_compatible_model_llm]
-model = "openai/MODEL_NAME"
-base_url = "https://OPENAI_COMPATIBLE_URL/v1"
-api_key = "XXX"
-temperature = 0.0
-```
-
-
-## How to use OpenHands in the command line
-
-OpenHands can be run from the command line using the following format:
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i <max_iterations> \
-        -t "<task_description>" \
-        -c <agent_class> \
-        -l <llm_config>
-```
-
-For example:
-
-```bash
-poetry run python ./openhands/core/main.py \
-        -i 10 \
-        -t "Write me a bash script that prints hello world." \
-        -c CodeActAgent \
-        -l llm
-```
-
-This command runs OpenHands with:
-- A maximum of 10 iterations
-- The specified task description
-- Using the CodeActAgent
-- With the LLM configuration defined in the `llm` section of your `config.toml` file
-
-## How does OpenHands work
-
-The main entry point for OpenHands is in `openhands/core/main.py`. Here's a simplified flow of how it works:
-
-1. Parse command-line arguments and load the configuration
-2. Create a runtime environment using `create_runtime()`
-3. Initialize the specified agent
-4. Run the controller using `run_controller()`, which:
-   - Attaches the runtime to the agent
-   - Executes the agent's task
-   - Returns a final state when complete
-
-The `run_controller()` function is the core of OpenHands's execution. It manages the interaction between the agent, the runtime, and the task, handling things like user input simulation and event processing.
-
-
-## Easiest way to get started: Exploring Existing Benchmarks
-
-We encourage you to review the various evaluation benchmarks available in the [`evaluation/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) of our repository.
-
-To integrate your own benchmark, we suggest starting with the one that most closely resembles your needs. This approach can significantly streamline your integration process, allowing you to build upon existing structures and adapt them to your specific requirements.
-
-## How to create an evaluation workflow
-
-
-To create an evaluation workflow for your benchmark, follow these steps:
-
-1. Import relevant OpenHands utilities:
-   ```python
-    import openhands.agenthub
-    from evaluation.utils.shared import (
-        EvalMetadata,
-        EvalOutput,
-        make_metadata,
-        prepare_dataset,
-        reset_logger_for_multiprocessing,
-        run_evaluation,
-    )
-    from openhands.controller.state.state import State
-    from openhands.core.config import (
-        AppConfig,
-        SandboxConfig,
-        get_llm_config_arg,
-        parse_arguments,
-    )
-    from openhands.core.logger import openhands_logger as logger
-    from openhands.core.main import create_runtime, run_controller
-    from openhands.events.action import CmdRunAction
-    from openhands.events.observation import CmdOutputObservation, ErrorObservation
-    from openhands.runtime.runtime import Runtime
-   ```
-
-2. Create a configuration:
-   ```python
-   def get_config(instance: pd.Series, metadata: EvalMetadata) -> AppConfig:
-       config = AppConfig(
-           default_agent=metadata.agent_class,
-           runtime='eventstream',
-           max_iterations=metadata.max_iterations,
-           sandbox=SandboxConfig(
-               base_container_image='your_container_image',
-               enable_auto_lint=True,
-               timeout=300,
-           ),
-       )
-       config.set_llm_config(metadata.llm_config)
-       return config
-   ```
-
-3. Initialize the runtime and set up the evaluation environment:
-   ```python
-   def initialize_runtime(runtime: Runtime, instance: pd.Series):
-       # Set up your evaluation environment here
-       # For example, setting environment variables, preparing files, etc.
-       pass
-   ```
-
-4. Create a function to process each instance:
-   ```python
-   from openhands.utils.async_utils import call_async_from_sync
-   def process_instance(instance: pd.Series, metadata: EvalMetadata) -> EvalOutput:
-       config = get_config(instance, metadata)
-       runtime = create_runtime(config)
-       call_async_from_sync(runtime.connect)
-       initialize_runtime(runtime, instance)
-
-       instruction = get_instruction(instance, metadata)
-
-       state = run_controller(
-           config=config,
-           task_str=instruction,
-           runtime=runtime,
-           fake_user_response_fn=your_user_response_function,
-       )
-
-       # Evaluate the agent's actions
-       evaluation_result = await evaluate_agent_actions(runtime, instance)
-
-       return EvalOutput(
-           instance_id=instance.instance_id,
-           instruction=instruction,
-           test_result=evaluation_result,
-           metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
-           metrics=state.metrics.get() if state.metrics else None,
-           error=state.last_error if state and state.last_error else None,
-       )
-   ```
-
-5. Run the evaluation:
-   ```python
-   metadata = make_metadata(llm_config, dataset_name, agent_class, max_iterations, eval_note, eval_output_dir)
-   output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-   instances = prepare_dataset(your_dataset, output_file, eval_n_limit)
-
-   await run_evaluation(
-       instances,
-       metadata,
-       output_file,
-       num_workers,
-       process_instance
-   )
-   ```
-
-This workflow sets up the configuration, initializes the runtime environment, processes each instance by running the agent and evaluating its actions, and then collects the results into an `EvalOutput` object. The `run_evaluation` function handles parallelization and progress tracking.
-
-Remember to customize the `get_instruction`, `your_user_response_function`, and `evaluate_agent_actions` functions according to your specific benchmark requirements.
-
-By following this structure, you can create a robust evaluation workflow for your benchmark within the OpenHands framework.
-
-
-## Understanding the `user_response_fn`
-
-The `user_response_fn` is a crucial component in OpenHands's evaluation workflow. It simulates user interaction with the agent, allowing for automated responses during the evaluation process. This function is particularly useful when you want to provide consistent, predefined responses to the agent's queries or actions.
-
-
-### Workflow and Interaction
-
-The correct workflow for handling actions and the `user_response_fn` is as follows:
-
-1. Agent receives a task and starts processing
-2. Agent emits an Action
-3. If the Action is executable (e.g., CmdRunAction, IPythonRunCellAction):
-   - The Runtime processes the Action
-   - Runtime returns an Observation
-4. If the Action is not executable (typically a MessageAction):
-   - The `user_response_fn` is called
-   - It returns a simulated user response
-5. The agent receives either the Observation or the simulated response
-6. Steps 2-5 repeat until the task is completed or max iterations are reached
-
-Here's a more accurate visual representation:
-
-```
-                 [Agent]
-                    |
-                    v
-               [Emit Action]
-                    |
-                    v
-            [Is Action Executable?]
-           /                       \
-         Yes                        No
-          |                          |
-          v                          v
-     [Runtime]               [user_response_fn]
-          |                          |
-          v                          v
-  [Return Observation]    [Simulated Response]
-           \                        /
-            \                      /
-             v                    v
-           [Agent receives feedback]
-                    |
-                    v
-         [Continue or Complete Task]
-```
-
-In this workflow:
-
-- Executable actions (like running commands or executing code) are handled directly by the Runtime
-- Non-executable actions (typically when the agent wants to communicate or ask for clarification) are handled by the `user_response_fn`
-- The agent then processes the feedback, whether it's an Observation from the Runtime or a simulated response from the `user_response_fn`
-
-This approach allows for automated handling of both concrete actions and simulated user interactions, making it suitable for evaluation scenarios where you want to test the agent's ability to complete tasks with minimal human intervention.
-
-### Example Implementation
-
-Here's an example of a `user_response_fn` used in the SWE-Bench evaluation:
-
-```python
-def codeact_user_response(state: State | None) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have solved the task, please first send your answer to user through message and then <execute_bash> exit </execute_bash>.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
-    )
-
-    if state and state.history:
-        # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
-        user_msgs = [
-            event
-            for event in state.history.get_events()
-            if isinstance(event, MessageAction) and event.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # let the agent know that it can give up when it has tried 3 times
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-```
-
-This function does the following:
-
-1. Provides a standard message encouraging the agent to continue working
-2. Checks how many times the agent has attempted to communicate with the user
-3. If the agent has made multiple attempts, it provides an option to give up
-
-By using this function, you can ensure consistent behavior across multiple evaluation runs and prevent the agent from getting stuck waiting for human input.

docs/modules/usage/how-to/headless-mode.md

@@ -11,48 +11,4 @@ To run OpenHands in headless mode with Python,
 [follow the Development setup instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
 and then run:
 
-```bash
-poetry run python -m openhands.core.main -t "write a bash script that prints hi"
-```
 
-You'll need to be sure to set your model, API key, and other settings via environment variables
-[or the `config.toml` file](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml).
-
-## With Docker
-
-1. Set `WORKSPACE_BASE` to the directory you want OpenHands to edit:
-
-```bash
-WORKSPACE_BASE=$(pwd)/workspace
-```
-
-2. Set `LLM_MODEL` to the model you want to use:
-
-```bash
-LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
-
-```
-
-3. Set `LLM_API_KEY` to your API key:
-
-```bash
-LLM_API_KEY="sk_test_12345"
-```
-
-4. Run the following Docker command:
-
-```bash
-docker run -it \
-    --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_API_KEY=$LLM_API_KEY \
-    -e LLM_MODEL=$LLM_MODEL \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.12 \
-    python -m openhands.core.main -t "write a bash script that prints hi"
-```

docs/modules/usage/how-to/openshift-example.md

@@ -150,7 +150,7 @@ metadata:
 spec:
   containers:
   - name: openhands-app-2024
-    image: docker.all-hands.dev/all-hands-ai/openhands:main
+    image: ghcr.io/all-hands-ai/openhands:main
     env:
     - name: SANDBOX_USER_ID
       value: "1000"
@@ -164,7 +164,7 @@ spec:
     ports:
     - containerPort: 3000
   - name: openhands-sandbox-2024
-    image: docker.all-hands.dev/all-hands-ai/runtime:main
+    image: ghcr.io/all-hands-ai/sandbox:main
     ports:
     - containerPort: 51963
     command: ["/usr/sbin/sshd", "-D", "-p 51963", "-o", "PermitRootLogin=yes"]
@@ -205,10 +205,10 @@ LAST SEEN   TYPE     REASON                   OBJECT
 9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252"
 9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-31f15b25-faad-4665-a25f-201a530379af"
 6s          Normal   AddedInterface           pod/openhands-app-2024                Add eth0 [10.128.2.48/23] from openshift-sdn
-6s          Normal   Pulled                   pod/openhands-app-2024                Container image "docker.all-hands.dev/all-hands-ai/openhands:main" already present on machine
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/openhands:main" already present on machine
 6s          Normal   Created                  pod/openhands-app-2024                Created container openhands-app-2024
 6s          Normal   Started                  pod/openhands-app-2024                Started container openhands-app-2024
-6s          Normal   Pulled                   pod/openhands-app-2024                Container image "docker.all-hands.dev/all-hands-ai/sandbox:main" already present on machine
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/sandbox:main" already present on machine
 5s          Normal   Created                  pod/openhands-app-2024                Created container openhands-sandbox-2024
 5s          Normal   Started                  pod/openhands-app-2024                Started container openhands-sandbox-2024
 83s         Normal   WaitForFirstConsumer     persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
@@ -334,7 +334,7 @@ spec:
     spec:
       containers:
       - name: openhands-app-2024
-        image: docker.all-hands.dev/all-hands-ai/openhands:main
+        image: ghcr.io/all-hands-ai/openhands:main
         env:
         - name: SANDBOX_USER_ID
           value: "1000"
@@ -356,7 +356,7 @@ spec:
         ports:
         - containerPort: 3000
       - name: openhands-sandbox-2024
-        image: docker.all-hands.dev/all-hands-ai/runtime:main
+        image: ghcr.io/opendevin/sandbox:main
     #    securityContext:
     #      privileged: true  # Add this to allow privileged access
         ports:

docs/modules/usage/installation.mdx

@@ -8,18 +8,24 @@
 
 ## Start the app
 
-The easiest way to run OpenHands is in Docker.
+The easiest way to run OpenHands is in Docker. You can change `WORKSPACE_BASE` below to point OpenHands to
+existing code that you'd like to modify.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik
+export WORKSPACE_BASE=$(pwd)/workspace
 
-docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.12-nikolaik \
+docker pull ghcr.io/all-hands-ai/runtime:0.11-nikolaik
+
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_USER_ID=$(id -u) \
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
-    --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.12
+    --name openhands-app-$(date +%Y%m%d%H%M%S) \
+    ghcr.io/all-hands-ai/openhands:0.11
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).
@@ -28,6 +34,9 @@ You can also run OpenHands in a scriptable [headless mode](https://docs.all-hand
 
 After running the command above, you'll find OpenHands running at [http://localhost:3000](http://localhost:3000).
 
+The agent will have access to the `./workspace` folder to do its work. You can copy existing code here, or change `WORKSPACE_BASE` in the
+command to point to an existing folder.
+
 Upon launching OpenHands, you'll see a settings modal. You **must** select an `LLM Provider` and `LLM Model` and enter a corresponding `API Key`.
 These can be changed at any time by selecting the `Settings` button (gear icon) in the UI.
 
@@ -43,9 +52,9 @@ The `Advanced Options` also allow you to specify a `Base URL` if required.
 ## Versions
 
 The command above pulls the most recent stable release of OpenHands. You have other options as well:
-- For a specific release, use `docker.all-hands.dev/all-hands-ai/openhands:$VERSION`, replacing $VERSION with the version number.
+- For a specific release, use `ghcr.io/all-hands-ai/openhands:$VERSION`, replacing $VERSION with the version number.
 - We use semver, and release major, minor, and patch tags. So `0.9` will automatically point to the latest `0.9.x` release, and `0` will point to the latest `0.x.x` release.
-- For the most up-to-date development version, you can use `docker.all-hands.dev/all-hands-ai/openhands:main`. This version is unstable and is recommended for testing or development purposes only.
+- For the most up-to-date development version, you can use `ghcr.io/all-hands-ai/openhands:main`. This version is unstable and is recommended for testing or development purposes only.
 
 You can choose the tag that best suits your needs based on stability requirements and desired features.
 

docs/modules/usage/llms/local-llms.md

@@ -35,15 +35,32 @@ Use the instructions [here](../getting-started) to start OpenHands using Docker.
 But when running `docker run`, you'll need to add a few more arguments:
 
 ```bash
-docker run # ...
-    --add-host host.docker.internal:host-gateway \
-    -e LLM_OLLAMA_BASE_URL="http://host.docker.internal:11434" \
-    # ...
+--add-host host.docker.internal:host-gateway \
+-e LLM_OLLAMA_BASE_URL="http://host.docker.internal:11434" \
 ```
 
-LLM_OLLAMA_BASE_URL is optional. If you set it, it will be used to show
-the available installed models in the UI.
+LLM_OLLAMA_BASE_URL is optional. If you set it, it will be used to show the available installed models in the UI.
 
+Example:
+
+```bash
+# The directory you want OpenHands to modify. MUST be an absolute path!
+export WORKSPACE_BASE=$(pwd)/workspace
+
+docker run \
+    -it \
+    --pull=always \
+    --add-host host.docker.internal:host-gateway \
+    -e SANDBOX_USER_ID=$(id -u) \
+    -e LLM_OLLAMA_BASE_URL="http://host.docker.internal:11434" \
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -p 3000:3000 \
+    ghcr.io/all-hands-ai/openhands:main
+```
+
+You should now be able to connect to `http://localhost:3000/`
 
 ### Configure the Web Application
 
@@ -159,11 +176,18 @@ CUSTOM_LLM_PROVIDER="openai"
 ### Docker
 
 ```bash
-docker run # ...
+docker run \
+    -it \
+    --pull=always \
+    -e SANDBOX_USER_ID=$(id -u) \
     -e LLM_MODEL="openai/lmstudio" \
     -e LLM_BASE_URL="http://host.docker.internal:1234/v1" \
     -e CUSTOM_LLM_PROVIDER="openai" \
-    # ...
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -p 3000:3000 \
+    ghcr.io/all-hands-ai/openhands:main
 ```
 
 You should now be able to connect to `http://localhost:3000/`

docs/modules/usage/runtimes.md

@@ -1,79 +0,0 @@
-# Runtime Configuration
-
-A Runtime is an environment where the OpenHands agent can edit files and run
-commands.
-
-By default, OpenHands uses a Docker-based runtime, running on your local computer.
-This means you only have to pay for the LLM you're using, and your code is only ever sent to the LLM.
-
-We also support "remote" runtimes, which are typically managed by third-parties.
-They can make setup a bit simpler and more scalable, especially
-if you're running many OpenHands conversations in parallel (e.g. to do evaluation).
-
-## Docker Runtime
-This is the default Runtime that's used when you start OpenHands. You might notice
-some flags being passed to `docker run` that make this possible:
-
-```
-docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    # ...
-```
-
-The `SANDBOX_RUNTIME_CONTAINER_IMAGE` from nikolaik is a pre-built runtime image
-that contains our Runtime server, as well as some basic utilities for Python and NodeJS.
-You can also [build your own runtime image](how-to/custom-sandbox-guide).
-
-### Connecting to Your filesystem
-One useful feature here is the ability to connect to your local filesystem.
-
-To mount your filesystem into the runtime, add the following options to
-the `docker run` command:
-
-```bash
-export WORKSPACE_BASE=/path/to/your/code
-
-docker run # ...
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    # ...
-```
-
-Be careful! There's nothing stopping the OpenHands agent from deleting or modifying
-any files that are mounted into its workspace.
-
-This setup can cause some issues with file permissions (hence the `SANDBOX_USER_ID` variable)
-but seems to work well on most systems.
-
-## All Hands Runtime
-The All Hands Runtime is currently in beta. You can request access by joining
-the #remote-runtime-limited-beta channel on Slack (see the README for an invite).
-
-To use the All Hands Runtime, set the following environment variables when
-starting OpenHands:
-
-```bash
-docker run # ...
-    -e RUNTIME=remote \
-    -e SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.app.all-hands.dev" \
-    -e SANDBOX_API_KEY="your-all-hands-api-key" \
-    -e SANDBOX_KEEP_REMOTE_RUNTIME_ALIVE="true" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
-    # ...
-```
-
-## Modal Runtime
-Our partners at [Modal](https://modal.com/) have also provided a runtime for OpenHands.
-
-To use the Modal Runtime, create an account, and then [create an API key](https://modal.com/settings)
-
-You'll then need to set the following environment variables when starting OpenHands:
-```bash
-docker run # ...
-    -e RUNTIME=modal \
-    -e MODAL_API_TOKEN_ID="your-id" \
-    -e MODAL_API_TOKEN_SECRET="your-secret" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
-```

docs/modules/usage/troubleshooting/troubleshooting.md

@@ -16,7 +16,6 @@ Check out [Notes for WSL on Windows Users](troubleshooting/windows) for some tro
 * [404 Resource not found](#404-resource-not-found)
 * [`make build` getting stuck on package installations](#make-build-getting-stuck-on-package-installations)
 * [Sessions are not restored](#sessions-are-not-restored)
-* [Connection to host.docker.internal timed out](#connection-to-host-docker-internal-timed-out)
 
 ### Unable to connect to Docker
 
@@ -154,27 +153,3 @@ should stay accepted.
 ```bash
 EXPORT JWT_SECRET=A_CONST_VALUE
 ```
-
----
-### Connection to host docker internal timed out
-
-**Symptoms**
-
-When you start the server using the docker command from the main [README](https://github.com/All-Hands-AI/OpenHands/README.md), you get a long timeout
-followed by the a stack trace containing messages like:
-
-* `Connection to host.docker.internal timed out. (connect timeout=310)`
-* `Max retries exceeded with url: /alive`
-
-**Details**
-
-If Docker Engine is installed rather than Docker Desktop, the main command will not work as expected.
-Docker Desktop includes easy DNS configuration for connecting processes running in different containers
-which OpenHands makes use of when the main server is running inside a docker container.
-(Further details: https://forums.docker.com/t/difference-between-docker-desktop-and-docker-engine/124612)
-
-**Workarounds**
-
-* [Install Docker Desktop](https://www.docker.com/products/docker-desktop/)
-* Run OpenHands in [Development Mode](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
-  So that the main server is not run inside a container, but still creates dockerized runtime sandboxes.

docs/sidebars.ts

@@ -90,11 +90,6 @@ const sidebars: SidebarsConfig = {
             },
           ],
         },
-        {
-          type: 'doc',
-          label: 'Runtime Configuration',
-          id: 'usage/runtimes',
-        },
         {
           type: 'doc',
           label: 'Custom Sandbox',

evaluation/discoverybench/README.md

@@ -1,37 +0,0 @@
-# DiscoveryBench with OpenHands
-
-[DiscoveryBench](https://github.com/allenai/discoverybench/) [(Paper)](https://arxiv.org/abs/2407.01725v1) contains 264 tasks collected across 6 diverse domains, such as biology, economics, and sociology. It incorporates discovery workflows from published papers to approximate the real-world challenges faced by researchers.
-
-<p align="center">
-  <a href="[https://github.com/allenai/discoverybench](https://github.com/allenai/discoverybench)">
-    <img src="https://raw.githubusercontent.com/allenai/discoverybench/refs/heads/main/assets/discoverybench-openhands-teaser.png" width="100%" alt="DiscoveryBench Background" />
-  </a>
-</p>
-
-
-## Setup Environment and LLM Configuration
-
-1. Please follow instructions mentioned [here](https://github.com/openlocus/OpenHands/blob/discoverybench-openhands-integration/evaluation/README.md#setup) to setup OpenHands development environment and LLMs locally
-
-2. Execute the bash script to start DiscoveryBench Evaluation
-
-```
-./evaluation/discoverybench/scripts/run_infer.sh [YOUR MODEL CONFIG]
-```
-Replace `[YOUR MODEL CONFIG]` with any model the model that you have set up in `config.toml`
-
-
-## Run Inference on DiscoveryBench Instances
-
-When the `run_infer.sh` script is started, it will automatically pull the latest DiscoveryBench instances & set up the agent environment. The OpenHands agent is invoked to process the task within this environment, producing a hypothesis. We then evaluate it against the “gold” hypothesis provided by DiscoveryBench. The evaluation result, along with the agent chat history is logged to `output.jsonl` under `evaluation_outputs`.
-
-
-```
-./evaluation/discoverybench/scripts/run_infer.sh [MODEL_CONFIG] [GIT_COMMIT] [AGENT] [EVAL_LIMIT] [NUM_WORKERS]
-```
-
-- `MODEL_CONFIG`: Name of the model you want to evaluate with
-- `GIT_COMMIT`: This should be the git commit hash or release tag for OpenHands, e.g., HEAD or a specific tag like 0.6.2.
-- `AGENT`: Use CoderActAgent, right now it only supports that.
-- `EVAL_LIMIT`: Number of samples to evaluate.
-- `NUM_WORKERS`: Number of workers to parallelize the evaluation process.

evaluation/discoverybench/eval_utils/README.md

@@ -1,7 +0,0 @@
-## DiscoveryBench Evaluation Utils
-
-- **`eval_w_subhypo_gen.py`**: Implements the DiscoveryBench logic for evaluating agent-generated hypotheses.
-- **`lm_utils.py`**: Provides utility functions necessary for the evaluation process.
-- **`openai_helpers.py`**: Includes helper functions for OpenAI-related tasks.
-- **`openai_semantic_gen_prompts.py`**: Contains prompts used for semantic generation.
-- **`response_parser.py`**: Handles the parsing of agent-generated hypotheses.

evaluation/discoverybench/eval_utils/eval_w_subhypo_gen.py

@@ -1,538 +0,0 @@
-import json
-import logging
-
-from openai import OpenAI
-
-from .lm_utils import run_chatgpt_query_multi_turn
-from .openai_helpers import get_response
-
-logging.basicConfig(
-    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
-    datefmt='%m/%d/%Y %H:%M:%S',
-    level=logging.INFO,
-)
-logger = logging.getLogger(__name__)
-
-
-def get_score_from_answer(type, answer):
-    if type == 'context':
-        answer = answer.replace('Answer:', '').strip()
-        if answer.startswith('A)'):
-            return 1.0
-        elif answer.startswith('B)'):
-            return 0.0
-        return -1.0
-
-    elif type == 'var':
-        try:
-            var_json = json.loads(answer)
-            # print(f"var_json:{var_json}")
-            p = 0.0
-            r = 0.0
-            f1 = 0.0
-            if var_json['sizeB']:
-                p = var_json['intersection'] / var_json['sizeB']
-            if var_json['sizeA']:
-                r = var_json['intersection'] / var_json['sizeA']
-            if p > 0.0 and r > 0.0:
-                f1 = (2 * p * r) / (p + r)
-            else:
-                f1 = 0.0
-            eval_rec = {
-                'p': p,
-                'r': r,
-                'f1': f1,
-                'sizeA': var_json['sizeA'],
-                'sizeB': var_json['sizeB'],
-                'intersection': var_json['intersection'],
-                'explanation': var_json['explanation'],
-            }
-            print(f'var_eval: {eval_rec}')
-            return eval_rec
-        except Exception:  # COMMENT: added Exception
-            return {'p': -1.0, 'r': -1.0, 'f1': -1.0}
-    elif type == 'rel':
-        print(answer)
-        rel_json = json.loads(answer)
-        answer_str = rel_json['answer'].strip()
-        if answer_str.startswith('A') or 'very similar' in answer_str:
-            return 1.0
-        elif (
-            answer_str.startswith('B') or 'similar but general than HypoA' in answer_str
-        ):
-            return 0.5
-        elif answer_str.startswith('C') or 'different' in answer_str:
-            return 0.0
-        return -1.0
-    return -1.0
-
-
-def ask_dimension_question(
-    query,
-    gold_hypo,
-    gold_workflow,
-    gen_hypo,
-    gen_workflow,
-    dataset_meta,
-    llm_used,
-    dimension,
-    dataset_type,
-    use_column_metadata=True,
-):
-    dimension_question = ''
-    answer = ''
-    score = 0.0
-    if dimension == 'var':
-        score = {'p': -1.0, 'r': -1.0, 'f1': -1.0}
-    num_tokens = 256
-    num_retries = 1
-    json_response = False
-
-    messages = [
-        {
-            'role': 'system',
-            'content': 'You are an AI assistant that helps evaluate a data-driven hypothesis. You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-    ]
-    if dimension == 'context':
-        dimension_question = """\
-        Question: Is HypoB defined in the same context as HypoA?
-        (Context refers to assumptions/stratification under which the hypotheses are defined.)
-        Options: A) same   B) different
-        What is your answer?"""
-    elif dimension == 'var':
-        dimension_question = """\
-        Question: For both HypoA and HypoB, what are the different variables found in the hypotheses? \
-        Return your answer as a JSON object in the following format:
-        ```json
-        {{
-        "sizeA": num of variables used in HypoA
-        "sizeB": num of variables used in HypoB
-        "intersection": num of variables common in HypoA and HypoB. Use *fuzzy matching* to determine intersection, accounting for paraphrases or slightly different surface forms
-        "explanation": a short text explanation about the variables
-        }}```
-        Answer:"""
-        num_tokens = 512
-        num_retries = 1
-        json_response = True
-    elif dimension == 'rel':
-        dimension_question = """\
-        Question: Does HypoB exhibit the same relation as HypoA?
-        Compare using following example hierarchy of relationships (based on specificity): \
-        "there exists a relationship" > "positive relationship" > "positive AND (linear OR quadratic)" > "positive AND linear".
-        Options: A) very similar B) similar but general than HypoA C) different
-        Return your answer as a JSON object in the following format:
-        ```json
-        {{
-        "answer": one of the options from A) very similar B) similar but general than HypoA C) different
-        "explanation": a short text explanation about the relationship comparison
-        }}```
-        Answer:"""
-        num_tokens = 512
-        num_retries = 1
-        json_response = True
-
-    datasets_json = prepare_dataset_metadata_json(
-        dataset_meta, dataset_type=dataset_type, use_column_metadata=use_column_metadata
-    )
-
-    dimension_question_str = f"""\
-        You are going to compare two natural-language hypotheses HypoA and HypoB accompanied with optional workflows: WorkflowA for HypoA and WorkflowB for HypoB. \
-        Both the hypotheses answer the natural language query "QUERY" over the dataset(s) described by dataset description(s) and column description(s) below. \
-        Compare HypoA and HypoB in terms of three aspects: Contexts, Variables, and Relations. \
-        E.g., for the hypothesis "From 1995 to 2009, the number of sandhill cranes around the tundra (Indigilka River) surged by an astounding ~10X":
-        * Contexts refer to stratification of the data under which the given hypothesis is True. E.g., "For all women", "From 1995 to 2009".
-        * Variables refer to the set of variables (either dependent or independent) that are mentioned in the hypothesis. E.g., number of sandhill cranes, location.
-        * Relations refer to the form of relation between the variables. E.g., "surged by ~10x".
-
-        Answer following questions for a given pair of hypotheses, HypoA and HypoB, along with an explanation grounded on the QUERY and the DATASET(S).
-
-        Here is the metadata for the task:
-        ```json
-        {{
-        "datasets": {datasets_json},
-        "query": {query},
-        "HypoA": {gold_hypo},
-        "WorkflowA": {gold_workflow},
-        "HypoB": {gen_hypo},
-        "WorkflowB": {gen_workflow}
-        }}
-        ```
-
-        {dimension_question}"""
-
-    messages.append({'role': 'user', 'content': dimension_question_str})
-    for retry in range(num_retries):
-        response = run_chatgpt_query_multi_turn(
-            messages=messages,
-            model_name=llm_used,
-            max_tokens=num_tokens,
-            temperature=0,  # 0 for greedy best decoding
-            json_response=json_response,
-        )
-        if response is not None:  # COMMENT: changed from != to is not
-            break
-
-    if response is not None:  # COMMENT: changed from != to is not
-        answer = response.choices[0].message.content.strip()
-        score = get_score_from_answer(type=dimension, answer=answer)
-
-    return dimension_question, answer, score
-
-
-def prepare_dataset_metadata_json(dataset_meta, dataset_type, use_column_metadata=True):
-    if dataset_meta is None:  # COMMENT: changed from == to is None
-        return [
-            {
-                'dataset_description': '',
-                'columns': [],
-            }
-        ]
-    datasets_json = []
-    if dataset_type == 'real':
-        for d in dataset_meta['datasets']:
-            datasets_json.append(
-                {
-                    'dataset_description': d['description'],
-                    'columns': [
-                        {'name': col['name'], 'description': col['description']}
-                        for col in d['columns']['raw']
-                    ]
-                    if use_column_metadata
-                    else [],
-                }
-            )
-    else:
-        for d in dataset_meta['datasets']:
-            datasets_json.append(
-                {
-                    'dataset_description': d['description'],
-                    'columns': [
-                        {'name': col['name'], 'description': col['description']}
-                        for col in d['columns']
-                    ]
-                    if use_column_metadata
-                    else [],
-                }
-            )
-    return datasets_json
-
-
-def get_sub_hypotheses(
-    query,
-    hypo,
-    workflow,
-    dataset_meta,
-    llm_used,
-    dataset_type,
-    use_column_metadata=True,
-):
-    client = OpenAI()
-    extraction_prompt = """\
-        Given a set of dataset columns, a ground-truth hypothesis, and the analysis workflow used, your task is to extract three dimensions that define the hypothesis: Context, Variables, and Relations. \
-        Here are the definitions for these dimensions:
-        - Contexts: Boundary conditions that limit the scope of a hypothesis. E.g., “for men over \
-        the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then extract the context from the dataset_descrption.
-        - Variables: Known concepts that interact in a meaningful way under a given context to \
-        produce the hypothesis. E.g., gender, age, income, or "None" if there is no interacting variable.
-        - Relations: Interactions between a given set of variables under a given context to produce \
-        the hypothesis. E.g., “quadratic relationship”, “inversely proportional”, piecewise conditionals, \
-        or "None" if there is no interacting relationship.
-        Make sure to only use the information present in the hypothesis and the workflow. Do not add any new information. \
-        For each dimension, be specific, and do not omit any important details.
-
-        Here is the metadata for the task:
-        ```json
-        {
-        "datasets": %s,
-        "hypothesis": "%s",
-        "workflow": "%s"
-        }
-        ```
-
-        Return your answer as a JSON object in the following format:
-        ```json
-        {
-        "sub_hypo": [
-            {
-                "text": the hypothesis in natural language,
-                "context": a short text description of the context of the hypothesis,
-                "variables": a list of columns involved in the hypothesis,
-                "relations": a short text description of the relationship between the variables of the hypothesis
-            },
-            ...
-        ]
-        }```
-        """
-    datasets_json = prepare_dataset_metadata_json(
-        dataset_meta, dataset_type, use_column_metadata=use_column_metadata
-    )
-    _prompt = extraction_prompt % (datasets_json, hypo, workflow)
-    sub_hypo_json = get_response(client, _prompt, model=llm_used, max_retry=1)
-
-    if sub_hypo_json is not None:  # COMMENT: changed from != to is not
-        # print(f"full hypothesis: {hypo}")
-        print(f'sub_hypo_json: {sub_hypo_json}')
-    else:
-        sub_hypo_json = {
-            'sub_hypo': [],
-        }
-
-    sub_hypo_json['full_hypo'] = hypo
-
-    return sub_hypo_json
-
-
-def match_context_with_gpt(
-    gold_hyp, gold_context, pred_hyp, pred_context, model='gpt-3.5-turbo'
-):
-    prompt = f"""\
-        Given a gold hypothesis, a gold context, a predicted hypothesis, and a predicted context, your task is \
-        to determine if the predicted context semantically matches the ground-truth context. \
-        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
-        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
-        If the predicted context matches the gold context, return true, otherwise return false.
-        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
-        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
-
-        Here is the metadata for the task:
-        ```json
-        {{
-            "gold_hypothesis": "{gold_hyp}",
-            "gold_context": "{gold_context}",
-            "predicted_hypothesis": "{pred_hyp}",
-            "predicted_context": "{pred_context}"
-        }}
-        ```
-
-        Return your answer as a JSON object in the following format:
-        ```json
-        {{
-            "match": true or false
-        }}
-        ```"""
-
-    client = OpenAI()
-    output = get_response(client, prompt, model=model)
-    return output.get('match', False)
-
-
-def is_matching_context(gold_hyp, gold_context, pred_hyp, pred_context, llm_used):
-    if gold_context == pred_context:
-        return True
-    if 'None' in [gold_context, pred_context]:
-        return False
-    return match_context_with_gpt(
-        gold_hyp, gold_context, pred_hyp, pred_context, model=llm_used
-    )
-
-
-def run_eval_gold_vs_gen_NL_subhypo(
-    query,
-    gold_hypo,
-    gold_workflow,
-    gen_hypo,
-    gen_workflow,
-    dataset_meta,
-    llm_used,
-    context_score,
-    dataset_type,
-    use_column_metadata=True,
-):
-    # GPT-4 based evaluation to evaluate generated hypothesis in terms of context, variables, relation
-
-    eval_rec = {
-        'query': query,
-        'HypoA': gold_hypo,
-        'WorkflowA': gold_workflow,
-        'HypoB': gen_hypo,
-        'WorkflowB': gen_workflow,
-    }
-
-    for dimension in ['var', 'rel']:
-        question, answer, score = ask_dimension_question(
-            query,
-            gold_hypo,
-            gold_workflow,
-            gen_hypo,
-            gen_workflow,
-            dataset_meta,
-            llm_used,
-            dimension=dimension,
-            dataset_type=dataset_type,
-            use_column_metadata=use_column_metadata,
-        )
-
-        eval_rec[dimension] = {'question': question, 'answer': answer, 'score': score}
-
-    eval_rec['context'] = context_score
-    eval_rec['accuracy_score'] = (
-        1.0
-        * eval_rec['context']['score']
-        * eval_rec['var']['score']['f1']
-        * eval_rec['rel']['score']
-    )
-
-    return eval_rec
-
-
-def run_eval_gold_vs_gen_NL_hypo_workflow(
-    query,
-    gold_hypo,
-    gold_workflow,
-    gen_hypo,
-    gen_workflow,
-    dataset_meta,
-    llm_used,
-    dataset_type,
-    use_column_metadata=True,
-):
-    # Input: Dataset Metadata, Query, Gold {Hg, Wg}, Predicted {Hp, Wp}
-    # Output: eval_rec json includes final_score
-
-    # Procedure:
-    # Dataset Metadata, Query, Gold {Hg, Wg}, Pred {Hg, Wg}
-    # Gold: [Hg1, Hg2] (compute on the fly) Hg1 is a NL form of subhypothesis
-    # Predicted: [Hp1, Hp2] (compute on the fly)
-
-    # Compute Intersection: [(Hg_i, Hp_j), …]  # tuples of (gold,pred) that matched with context (do this w/o explicit extraction)
-    # # filter so that a gold context and a predicted context are only attached to one tuple
-    # Compute recall_context (programmatically)
-
-    # r_v_list = []
-    # For (Hg_i, Hp_j) in the intersection:
-    #             With Hg_i, Hp_j in NL, ask GPT4 → #variables and #intersection and a paragraph explanation and programmatically calculate f1_v
-    # Hg_i, Hp_j in NL, ask GPT4 → matching score (0, 0.5 or 1) : A) very similar B) similar but general than HypoA C) different + explanation
-    # 	r_v_list ← f1_v * score_r
-    # accuracy_score = mean(r_v_list)
-    # score =   [ recall_context * mean over predicted context(context_score * var_score *rel_score )]
-
-    # recall_context = 1.0  # COMMENT: never used
-    eval_rec = {
-        'query': query,
-        'HypoA': gold_hypo,
-        'WorkflowA': gold_workflow,
-        'HypoB': gen_hypo,
-        'WorkflowB': gen_workflow,
-    }
-
-    gold_sub_hypo_json = get_sub_hypotheses(
-        query=query,
-        hypo=gold_hypo,
-        workflow=gold_workflow,
-        dataset_meta=dataset_meta,
-        llm_used=llm_used,
-        dataset_type=dataset_type,
-        use_column_metadata=use_column_metadata,
-    )
-    if len(gold_sub_hypo_json['sub_hypo']) == 0:
-        gold_sub_hypo_json['sub_hypo'] = [
-            {
-                'text': gold_hypo,
-                'context': 'None',
-                'variables': [],
-                'relations': '',
-                'explanation': 'unable to segment',
-            }
-        ]
-    print(f'gold_sub_hypo_json: {gold_sub_hypo_json}')
-
-    gen_sub_hypo_json = get_sub_hypotheses(
-        query=query,
-        hypo=gen_hypo,
-        workflow=gen_workflow,
-        dataset_meta=dataset_meta,
-        llm_used=llm_used,
-        dataset_type=dataset_type,
-        use_column_metadata=use_column_metadata,
-    )
-    if len(gen_sub_hypo_json['sub_hypo']) == 0:
-        gen_sub_hypo_json['sub_hypo'] = [
-            {
-                'text': gen_hypo,
-                'context': 'None',
-                'variables': [],
-                'relations': '',
-                'explanation': 'unable to segment',
-            }
-        ]
-    print(f'gen_sub_hypo_json: {gen_sub_hypo_json}')
-
-    eval_rec['gold_sub_hypo'] = gold_sub_hypo_json
-    eval_rec['gen_sub_hypo'] = gen_sub_hypo_json
-
-    gold_subh_covered = []
-    gen_subh_to_gold_subh = dict()
-    gen_gold_subh_to_context = dict()
-
-    for p_id, gen_subh in enumerate(gen_sub_hypo_json['sub_hypo']):
-        gen_subh_to_gold_subh[p_id] = -1
-
-        for g_id, gold_subh in enumerate(gold_sub_hypo_json['sub_hypo']):
-            if g_id in gold_subh_covered:
-                continue
-
-            # match context
-            context_bool = is_matching_context(
-                gold_subh['text'],
-                gold_subh.get('context', ''),
-                gen_subh['text'],
-                gen_subh.get('context', ''),
-                llm_used,
-            )
-            if context_bool:
-                context_score = 1.0
-            else:
-                context_score = 0.0
-
-            if context_score == 1.0:  # match only when context_score = 1.0
-                gen_subh_to_gold_subh[p_id] = g_id
-                gold_subh_covered.append(g_id)
-                gen_gold_subh_to_context[f'P{p_id}||G{g_id}'] = {
-                    'question': f"""Comapring: GoldH: {gold_subh["text"]}, GoldC: {gold_subh['context']}\nGenH: {gen_subh['text']}, GenC: {gen_subh['context']}""",
-                    'answer': context_bool,
-                    'score': context_score,
-                }
-                break
-
-    print(f'gen_subh_to_gold_subh: {gen_subh_to_gold_subh}')
-    eval_rec['gen_subh_to_gold_subh'] = gen_subh_to_gold_subh
-    eval_rec['gold_subh_covered'] = gold_subh_covered
-    matched_gold_gen_subh_evals = dict()
-    sum_accuracy_score = 0.0
-    for p_id, g_id in gen_subh_to_gold_subh.items():
-        if g_id >= 0:
-            key = f'P{p_id}||G{g_id}'
-            context_score = gen_gold_subh_to_context[key]
-            subh_eval_rec = run_eval_gold_vs_gen_NL_subhypo(
-                query,
-                gold_hypo,
-                gold_workflow,
-                gen_hypo,
-                gen_workflow,
-                dataset_meta,
-                llm_used,
-                context_score,
-                dataset_type=dataset_type,
-                use_column_metadata=use_column_metadata,
-            )
-            sum_accuracy_score += subh_eval_rec['accuracy_score']
-            matched_gold_gen_subh_evals[key] = subh_eval_rec
-
-    eval_rec['matched_gold_gen_subh_evals'] = matched_gold_gen_subh_evals
-    eval_rec['recall_context'] = (
-        len(gold_subh_covered) / len(gold_sub_hypo_json['sub_hypo'])
-        if len(gold_sub_hypo_json['sub_hypo'])
-        else 0.0
-    )
-    mean_accuracy_score = (
-        sum_accuracy_score / len(gen_subh_to_gold_subh)
-        if len(gen_subh_to_gold_subh)
-        else 0.0
-    )
-    eval_rec['mean_accuracy_score'] = mean_accuracy_score
-    final_score = eval_rec['recall_context'] * mean_accuracy_score
-    eval_rec['final_score'] = final_score
-    print(f'eval_rec: {json.dumps(eval_rec, indent=2)}')
-
-    return eval_rec

evaluation/discoverybench/eval_utils/lm_utils.py

@@ -1,64 +0,0 @@
-import os
-import sys
-import time
-
-from openai import OpenAI
-from tenacity import (
-    retry,
-    stop_after_attempt,  # type: ignore
-    wait_random_exponential,  # type: ignore
-)
-
-if sys.version_info >= (3, 8):
-    from typing import Literal
-else:
-    from typing_extensions import Literal
-
-
-Model = Literal['gpt-4', 'gpt-3.5-turbo', 'text-davinci-003']
-
-OpenAI.api_key = os.getenv('OPENAI_API_KEY')
-OPENAI_GEN_HYP = {
-    'temperature': 0,
-    'max_tokens': 250,
-    'top_p': 1.0,
-    'frequency_penalty': 0,
-    'presence_penalty': 0,
-}
-
-
-@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
-def run_chatgpt_query_multi_turn(
-    messages,
-    model_name='gpt-4-turbo',  # pass "gpt4" for more recent model output
-    max_tokens=256,
-    temperature=0.0,
-    json_response=False,
-):
-    response = None
-    num_retries = 3
-    retry = 0
-    while retry < num_retries:
-        retry += 1
-        try:
-            client = OpenAI()
-
-            if json_response:
-                response = client.chat.completions.create(
-                    model=model_name,
-                    response_format={'type': 'json_object'},
-                    messages=messages,
-                    **OPENAI_GEN_HYP,
-                )
-            else:
-                response = client.chat.completions.create(
-                    model=model_name, messages=messages, **OPENAI_GEN_HYP
-                )
-            break
-
-        except Exception as e:
-            print(e)
-            print('GPT error. Retrying in 2 seconds...')
-            time.sleep(2)
-
-    return response

evaluation/discoverybench/eval_utils/openai_helpers.py

@@ -1,190 +0,0 @@
-import json
-
-
-def OPENAI_TOPIC_GEN_MESSAGES(n=10):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {
-            'role': 'user',
-            'content': f'Given `n`, come up with a list of `n` distinct topics and their descriptions. The topics can be absolutely anything. Be as creative as possible. Return your answer as a JSON object. \n\nFor example, for `n`=3, a valid answer might be:\n```json\n{{"topics": [\n  {{"id": 1, "topic": "cooking", "description": "Related to recipes, ingredients, chefs, etc."}},\n  {{"id": 2, "topic": "sports", "description": "Related to players, stadiums, trophies, etc."}},\n  {{"id": 3, "topic": "antiquing", "description": "Related to unique items, history, etc."}}\n]}}```\n\nNow, give me a list for `n`={n}. Remember, pick diverse topics from everything possible. No consecutive topics should be broadly similar. Directly respond with the answer JSON object.',
-        },
-    ]
-
-
-OPENAI_GEN_HYP = {
-    'temperature': 1.0,
-    'max_tokens': 4096,
-    'top_p': 1.0,
-    'frequency_penalty': 0,
-    'presence_penalty': 0,
-}
-
-
-def OPENAI_SEMANTICS_GEN_MESSAGES(dependent, relationship, domain, domain_desc):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {
-            'role': 'user',
-            'content': f'Given the true relationship in a dataset and a given domain, your task is to come up with an interpretation of some real-world concepts that the relationship could be modeling from the provided domain. It\'s okay to be wrong, but suggest something reasonable. Try as much as possible to make sure that the TARGET is actually derivable from the other variables. Give your answer as a JSON object. Here\'s an example:\n\nRelationship for x2 = "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)"\nDomain="Sales"\nDomain description="Related to product distribution, revenues, marketing, etc."\n\nBased on this, the following real-world concepts might be applicable:\n```json\n{{\n  "dependent": "x2",\n  "relationship": "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)",\n  "domain": "Sales",\n  "trends": {{\n    "x1": "Positive, cubic factor",\n    "x2": "TARGET",\n    "x3": "Positive, linear factor",\n    "x4": "No relation",\n    "x5": "Positive quadratic factor",\n    "x6": "Positive, inverse quadratic factor"\n  }},\n  "interpretation": {{\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area", "is_target": true}},\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n}}```\n\nHere\'s a new test question:\nRelationship for {dependent} = "{relationship}"\nDomain = "{domain}"\nDomain description="{domain_desc}"\n\nRespond only with the answer JSON. Make sure that you do not forget to include the TARGET variable in the interpretation object.',
-        },
-    ]
-
-
-def OPENAI_SEMANTICS_GEN_W_MAP_MESSAGES(
-    dependent, relationship, domain, domain_desc, mapping
-):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {
-            'role': 'user',
-            'content': f'Given a partial mapping from variables to real-world concepts and a true relationship in a dataset, your task is to come up with an interpretation of real-world concepts for the variables without any assigned mapping (those starting with x). Suggest something reasonable. The dependent variable must be derivable only from the other variables in the dependent relationship. Give your answer as a JSON object. Here\'s an example:\n\nExample partial mapping and relationship:\n```json\n{{\n  "domain": "Sales",\n  "domain_description": "Related to product distribution, revenues, marketing, etc.",\n  "variable_mapping": {{\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n  }},\n  "dependent_variable": "sales_area",\n  "dependent_relationship": "(96.4 * pop_area ** 3) + (88.72 * x5 ** 2) + (81.96 * dist_to_distr_ctr ** -2) + (28.13 * x3)  + (97.0)"\n}}```\nBased on this, an example answer would be:\n```json\n{{\n  "dependent_variable": "sales_area",\n  "missing_mapping": ["x3", "x5"],\n  "trends": {{\n    "x3": "Positive, linear factor",\n    "x5": "Positive quadratic factor"\n  }},\n  "interpretation": {{\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }}\n}}```\n\nHere\'s a new test question:\n```json\n{{\n  "domain": "{domain}",\n  "domain_description": "{domain_desc}",\n  "variable_mapping": {json.dumps(mapping, indent=2)},\n  "dependent_variable": "{dependent}",\n  "dependent_relationship": "{relationship}"\n}}```\nRespond only with the answer JSON.',
-        },
-    ]
-
-
-def OPENAI_SEMANTICS_GEN_SUMMARY_MESSAGES(dataset):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {
-            'role': 'user',
-            'content': f'Given the following descriptions of the columns of a dataset, your task is to come up with a natural language overview of the dataset, which should include (1) what the dataset is about, (2) how the data was collected, (3) when the data was collected, and (3) for what purpose the data was collected. Be specific and creative.\n\nExample dataset:\n```json\n{{  \n  "dataset": {{                                                                                                                                                                                       \n    "x6": {{"description": "Ancient artifact significance score", "name": "artifact_significance_score", "is_target": true}},\n    "x1": {{"description": "Distance to ancient city center", "name": "dist_to_ancient_city_ctr"}},\n    "x2": {{"description": "Quantity of discovered relics", "name": "relic_discovery_qty"}},\n    "x3": {{"description": "Years since last archaeological expedition", "name": "years_since_exp"}},\n    "x4": {{"description": "Number of artifacts in excavation site", "name": "artifact_qty"}},\n    "x5": {{"description": "Soil fertility coefficient", "name": "soil_fertility_coef"}},\n    "x7": {{"description": "Distance to ancient burial grounds", "name": "dist_to_burial_grounds"}},\n    "x8": {{"description": "Population estimate of ancient civilization", "name": "ancient_civilization_pop_estimate"}},\n    "x9": {{"description": "Temperature variation in excavation region", "name": "temp_variation"}}\n  }}\n}}```\nExample description:\nThis dataset is about archaeological explorations and findings linked to ancient civilizations. The data was collected in the form of field metrics during various archaeological expeditions during the late mid-20th century. The purpose of the data collection is to evaluate the significance of ancient artifacts discovered during excavations.\n\nHere is a new test dataset.\n{json.dumps(dataset, indent=2)}\nProvide only the description.',
-        },
-    ]
-
-
-def OPENAI_GEN_HYPO_MESSAGES(dataset):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {
-            'role': 'user',
-            'content': f'Given a dataset with its descriptions and the true functional relationship between its variables, your task is to generate 3 levels of hypotheses for the stated relationship in plain English. The three levels are "broad", "medium" and "narrow". Make sure that the hypotheses sound natural. *Only include concepts for variables that are present in the provided functional relationship.* Give your answer as a JSON.\n\nFor example, an example dataset might be the following:\n```json\n{{\n  "domain": "cybersecurity",\n  "summary": "This dataset is about measuring cybersecurity threats in a system. The data was collected by monitoring various cybersecurity metrics in a network environment. The purpose of the data collection is to assess and predict potential cybersecurity risks and vulnerabilities.",\n  "variables": [\n    {{\n      "description": "Level of cybersecurity threat",\n      "name": "cybersecurity_threat",\n      "is_target": true\n    }},\n    {{\n      "description": "Number of failed login attempts",\n      "name": "failed_login_attempts"\n    }},\n    {{\n      "description": "Amount of encrypted data",\n      "name": "encrypted_data"\n    }},\n    {{\n      "description": "Frequency of software updates",\n      "name": "software_updates"\n    }},\n    {{\n      "description": "Number of antivirus software installed",\n      "name": "antivirus_software"\n    }},\n    {{\n      "description": "Quality of firewall protection",\n      "name": "firewall_quality"\n    }}\n  ],\n  "relationship": {{\n    "dependent": "cybersecurity_threat",\n    "relation": "-53.5*encrypted_data**2 - 53.85*failed_login_attempts**2 + 67.75*firewall_quality - 92.16 - 36.68/software_updates**3"\n  }}\n}}```\nGiven this dataset, the following is a valid answer:\n```json\n{{\n  "broad": {{\n    "instruction": "Be vague. Only indicate which concepts might be related but not how they are related",\n    "hypothesis": "Threat to cybersecurity is influenced by several factors including the amount of encrypted data, the number of failed login attempts, the quality of the firewall, as well as how often the software is updated."\n  }},\n  "medium": {{\n    "instruction": "Be slightly more specific. For each factor, indicate carefully whether it positively or negatively affects the relationship, but do not indicate what the exponent is.",\n    "hypothesis": "Cybersecurity threat tends to decrease with the amount of data encryption, the number of failed login attempts, as well as the frequency of software updates to some extent, while improvement in the firewall quality has a positive effect."\n  }},\n  "narrow": {{\n    "instruction": "Be specific. Communicate the concepts, whether there is a positive or negative effect (be careful), and the meaning of the exponent",\n    "hypothesis": "The threat to cybersecurity interacts in a complex manner with various factors. As the amount of encrypted data increases, there is a quadratic decrease in threat. Similarly for the number of failed login attempts, there is a negative quadratic relationship. The quality of the firewall protection on the other hand demonstrates a positive and linear relationship. Finally, the frequency of software updates has an inverse cubic relationship to the threat."\n  }},\n}}\n```\n\nBased on this, provide an answer for the following test dataset:\n```json\n{dataset}```\nRespond only with a JSON.',
-        },
-    ]
-
-
-def create_prompt(usr_msg):
-    return [
-        {
-            'role': 'system',
-            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
-        },
-        {'role': 'user', 'content': usr_msg},
-    ]
-
-
-def get_response(client, prompt, max_retry=5, model='gpt-3.5-turbo', verbose=False):
-    n_try = 0
-    while n_try < max_retry:
-        response = client.chat.completions.create(
-            model=model, messages=create_prompt(prompt), **OPENAI_GEN_HYP
-        )
-
-        # COMMENT: changed from
-        # response.choices[0].message.content.strip().strip('```json').strip('```')
-        content = response.choices[0].message.content
-        cleaned_content = content.split('```json')[1].split('```')[0].strip()
-        output = cleaned_content
-        try:
-            response_json = json.loads(output)
-            return response_json
-        except ValueError:
-            if verbose:
-                print(f'Bad JSON output:\n\n{output}')
-            n_try += 1
-            if n_try < max_retry:
-                if verbose:
-                    print('Retrying...')
-            else:
-                if verbose:
-                    print('Retry limit reached')
-    return None
-
-
-def get_code_fix(
-    client, code, error, max_retry=5, model='gpt-3.5-turbo', verbose=False
-):
-    prompt = f"""\
-Given the following code snippet and error message, provide a single-line fix for the error. \
-Note that the code is going to be executed using python `eval`. \
-The code should be executable and should not produce the error message. Be as specific as possible.
-
-Here's the code and the error:
-{{
-    "code": "{code}",
-    "error": "{error}"
-}}
-
-Return only a JSON object with the fixed code in the following format:
-```json
-{{
-    "fixed_code": "..."
-}}"""
-    response = get_response(
-        client, prompt, max_retry=max_retry, model=model, verbose=verbose
-    )
-    return response
-
-
-def get_new_hypothesis(
-    client, target, old, expr, cols, model='gpt-3.5-turbo', verbose=False
-):
-    prompt = f"""\
-Given a target column from a dataset, a pandas expression to derive the column from existing columns, a list of \
-existing columns, and a previously written hypothesis text, carefully check if the hypothesis text is consistent with \
-the pandas expression or not. If it is consistent, simply return the hypothesis as it is. If it is not consistent, \
-provide a new natural language hypothesis that is consistent with the pandas expression using only the provided \
-information. Be specific.
-
-Here's the information:
-```json
-{{
-    "target_column": "{target}",
-    "pandas_expression": "{expr}",
-    "existing_columns": {json.dumps(cols, indent=4)}
-    "old_hypothesis": "{old}",
-}}```
-
-Give your answer as a new JSON with the following format:
-```json
-{{
-    "hypothesis": "..."
-}}"""
-    response = get_response(client, prompt, model=model, verbose=verbose)
-    return response
-
-
-def replace_variable(client, expr, old, new, model='gpt-3.5-turbo', verbose=False):
-    prompt = f"""\
-Given a pandas "expression", replace mentions of the "old" column with its "new" value such that the resultant \
-expression is equivalent to the original expression.
-
-Here's the information:
-```json
-{{
-    "expression": "{expr}",
-    "old": "{old}",
-    "new": "{new}"
-}}```
-
-Give your answer as a new JSON with the following format:
-```json
-{{
-    "new_expression": "..."
-}}"""
-    response = get_response(client, prompt, model=model, verbose=verbose)
-    return response

evaluation/discoverybench/eval_utils/openai_semantic_gen_prompts.py

@@ -1,151 +0,0 @@
-common_hypothesis_features = [
-    '1-2 sentences',
-    'surprising finding',
-    'includes numeric concepts',
-    'includes categorical concepts',
-    'includes binary concepts',
-]
-hypothesis_features = [
-    ['requires within-cluster analysis'],
-    ['requires across-cluster analysis'],
-    ['corresponds to a polynomial relationship of some columns'],
-    ['corresponds to a ratio between some columns'],
-    ['requires temporal analysis'],
-    ['relationship is based on descriptive statistics of some columns'],
-    ['requires concepts based on percentage or percentiles'],
-    ['relationship is only applicable to one cluster in the data and not the others'],
-]
-
-column_features = [
-    [
-        'must have one target column',
-        'must have quantifiable columns',
-        'must have a few categorical columns',
-        'make sure the categorical column values do not contain special characters',
-        'include a few distractor columns',
-    ]
-]
-
-common_pandas_features = [
-    'must be executable using python `eval` to create the target column in variable `df` (pandas dataframe)',
-    "for e.g., df['A']**2 + 3*df['B'] + 9, np.where(df['A'] > 3, 'Yes', 'No'), etc.",
-    'variables in pandas_expression must be from the existing columns listed above',
-    'variables in pandas_expression must NOT contain the target column itself',
-]
-pandas_features = [
-    ['expression is a quadratic polynomial'],
-    ['expression is a cubic polynomial'],
-    ['expression is a ratio of existing columns'],
-    ['expression is derived through logical combination of existing columns'],
-    # workflow
-]
-pandas_features = [common_pandas_features + p for p in pandas_features]
-
-common_derived_features = [
-    '1-2 sentences',
-    'includes numeric concepts',
-    'includes categorical concepts',
-    'includes binary concepts',
-]
-derived_features = [common_derived_features + h for h in hypothesis_features]
-hypothesis_features = [common_hypothesis_features + h for h in hypothesis_features]
-
-PROMPT_HYP = """\
-Given a dataset topic and description, generate an interesting hypothesis based on \
-the provided instructions. Be creative and come up with an unusual finding.
-
-```json
-{
-    "topic": "%s",
-    "description": "%s",
-    "hypothesis_features": %s,
-    "hypothesis": "..."
-}```
-
-Give your answer as a new JSON with the following format:
-```json
-{
-    "hypothesis": "..."
-}
-```"""
-
-PROMPT_COL = """\
-Given a dataset topic, its description, and a true hypothesis that can be determined from it, \
-generate a list of valid columns based on the provided instructions.
-
-```json
-{
-    "topic": "%s",
-    "description": "%s",
-    "hypothesis": "%s",
-    "column_instructions": %s,
-    "columns": [
-        {
-            "col_name": "...",  # should be an "_"-separated string
-            "description": "...",
-            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
-            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
-            "is_distractor": true/false,  # boolean indicating whether this is a distractor that could cause confusion during data analysis
-            "is_target": true/false  # boolean indicating whether this is the target variable for the hypothesis; at least one column should be the target
-        },
-        ...
-    ],
-    "pandas_instructions": %s,
-    "pandas_equation_for_hypothesis": {
-        "target_col": "...",
-        "target_col_type": "...",
-        "target_col_range": {...},
-        "independent_cols_in_pandas_expression": [],  # list of column names that will be used to derive the target column
-        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
-    }
-}```
-
-Give your answer as a new JSON with the "columns" and "pandas_equation_for_hypothesis" keys filled using the following format:
-```json
-{
-    "columns": [...],
-    "pandas_equation_for_hypothesis": {...}
-}
-```"""
-
-PROMPT_DER = """\
-Given a dataset topic, description, a true hypothesis that can be determined from the data, \
-and a target column from the dataset, generate a hypothesis for the target column using new independent columns not present in the existing columns.
-
-```json
-{
-    "topic": "%s",
-    "description": "%s",
-    "hypothesis": "%s",
-    "existing_columns": %s,
-    "target_column": "%s",
-    "new_to_target_instructions": %s,
-    "new_to_target_hypothesis": "...",  # describe a relationship between new columns that explains the target column
-    "new_columns_for_target": [  # do not repeat any of the existing columns in the dataset
-        {
-            "col_name": "...",  # should be an "_"-separated string
-            "description": "...",
-            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
-            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
-        },
-        ...
-    ],
-    "pandas_instructions": %s,
-    "pandas_equation_for_new_to_target_hypothesis": {
-        "target_col": "...",
-        "target_col_type": "...",
-        "target_col_range": {...},
-        "independent_cols_in_pandas_expression": [],  # list of column names from new_columns_for_target that will be used to derive target_col
-        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
-    }
-}```
-
-Give your answer as a new JSON with the "new_to_target_hypothesis", "new_columns_for_target", and \
-"pandas_equation_for_new_to_target_hypothesis" keys filled using the following format:
-```json
-{
-    "new_to_target_hypothesis": "...",
-    "new_columns_for_target": [...],
-    "pandas_equation_for_new_to_target_hypothesis": {...}
-}
-```"""

evaluation/discoverybench/eval_utils/response_parser.py

@@ -1,52 +0,0 @@
-workflow_summary_markers = [
-    'WORKFLOW SUMMARY',
-    'WORKFLOW_SUMMARY',
-    'WORKFLOW-SUMMARY',
-    'Workflow Summary',
-]
-
-final_answer_markers = [
-    'FINAL ANSWER',
-    'FINAL_ANSWER',
-    'FINAL-ANSWER',
-    'Final Answer',
-    'Scientific Hypothesis',
-    'Hypothesis',
-]
-
-next_agent_markers = [
-    'NEXT AGENT',
-    'NEXT-AGENT',
-    'NEXT_AGENT',
-    'FEEDBACK',
-]
-
-
-def extract_between(content, start_markers, end_markers=None):
-    for marker in start_markers:
-        if marker in content:
-            result = content.split(marker, 1)[1]
-            if end_markers:
-                for end_marker in end_markers:
-                    if end_marker in result:
-                        result = result.split(end_marker, 1)[0]
-            return result
-    return ''
-
-
-def extract_gen_hypo_from_logs(content: str):
-    error = ''
-
-    gen_workflow = extract_between(
-        content, workflow_summary_markers, final_answer_markers
-    )
-
-    if not gen_workflow:
-        error += 'No Workflow Summary found in the line. | '
-
-    gen_hypothesis = extract_between(content, final_answer_markers, next_agent_markers)
-
-    if not gen_hypothesis:
-        error += 'No Final Answer in the line.'
-
-    return gen_hypothesis, gen_workflow, error

evaluation/discoverybench/run_infer.py

@@ -1,491 +0,0 @@
-import asyncio
-import json
-import os
-
-import git
-import pandas as pd
-
-from evaluation.discoverybench.eval_utils.eval_w_subhypo_gen import (
-    run_eval_gold_vs_gen_NL_hypo_workflow,
-)
-from evaluation.discoverybench.eval_utils.response_parser import (
-    extract_gen_hypo_from_logs,
-)
-from evaluation.utils.shared import (
-    EvalMetadata,
-    EvalOutput,
-    codeact_user_response,
-    make_metadata,
-    prepare_dataset,
-    reset_logger_for_multiprocessing,
-    run_evaluation,
-)
-from openhands.controller.state.state import State
-from openhands.core.config import (
-    AgentConfig,
-    AppConfig,
-    SandboxConfig,
-    get_llm_config_arg,
-    parse_arguments,
-)
-from openhands.core.logger import openhands_logger as logger
-from openhands.core.main import create_runtime, run_controller
-from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
-from openhands.events.observation import CmdOutputObservation
-from openhands.runtime.base import Runtime
-from openhands.utils.async_utils import call_async_from_sync
-
-EVALUATION_LLM = 'gpt-4-1106-preview'
-
-DATA_FILES = {}
-
-LIBRARIES = [
-    'pandas',
-    'numpy',
-    'scipy',
-    'matplotlib',
-    'seaborn',
-    'scikit-learn',
-    'statsmodels',
-]
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-}
-
-AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
-}
-
-
-def get_config(
-    metadata: EvalMetadata,
-) -> AppConfig:
-    config = AppConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime='eventstream',
-        max_iterations=metadata.max_iterations,
-        sandbox=SandboxConfig(
-            base_container_image='python:3.12-bookworm',
-            enable_auto_lint=True,
-            use_host_network=False,
-        ),
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
-    config.set_llm_config(metadata.llm_config)
-    agent_config = AgentConfig(
-        function_calling=False,
-        codeact_enable_jupyter=True,
-        codeact_enable_browsing_delegate=True,
-    )
-    config.set_agent_config(agent_config)
-    return config
-
-
-def get_dv_query_for_real(
-    datasets, question, domain_knowledge=None, workflow_tags=None
-):
-    """
-    Prepare a structured query for the agent to execute on the specified datasets.
-
-    This function constructs a query by compiling metadata from the provided datasets, along with any relevant domain knowledge and workflow tags.
-
-    Args:
-        datasets: List of datasets
-        question: Query to be answered
-        domain_knowledge: Domain knowledge if any
-        workflow_tags: Workflow tags if any
-
-    Returns:
-        query_to_dv: Query to be run on the dataset
-        dataset_meta: Metadata of the dataset
-    """
-
-    dataset_meta = ''
-    for dataset_metadata in datasets:
-        dataset_meta += 'Dataset name: ' + dataset_metadata['name']
-        dataset_meta += 'Dataset description: ' + dataset_metadata['description']
-        dataset_meta += '\nBrief description of columns: '
-        for col in dataset_metadata['columns']['raw']:
-            dataset_meta += col['name'] + ': ' + col['description'] + ', '
-
-    query_to_dv = dataset_meta
-
-    query_to_dv += f'\nQuery: {question}'
-
-    if domain_knowledge:
-        query_to_dv += (
-            '\nAdditionally, we provide some hints that might be useful to solve the task. Domain Knowledge: \n'
-            + domain_knowledge
-            + '.\n'
-        )
-
-    if workflow_tags:
-        query_to_dv += 'The meta tags are: ' + workflow_tags + '.\n'
-
-    query_to_dv += (
-        'In the final answer, please write down a scientific hypothesis in '
-        'natural language, derived from the provided dataset, clearly stating the '
-        'context of hypothesis (if any), variables chosen (if any) and '
-        'relationship between those variables (if any) including any statistical significance.'
-        'Also generate a summary of the full workflow starting from data loading that led to the final answer as WORKFLOW SUMMARY:'
-    )
-
-    # Run the NL query through datavoyager
-    return query_to_dv, dataset_meta
-
-
-def initialize_runtime(runtime: Runtime, data_files: list[str]):
-    """
-    Initialize the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    """
-    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
-    obs: CmdOutputObservation
-
-    action = CmdRunAction(command='mkdir -p /workspace')
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    assert obs.exit_code == 0
-
-    action = CmdRunAction(command='cd /workspace')
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    assert obs.exit_code == 0
-
-    for file in data_files:
-        runtime.copy_to(
-            file,
-            '/workspace',
-        )
-
-    for lib in LIBRARIES:
-        action = CmdRunAction(command=f'pip install {lib}')
-        logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = runtime.run_action(action)
-        assert obs.exit_code == 0
-
-    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
-
-
-def get_last_agent_finish_action(state: State) -> AgentFinishAction:
-    for event in state.history.get_events(reverse=True):
-        if isinstance(event, AgentFinishAction):
-            return event
-    return None
-
-
-def get_last_message_action(state: State) -> MessageAction:
-    for event in state.history.get_events(reverse=True):
-        if isinstance(event, MessageAction):
-            return event
-    return None
-
-
-def complete_runtime(state: State):
-    last_agent_finish_action = get_last_agent_finish_action(state)
-    last_agent_message_action = get_last_message_action(state)
-
-    if last_agent_finish_action is not None:
-        final_message_1 = last_agent_finish_action.thought
-        gen_hypo_1, gen_workflow_1, error_1 = extract_gen_hypo_from_logs(
-            final_message_1
-        )
-    else:
-        gen_hypo_1, gen_workflow_1, error_1 = '', '', ''
-
-    if last_agent_message_action is not None:
-        final_message_2 = last_agent_message_action.content
-        gen_hypo_2, gen_workflow_2, error_2 = extract_gen_hypo_from_logs(
-            final_message_2
-        )
-    else:
-        gen_hypo_2, gen_workflow_2, error_2 = '', '', ''
-
-    if gen_hypo_1 and gen_hypo_2:
-        test_result = {
-            'gen_hypo': last_agent_finish_action.thought
-            if last_agent_finish_action
-            else last_agent_message_action.content,
-            'gen_workflow': '',
-            'error': '',
-        }
-        return test_result
-
-    test_result = {
-        'gen_hypo': gen_hypo_1 if gen_hypo_1 else gen_hypo_2,
-        'gen_workflow': gen_workflow_1 if gen_workflow_1 else gen_workflow_2,
-        'error': error_1 if error_1 else error_2,
-    }
-
-    return test_result
-
-
-def process_instance(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-    reset_logger: bool = True,
-):
-    """
-    Process and evaluate a single instance of the dataset.
-
-    This function executes the OpenHands agent
-    for a specific instance of the dataset. It retrieves
-    the agent's results and evaluates them against the gold
-    hypothesis.
-
-    Args:
-        instance: A single row of the dataset
-        metadata: Metadata for the evaluation
-        reset_logger: Whether to reset the logger
-
-    Returns:
-        output: EvalOutput object
-    """
-
-    config = get_config(metadata)
-
-    # use a session id for concurrent evaluation
-    sid = 'ID_' + str(instance.instance_id)
-
-    # Setup the logger properly, so you can run
-    # multi-processing to parallelize the evaluation
-    if reset_logger:
-        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
-        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
-    else:
-        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
-
-    problem_statement, dataset_metadata = get_dv_query_for_real(
-        datasets=instance.datasets,
-        question=instance.query,
-        domain_knowledge=instance.domain_knowledge,
-        workflow_tags=instance.workflow_tags,
-    )
-
-    # Prepare instruction
-    instruction = (
-        f'You are a discovery agent who can execute a python code only once to answer a query based on one or more datasets. The datasets will be present in the current directory.\n\n'
-        'Environment has been set up for you to start working. You may assume all necessary tools and datasets are installed.\n\n'
-        '# Problem Statement\n'
-        f'{problem_statement}\n\n'
-    )
-    instruction += (
-        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
-        'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
-        'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
-    )
-    # NOTE: You can actually set slightly different instruction for different agents
-    instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
-
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    runtime = create_runtime(config, sid=sid)
-    call_async_from_sync(runtime.connect)
-    initialize_runtime(runtime, instance.data_files)
-
-    state: State | None = asyncio.run(
-        run_controller(
-            config=config,
-            initial_user_action=MessageAction(content=instruction),
-            runtime=runtime,
-            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-                metadata.agent_class
-            ),
-        )
-    )
-
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    metrics = state.metrics.get() if state.metrics else None
-    test_result = complete_runtime(state)
-
-    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
-    # for compatibility with the existing output format, we can remake the pairs here
-    # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
-
-    # DiscoveryBench Evaluation
-    eval_rec = run_eval_gold_vs_gen_NL_hypo_workflow(
-        query=instance.query,
-        gold_hypo=instance.gold_hypo,
-        gold_workflow='',
-        gen_hypo=test_result['gen_hypo'],
-        gen_workflow='',
-        dataset_meta=instance.dataset_metadata,
-        llm_used=EVALUATION_LLM,
-        dataset_type='real',
-    )
-
-    test_result['eval_rec'] = eval_rec
-
-    output = EvalOutput(
-        instance_id=str(instance.instance_id),
-        instruction=instruction,
-        metadata=metadata,
-        history=histories,
-        metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
-        test_result=test_result,
-    )
-
-    return output
-
-
-def update_csv_name(name):
-    name = name.replace('-', '_')
-
-    if 'meta_regression' in name:
-        name = name.replace('meta_regression', 'meta-regression')
-    if 'ML_enabled' in name:
-        name = name.replace('ML_enabled', 'ML-enabled')
-
-    return name
-
-
-def list_csv_files(list_of_datasets):
-    res = []
-    for ele in list_of_datasets:
-        for key, value in ele.items():
-            if key == 'name':
-                csv_file_name = update_csv_name(value)
-                res.append(DATA_FILES[csv_file_name])
-    return res
-
-
-def create_dataset(repo_location: str, split: str = 'test'):
-    """
-    Create a dataset from the discoverybench repository
-    by walking through the repository and extracting metadata
-    from the metadata_{}.json files
-
-    Args:
-        repo_location: Location of the repository
-        split: Split of the dataset to use
-
-    Returns:
-        df: DataFrame containing the dataset instances
-    """
-
-    data_dict = {}
-
-    data_location = os.path.join(repo_location, 'discoverybench', 'real', split)
-    answer_key_location = os.path.join(repo_location, 'eval', 'answer_key_real.csv')
-
-    idx = 0
-
-    for root, dirs, files in os.walk(data_location):
-        for file in files:
-            if file.endswith('.json'):
-                if 'metadata' in file:
-                    metadata = json.load(open(os.path.join(root, file)))
-
-                    dataset = root.split('/')[-1]
-                    metadata_id = file.split('_')[-1].split('.')[0]
-                    domain = metadata.get('domain', '')
-                    domain_knowledge = metadata.get('domain_knowledge', '')
-                    workflow_tags = metadata.get('workflow_tags', '')
-                    datasets = metadata.get('datasets', [])
-                    queries = metadata.get('queries', [])
-                    gold_workflow = metadata.get('workflow')
-
-                    # loop through queries list to get queries
-                    # and each query has qid; add that to dictionary
-                    for query in queries[0]:
-                        qid = query.get('qid', '')
-
-                        data = {
-                            'dataset': dataset,
-                            'metadata_id': metadata_id,
-                            'qid': qid,
-                            'domain': domain,
-                            'domain_knowledge': domain_knowledge,
-                            'workflow_tags': workflow_tags,
-                            'datasets': datasets,
-                            'question_type': query['question_type'],
-                            'query': query['question'],
-                            'gold_workflow': gold_workflow,
-                            'dataset_metadata': metadata,
-                        }
-
-                        data_dict[idx] = data
-                        idx += 1
-
-            if file.endswith('.csv'):
-                DATA_FILES[file] = os.path.join(root, file)
-            if file.endswith('.txt'):
-                DATA_FILES[file] = os.path.join(root, file)
-
-    df = pd.DataFrame.from_dict(data_dict, orient='index')
-
-    df['instance_id'] = df.index
-
-    df['data_files'] = df['datasets'].apply(lambda x: list_csv_files(x))
-
-    answer_key = pd.read_csv(answer_key_location)
-
-    answer_key = answer_key.rename(
-        columns={
-            'metadataid': 'metadata_id',
-            'query_id': 'qid',
-            'gold_hypothesis': 'gold_hypothesis',
-        }
-    )
-
-    df['qid'] = df['qid'].astype(int)
-    df['metadata_id'] = df['metadata_id'].astype(int)
-
-    answer_key['qid'] = answer_key['qid'].astype(int)
-    answer_key['metadata_id'] = answer_key['metadata_id'].astype(int)
-
-    df = pd.merge(df, answer_key, on=['dataset', 'metadata_id', 'qid'], how='left')
-
-    return df
-
-
-if __name__ == '__main__':
-    args = parse_arguments()
-
-    # clone git repositor for csv files
-    repo_url = 'https://github.com/allenai/discoverybench.git'
-    repo_location = 'git-discoverybench-allenai'
-
-    try:
-        git.Repo.clone_from(repo_url, repo_location)
-    except git.exc.GitCommandError:
-        print('Repository already exists')
-
-    dataset = create_dataset(repo_location)
-
-    # check if there is any empty csv_file
-    if dataset['data_files'].isnull().any():
-        raise ValueError('Some csv files are missing.')
-
-    llm_config = None
-    if args.llm_config:
-        llm_config = get_llm_config_arg(args.llm_config)
-    if llm_config is None:
-        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
-
-    metadata = make_metadata(
-        llm_config,
-        'discoverybench-python',
-        args.agent_cls,
-        args.max_iterations,
-        args.eval_note,
-        args.eval_output_dir,
-    )
-    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
-
-    run_evaluation(
-        instances,
-        metadata,
-        output_file,
-        args.eval_num_workers,
-        process_instance,
-    )

evaluation/discoverybench/scripts/run_infer.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-source "evaluation/utils/version_control.sh"
-
-MODEL_CONFIG=$1
-COMMIT_HASH=$2
-AGENT=$3
-EVAL_LIMIT=$4
-NUM_WORKERS=$5
-
-if [ -z "$NUM_WORKERS" ]; then
-  NUM_WORKERS=1
-  echo "Number of workers not specified, use default $NUM_WORKERS"
-fi
-
-# ################################################################################
-
-checkout_eval_branch
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-get_agent_version
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-
-COMMAND="poetry run python evaluation/discoverybench/run_infer.py \
-  --agent-cls $AGENT \
-  --llm-config $MODEL_CONFIG \
-  --max-iterations 10 \
-  --max-chars 10000000 \
-  --eval-num-workers $NUM_WORKERS \
-  --eval-note $AGENT_VERSION"
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND

evaluation/scienceagentbench/Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.11-bookworm
-
-
-# For OpenHands agents to explore the dataset directories, please download the full benchmark [here](https://buckeyemailosu-my.sharepoint.com/:u:/g/personal/chen_8336_buckeyemail_osu_edu/EQuA6uJ3CtRHvRfZ2GiN1tYBRVJE4DSUD10MW61fr7HuSQ?e=sCBegG) and unzip it with password `scienceagentbench`.
-# **Please DO NOT redistribute the unzipped data files online.**
-# It will download a benchmark.zip file to the current directory.
-# unzip it and put the benchmark folder under evaluation/scienceagentbench/
-
-RUN mkdir -p /benchmark
-COPY benchmark /benchmark
-
-RUN mkdir -p /workspace
-WORKDIR /workspace
-
-# pushd evaluation/scienceagentbench
-# docker build -t xingyaoww/openhands-eval-scienceagentbench .
-# popd

evaluation/scienceagentbench/Dockerfile.evaluator

@@ -1,25 +0,0 @@
-FROM mambaorg/micromamba:debian12
-
-USER root
-# For https://github.com/OSU-NLP-Group/ScienceAgentBench/tree/main?tab=readme-ov-file#code-generation-with-agents
-
-RUN micromamba create -n sci-agent-eval python=3.10 pip setuptools wheel
-RUN micromamba run -n sci-agent-eval pip install pip-tools
-
-RUN mkdir -p /workspace
-WORKDIR /workspace
-
-RUN apt-get update && apt-get install -y git
-
-RUN git clone https://github.com/OSU-NLP-Group/ScienceAgentBench.git /workspace/
-RUN git checkout 4eddc7db6449a5ade3e37285747c8b208cd54ce7
-
-RUN micromamba create -n sci-agent python=3.10 pip setuptools wheel
-RUN micromamba run -n sci-agent pip install -r requirements.txt
-
-# Replace all occurence of conda with micromamba under the /workspace
-RUN find ./ -type f -exec sed -i 's/conda/micromamba/g' {} \;
-
-# pushd evaluation/scienceagentbench
-# docker build -t xingyaoww/openhands-eval-scienceagentbench-evaluator -f Dockerfile.evaluator .
-# popd

evaluation/scienceagentbench/README.md

@@ -1,54 +0,0 @@
-# ScienceAgentBench Evaluation with OpenHands
-
-This folder contains the evaluation harness for [ScienceAgentBench](https://osu-nlp-group.github.io/ScienceAgentBench/) (paper: https://arxiv.org/abs/2410.05080).
-
-## Setup Environment and LLM Configuration
-
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
-
-## Setup ScienceAgentBench
-
-To prevent benchmark data contamination, we only provide the annotation sheet on [Huggingface](https://huggingface.co/datasets/osunlp/ScienceAgentBench), which includes all necessary *inputs* to run an agent.
-
-## Run Inference on ScienceAgentBench
-
-```bash
-./evaluation/scienceagentbench/scripts/run_infer.sh [model_config] [git-version] [use_knowledge] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
-
-# Example
-./evaluation/scienceagentbench/scripts/run_infer.sh llm.eval_gpt4o 0.9.3
-```
-
-where `model_config` is mandatory, and the rest are optional.
-
-- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`.
-- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
-like to evaluate. It could also be a release tag like `0.6.2`.
-- `use_knowledge`, e.g. `true`, specifies whether allowing the agent to use expert-provided knowledge as additional input or not. By default, it is set to `false`.
-- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
-default, the script evaluates the entire SWE-bench_Lite test set (300 issues). Note:
-in order to use `eval_limit`, you must also set `agent`.
-- `max_iter`, e.g. `20`, is the maximum number of iterations for the agent to run. By
-default, it is set to 30.
-- `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
-default, it is set to 1.
-
-## Evaluate Generated Programs
-
-### Extract Necessary Information from OpenHands Log
-
-After the inference is completed, you may use the following command to extract necessary information from the output log for evaluation:
-
-```bash
-python post_proc.py [log_fname]
-```
-- `log_fname`, e.g. `evaluation/.../output.jsonl`, is the automatically saved trajectory log of an OpenHands agent.
-
-Output will be write to e.g. `evaluation/.../output.converted.jsonl`
-
-### Run evaluation
-
-Please follow the steps [here](https://github.com/OSU-NLP-Group/ScienceAgentBench/tree/main?tab=readme-ov-file#evaluation-of-generated-code) to evaluate the generated programs.

evaluation/scienceagentbench/post_proc.py

@@ -1,30 +0,0 @@
-import json
-from argparse import ArgumentParser
-
-if __name__ == '__main__':
-    parser = ArgumentParser()
-    parser.add_argument(
-        'log_fname',
-        type=str,
-    )
-    args = parser.parse_args()
-
-    fname = args.log_fname
-    out_fname = args.log_fname.replace('.jsonl', '.converted.jsonl')
-
-    log = [json.loads(line) for line in open(fname)]
-
-    simple_log = [
-        json.dumps(
-            {
-                'instance_id': ex['instance_id'],
-                'instruction': ex['instruction'],
-                'test_result': ex['test_result'],
-                'cost': ex['metrics']['accumulated_cost'],
-            }
-        )
-        for ex in log
-    ]
-
-    with open(out_fname, 'w+', encoding='utf-8') as f:
-        f.write('\n'.join(simple_log))

evaluation/scienceagentbench/run_infer.py

@@ -1,292 +0,0 @@
-import asyncio
-import os
-from typing import Any
-
-import pandas as pd
-from datasets import load_dataset
-from tqdm import tqdm
-
-from evaluation.utils.shared import (
-    EvalMetadata,
-    EvalOutput,
-    codeact_user_response,
-    make_metadata,
-    prepare_dataset,
-    reset_logger_for_multiprocessing,
-    run_evaluation,
-)
-from openhands.controller.state.state import State
-from openhands.core.config import (
-    AppConfig,
-    SandboxConfig,
-    get_llm_config_arg,
-    get_parser,
-)
-from openhands.core.logger import openhands_logger as logger
-from openhands.core.main import create_runtime, run_controller
-from openhands.events.action import CmdRunAction, MessageAction
-from openhands.events.observation import CmdOutputObservation
-from openhands.runtime.base import Runtime
-from openhands.utils.async_utils import call_async_from_sync
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-}
-
-LOCAL_DATASET_PATH = os.path.join(os.path.dirname(__file__), 'benchmark')
-
-
-def format_task_dict(example, use_knowledge):
-    task = {
-        'instance_id': example['instance_id'],
-        'task_inst': example['task_inst'],
-        'dataset_path': '/benchmark/datasets/'
-        + example['dataset_folder_tree'].split('\n')[0][4:],
-        'dataset_folder_tree': example['dataset_folder_tree'],
-        'dataset_preview': example['dataset_preview'],
-        'pred_program_name': 'pred_' + example['gold_program_name'],
-    }
-
-    if use_knowledge:
-        task['task_inst'] += '\n' + str(example['domain_knowledge'])
-
-    return task
-
-
-def get_config(
-    metadata: EvalMetadata,
-    instance_id: str,
-) -> AppConfig:
-    config = AppConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime=os.environ.get('RUNTIME', 'eventstream'),
-        max_budget_per_task=4,
-        max_iterations=metadata.max_iterations,
-        sandbox=SandboxConfig(
-            base_container_image='docker.io/xingyaoww/openhands-eval-scienceagentbench',
-            enable_auto_lint=True,
-            use_host_network=False,
-            timeout=300,
-            api_key=os.environ.get('ALLHANDS_API_KEY', None),
-            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_remote_runtime_alive=False,
-        ),
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
-    config.set_llm_config(metadata.llm_config)
-    if metadata.llm_config.log_completions:
-        metadata.llm_config.log_completions_folder = os.path.join(
-            metadata.eval_output_dir, 'llm_completions', instance_id
-        )
-        logger.info(
-            f'Logging LLM completions for instance {instance_id} to '
-            f'{metadata.llm_config.log_completions_folder}'
-        )
-    return config
-
-
-def initialize_runtime(
-    runtime: Runtime,
-    instance: pd.Series,  # this argument is not required
-):
-    """Initialize the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    """
-    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
-    obs: CmdOutputObservation
-
-    # Set up workspace directories
-    action = CmdRunAction(command='mkdir -p /workspace/pred_programs')
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    assert obs.exit_code == 0
-
-    action = CmdRunAction(command='mkdir -p /workspace/pred_results')
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    assert obs.exit_code == 0
-
-    dataset_name = instance['dataset_folder_tree'].split('\n')[0][4:].rstrip('/')
-
-    # Copy the dataset to the workspace
-    dataset_dir = os.path.join(
-        LOCAL_DATASET_PATH,
-        'datasets',
-        dataset_name,
-    )
-    runtime.copy_to(dataset_dir, '/workspace/benchmark/datasets', recursive=True)
-
-    # Check the dataset exists
-    action = CmdRunAction(
-        command='cd /workspace/benchmark/datasets && ls',
-        keep_prompt=False,
-    )
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert obs.exit_code == 0
-    assert dataset_name in obs.content
-
-    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
-
-
-def complete_runtime(
-    runtime: Runtime,
-    instance: pd.Series,
-) -> dict[str, Any]:
-    """Complete the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    If you need to do something in the sandbox to get the correctness metric after
-    the agent has run, modify this function.
-    """
-    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
-    obs: CmdOutputObservation
-
-    test_result = {}
-
-    action = CmdRunAction(command='cd /workspace')
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-
-    assert obs.exit_code == 0
-
-    action = CmdRunAction(
-        command=f'cat pred_programs/{instance.pred_program_name}',
-        keep_prompt=False,
-    )
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-
-    if obs.exit_code == 0:
-        test_result = {'program': obs.content}
-    else:
-        test_result = {'program': 'ERROR'}
-
-    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
-    return test_result
-
-
-def process_instance(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-    reset_logger: bool = True,
-) -> EvalOutput:
-    instance_id = instance.instance_id.replace('/', '__')
-    config = get_config(metadata, instance_id)
-
-    # Set up the logger properly, so you can run multi-processing to parallelize the evaluation
-    if reset_logger:
-        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
-        reset_logger_for_multiprocessing(logger, instance_id, log_dir)
-    else:
-        logger.info(f'Starting evaluation for instance {instance_id}.')
-
-    instruction = f"""You are an expert Python programming assistant that helps scientist users to write high-quality code to solve their tasks.
-Given a user request, you are expected to write a complete program that accomplishes the requested task and save any outputs to `/workspace/pred_results/` in the correct format.
-
-Here's the user request you need to work on:
-{instance.task_inst}
-
-You can access the dataset at `{instance.dataset_path}`. Here is the directory structure of the dataset:
-```
-{instance.dataset_folder_tree}
-```
-Here are some helpful previews for the dataset file(s):
-{instance.dataset_preview}
-
-Please save your program as `/workspace/pred_programs/{instance.pred_program_name}`.
-Then, please run the program to check and fix any errors.
-Please do NOT run the program in the background.
-If the program uses some packages that are incompatible, please figure out alternative implementations and do NOT restart the environment.
-
-"""
-
-    runtime = create_runtime(config)
-    call_async_from_sync(runtime.connect)
-    initialize_runtime(runtime, instance)
-
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = asyncio.run(
-        run_controller(
-            config=config,
-            initial_user_action=MessageAction(content=instruction),
-            runtime=runtime,
-            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-                metadata.agent_class
-            ),
-        )
-    )
-
-    # ======= Attempt to evaluate the agent's edits =======
-    test_result = complete_runtime(runtime, instance)
-
-    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-    if state is None:
-        raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
-
-    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
-    # for compatibility with the existing output format, we can remake the pairs here
-    # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
-
-    # Save the output
-    output = EvalOutput(
-        instance_id=instance.instance_id,
-        instruction=instruction,
-        metadata=metadata,
-        history=histories,
-        metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
-        test_result=test_result,
-    )
-    return output
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--use_knowledge',
-        type=str,
-        default='false',
-        choices=['true', 'false'],
-        help='use expert-provided knowledge or not',
-    )
-    args, _ = parser.parse_known_args()
-
-    sab_dataset = load_dataset('osunlp/ScienceAgentBench', split='validation')
-
-    dataset_processed = []
-    for example in tqdm(sab_dataset):
-        dataset_processed.append(
-            format_task_dict(example, args.use_knowledge == 'true')
-        )
-
-    dataset = pd.DataFrame(dataset_processed)
-
-    llm_config = None
-    if args.llm_config:
-        llm_config = get_llm_config_arg(args.llm_config)
-    if llm_config is None:
-        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
-
-    metadata = make_metadata(
-        llm_config,
-        'ScienceAgentBench',
-        args.agent_cls,
-        args.max_iterations,
-        args.eval_note,
-        args.eval_output_dir,
-    )
-    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-    dataset['instance_id'] = dataset['instance_id'].apply(str)
-    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
-
-    run_evaluation(
-        instances, metadata, output_file, args.eval_num_workers, process_instance
-    )

evaluation/scienceagentbench/scripts/run_infer.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-source "evaluation/utils/version_control.sh"
-
-MODEL_CONFIG=$1
-COMMIT_HASH=$2
-USE_KNOWLEDGE=$3
-AGENT=$4
-EVAL_LIMIT=$5
-NUM_WORKERS=$6
-
-if [ -z "$NUM_WORKERS" ]; then
-  NUM_WORKERS=1
-  echo "Number of workers not specified, use default $NUM_WORKERS"
-fi
-checkout_eval_branch
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-if [ -z "$USE_KNOWLEDGE" ]; then
-  echo "Use knowledge not specified, use default False"
-  USE_KNOWLEDGE=false
-fi
-
-get_agent_version
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-
-COMMAND="poetry run python evaluation/scienceagentbench/run_infer.py \
-  --agent-cls $AGENT \
-  --llm-config $MODEL_CONFIG \
-  --use_knowledge $USE_KNOWLEDGE \
-  --max-iterations 30 \
-  --eval-num-workers $NUM_WORKERS \
-  --eval-note $AGENT_VERSION" \
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND

evaluation/swe_bench/eval_infer.py

@@ -239,7 +239,7 @@ def process_instance(
                         # Create a directory structure that matches the expected format
                         # NOTE: this is a hack to make the eval report format consistent
                         # with the original SWE-Bench eval script
-                        log_dir = os.path.join(temp_dir, 'logs', instance_id.lower())
+                        log_dir = os.path.join(temp_dir, 'logs', instance_id)
                         os.makedirs(log_dir, exist_ok=True)
                         test_output_path = os.path.join(log_dir, 'test_output.txt')
                         with open(test_output_path, 'w') as f:

evaluation/swe_bench/run_infer.py

@@ -101,7 +101,7 @@ def get_instance_docker_image(instance_id: str) -> str:
     image_name = image_name.replace(
         '__', '_s_'
     )  # to comply with docker image naming convention
-    return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name).lower()
+    return DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name
 
 
 def get_config(

frontend/__tests__/components/feedback-form.test.tsx

@@ -5,6 +5,7 @@ import { FeedbackForm } from "#/components/feedback-form";
 
 describe("FeedbackForm", () => {
   const user = userEvent.setup();
+  const onSubmitMock = vi.fn();
   const onCloseMock = vi.fn();
 
   afterEach(() => {
@@ -12,7 +13,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should render correctly", () => {
-    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
 
     screen.getByLabelText("Email");
     screen.getByLabelText("Private");
@@ -23,7 +24,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should switch between private and public permissions", async () => {
-    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
     const privateRadio = screen.getByLabelText("Private");
     const publicRadio = screen.getByLabelText("Public");
 
@@ -39,11 +40,69 @@ describe("FeedbackForm", () => {
     expect(publicRadio).not.toBeChecked();
   });
 
+  it("should call onSubmit when the form is submitted", async () => {
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    const email = screen.getByLabelText("Email");
+
+    await user.type(email, "test@test.test");
+    await user.click(screen.getByRole("button", { name: "Submit" }));
+
+    expect(onSubmitMock).toHaveBeenCalledWith("private", "test@test.test"); // private is the default value
+  });
+
+  it("should not call onSubmit when the email is invalid", async () => {
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    const email = screen.getByLabelText("Email");
+    const submitButton = screen.getByRole("button", { name: "Submit" });
+
+    await user.click(submitButton);
+
+    expect(onSubmitMock).not.toHaveBeenCalled();
+
+    await user.type(email, "test");
+    await user.click(submitButton);
+
+    expect(onSubmitMock).not.toHaveBeenCalled();
+  });
+
+  it("should submit public permissions when the public radio is checked", async () => {
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
+    const email = screen.getByLabelText("Email");
+    const publicRadio = screen.getByLabelText("Public");
+
+    await user.type(email, "test@test.test");
+    await user.click(publicRadio);
+    await user.click(screen.getByRole("button", { name: "Submit" }));
+
+    expect(onSubmitMock).toHaveBeenCalledWith("public", "test@test.test");
+  });
+
   it("should call onClose when the close button is clicked", async () => {
-    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
+    render(<FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />);
     await user.click(screen.getByRole("button", { name: "Cancel" }));
 
+    expect(onSubmitMock).not.toHaveBeenCalled();
     expect(onCloseMock).toHaveBeenCalled();
   });
 
+  it("should disable the buttons if isSubmitting is true", () => {
+    const { rerender } = render(
+      <FeedbackForm onSubmit={onSubmitMock} onClose={onCloseMock} />,
+    );
+    const submitButton = screen.getByRole("button", { name: "Submit" });
+    const cancelButton = screen.getByRole("button", { name: "Cancel" });
+
+    expect(submitButton).not.toBeDisabled();
+    expect(cancelButton).not.toBeDisabled();
+
+    rerender(
+      <FeedbackForm
+        onSubmit={onSubmitMock}
+        onClose={onCloseMock}
+        isSubmitting
+      />,
+    );
+    expect(submitButton).toBeDisabled();
+    expect(cancelButton).toBeDisabled();
+  });
 });

frontend/__tests__/components/file-explorer/FileExplorer.test.tsx

@@ -16,16 +16,13 @@ vi.mock("../../services/fileService", async () => ({
 }));
 
 const renderFileExplorerWithRunningAgentState = () =>
-  renderWithProviders(
-    <FileExplorer error={null} isOpen onToggle={() => {}} />,
-    {
-      preloadedState: {
-        agent: {
-          curAgentState: AgentState.RUNNING,
-        },
+  renderWithProviders(<FileExplorer error={null} />, {
+    preloadedState: {
+      agent: {
+        curAgentState: AgentState.RUNNING,
       },
     },
-  );
+  });
 
 describe.skip("FileExplorer", () => {
   afterEach(() => {

frontend/__tests__/utils/extractModelAndProvider.test.ts

@@ -59,9 +59,9 @@ describe("extractModelAndProvider", () => {
       separator: "/",
     });
 
-    expect(extractModelAndProvider("claude-3-5-sonnet-20241022")).toEqual({
+    expect(extractModelAndProvider("claude-3-5-sonnet-20240620")).toEqual({
       provider: "anthropic",
-      model: "claude-3-5-sonnet-20241022",
+      model: "claude-3-5-sonnet-20240620",
       separator: "/",
     });
 
@@ -78,4 +78,3 @@ describe("extractModelAndProvider", () => {
     });
   });
 });
-

frontend/__tests__/utils/organizeModelsAndProviders.test.ts

@@ -15,7 +15,7 @@ test("organizeModelsAndProviders", () => {
     "gpt-4o",
     "together-ai-21.1b-41b",
     "gpt-4o-mini",
-    "claude-3-5-sonnet-20241022",
+    "claude-3-5-sonnet-20240620",
     "claude-3-haiku-20240307",
     "claude-2",
     "claude-2.1",
@@ -51,7 +51,7 @@ test("organizeModelsAndProviders", () => {
     anthropic: {
       separator: "/",
       models: [
-        "claude-3-5-sonnet-20241022",
+        "claude-3-5-sonnet-20240620",
         "claude-3-haiku-20240307",
         "claude-2",
         "claude-2.1",
@@ -63,4 +63,3 @@ test("organizeModelsAndProviders", () => {
     },
   });
 });
-

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "openhands-frontend",
-  "version": "0.12.0",
+  "version": "0.11.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "openhands-frontend",
-      "version": "0.12.0",
+      "version": "0.11.0",
       "dependencies": {
         "@monaco-editor/react": "^4.6.0",
         "@nextui-org/react": "^2.4.8",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.12.0",
+  "version": "0.11.0",
   "private": true,
   "type": "module",
   "engines": {

frontend/src/api/open-hands.ts

@@ -1,4 +1,4 @@
-import { request } from "#/services/api";
+import { getValidFallbackHost } from "#/utils/get-valid-fallback-host";
 import {
   SaveFileSuccessResponse,
   FileUploadSuccessResponse,
@@ -9,13 +9,36 @@ import {
   GetConfigResponse,
 } from "./open-hands.types";
 
+/**
+ * Generate the base URL of the OpenHands API
+ * @returns Base URL of the OpenHands API
+ */
+const generateBaseURL = () => {
+  const fallback = getValidFallbackHost();
+  const baseUrl = import.meta.env.VITE_BACKEND_BASE_URL || fallback;
+
+  if (typeof window === "undefined") {
+    return `http://${baseUrl}`;
+  }
+  return `${window.location.protocol}//${baseUrl}`;
+};
+
+/**
+ * Class to interact with the OpenHands API
+ */
 class OpenHands {
+  /**
+   * Base URL of the OpenHands API
+   */
+  static BASE_URL = generateBaseURL();
+
   /**
    * Retrieve the list of models available
    * @returns List of models available
    */
   static async getModels(): Promise<string[]> {
-    return request("/api/options/models");
+    const response = await fetch(`${OpenHands.BASE_URL}/api/options/models`);
+    return response.json();
   }
 
   /**
@@ -23,7 +46,8 @@ class OpenHands {
    * @returns List of agents available
    */
   static async getAgents(): Promise<string[]> {
-    return request(`/api/options/agents`);
+    const response = await fetch(`${OpenHands.BASE_URL}/api/options/agents`);
+    return response.json();
   }
 
   /**
@@ -31,123 +55,178 @@ class OpenHands {
    * @returns List of security analyzers available
    */
   static async getSecurityAnalyzers(): Promise<string[]> {
-    return request(`/api/options/security-analyzers`);
+    const response = await fetch(
+      `${OpenHands.BASE_URL}/api/options/security-analyzers`,
+    );
+    return response.json();
   }
 
   static async getConfig(): Promise<GetConfigResponse> {
-    return request("/config.json");
+    const response = await fetch("config.json", {
+      headers: {
+        "Cache-Control": "no-cache",
+      },
+    });
+    return response.json();
   }
 
   /**
    * Retrieve the list of files available in the workspace
+   * @param token User token provided by the server
    * @param path Path to list files from
    * @returns List of files available in the given path. If path is not provided, it lists all the files in the workspace
    */
-  static async getFiles(path?: string): Promise<string[]> {
-    let url = "/api/list-files";
-    if (path) url += `?path=${encodeURIComponent(path)}`;
-    return request(url);
+  static async getFiles(token: string, path?: string): Promise<string[]> {
+    const url = new URL(`${OpenHands.BASE_URL}/api/list-files`);
+    if (path) url.searchParams.append("path", path);
+
+    const response = await fetch(url.toString(), {
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
+    });
+
+    return response.json();
   }
 
   /**
    * Retrieve the content of a file
+   * @param token User token provided by the server
    * @param path Full path of the file to retrieve
    * @returns Content of the file
    */
-  static async getFile(path: string): Promise<string> {
-    const url = `/api/select-file?file=${encodeURIComponent(path)}`;
-    const data = await request(url);
+  static async getFile(token: string, path: string): Promise<string> {
+    const url = new URL(`${OpenHands.BASE_URL}/api/select-file`);
+    url.searchParams.append("file", path);
+    const response = await fetch(url.toString(), {
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
+    });
+
+    const data = await response.json();
     return data.code;
   }
 
   /**
    * Save the content of a file
+   * @param token User token provided by the server
    * @param path Full path of the file to save
    * @param content Content to save in the file
    * @returns Success message or error message
    */
   static async saveFile(
+    token: string,
     path: string,
     content: string,
   ): Promise<SaveFileSuccessResponse | ErrorResponse> {
-    return request(`/api/save-file`, {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/save-file`, {
       method: "POST",
       body: JSON.stringify({ filePath: path, content }),
       headers: {
+        Authorization: `Bearer ${token}`,
         "Content-Type": "application/json",
       },
     });
+
+    return response.json();
   }
 
   /**
    * Upload a file to the workspace
+   * @param token User token provided by the server
    * @param file File to upload
    * @returns Success message or error message
    */
   static async uploadFiles(
+    token: string,
     file: File[],
   ): Promise<FileUploadSuccessResponse | ErrorResponse> {
     const formData = new FormData();
     file.forEach((f) => formData.append("files", f));
 
-    return request(`/api/upload-files`, {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/upload-files`, {
       method: "POST",
       body: formData,
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
     });
+
+    return response.json();
   }
 
   /**
    * Get the blob of the workspace zip
+   * @param token User token provided by the server
    * @returns Blob of the workspace zip
    */
-  static async getWorkspaceZip(): Promise<Blob> {
-    const response = await request(`/api/zip-directory`, {}, false, true);
+  static async getWorkspaceZip(token: string): Promise<Blob> {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/zip-directory`, {
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
+    });
+
     return response.blob();
   }
 
   /**
    * Send feedback to the server
+   * @param token User token provided by the server
    * @param data Feedback data
    * @returns The stored feedback data
    */
-  static async submitFeedback(data: Feedback): Promise<FeedbackResponse> {
-    return request(`/api/submit-feedback`, {
+  static async sendFeedback(
+    token: string,
+    data: Feedback,
+  ): Promise<FeedbackResponse> {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/submit-feedback`, {
       method: "POST",
       body: JSON.stringify(data),
       headers: {
+        Authorization: `Bearer ${token}`,
         "Content-Type": "application/json",
       },
     });
+
+    return response.json();
   }
 
   /**
+   * Get the GitHub access token
    * @param code Code provided by GitHub
    * @returns GitHub access token
    */
   static async getGitHubAccessToken(
     code: string,
   ): Promise<GitHubAccessTokenResponse> {
-    return request(`/api/github/callback`, {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/github/callback`, {
       method: "POST",
       body: JSON.stringify({ code }),
       headers: {
         "Content-Type": "application/json",
       },
     });
+
+    return response.json();
   }
 
   /**
-   * Authenticate with GitHub token
-   * @returns Response with authentication status and user info if successful
+   * Check if the user is authenticated
+   * @param login The user's GitHub login handle
+   * @returns Whether the user is authenticated
    */
-  static async authenticate(): Promise<Response> {
-    return request(
-      `/api/authenticate`,
-      {
-        method: "POST",
+  static async isAuthenticated(login: string): Promise<boolean> {
+    const response = await fetch(`${OpenHands.BASE_URL}/api/authenticate`, {
+      method: "POST",
+      body: JSON.stringify({ login }),
+      headers: {
+        "Content-Type": "application/json",
       },
-      true,
-    );
+    });
+
+    return response.status === 200;
   }
 }
 

frontend/src/api/open-hands.types.ts

@@ -27,16 +27,11 @@ export interface GitHubAccessTokenResponse {
   access_token: string;
 }
 
-export interface AuthenticationResponse {
-  message: string;
-  login?: string; // Only present when allow list is enabled
-}
-
 export interface Feedback {
   version: string;
   email: string;
   token: string;
-  polarity: "positive" | "negative";
+  feedback: "positive" | "negative";
   permissions: "public" | "private";
   trajectory: unknown[];
 }

frontend/src/components/chat-interface.tsx

@@ -1,5 +1,6 @@
 import { useDispatch, useSelector } from "react-redux";
 import React from "react";
+import { useFetcher } from "@remix-run/react";
 import { useSocket } from "#/context/socket";
 import { convertImageToBase64 } from "#/utils/convert-image-to-base-64";
 import { ChatMessage } from "./chat-message";
@@ -12,6 +13,10 @@ import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import { generateAgentStateChangeEvent } from "#/services/agentStateService";
 import { FeedbackModal } from "./feedback-modal";
+import { Feedback } from "#/api/open-hands.types";
+import { getToken } from "#/services/auth";
+import { removeApiKey, removeUnwantedKeys } from "#/utils/utils";
+import { clientAction } from "#/routes/submit-feedback";
 import { useScrollToBottom } from "#/hooks/useScrollToBottom";
 import TypingIndicator from "./chat/TypingIndicator";
 import ConfirmationButtons from "./chat/ConfirmationButtons";
@@ -19,13 +24,16 @@ import { ErrorMessage } from "./error-message";
 import { ContinueButton } from "./continue-button";
 import { ScrollToBottomButton } from "./scroll-to-bottom-button";
 
+const FEEDBACK_VERSION = "1.0";
+
 const isErrorMessage = (
   message: Message | ErrorMessage,
 ): message is ErrorMessage => "error" in message;
 
 export function ChatInterface() {
-  const { send } = useSocket();
+  const { send, events } = useSocket();
   const dispatch = useDispatch();
+  const fetcher = useFetcher<typeof clientAction>({ key: "feedback" });
   const scrollRef = React.useRef<HTMLDivElement>(null);
   const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
     useScrollToBottom(scrollRef);
@@ -36,6 +44,7 @@ export function ChatInterface() {
   const [feedbackPolarity, setFeedbackPolarity] = React.useState<
     "positive" | "negative"
   >("positive");
+  const [feedbackShared, setFeedbackShared] = React.useState(0);
   const [feedbackModalIsOpen, setFeedbackModalIsOpen] = React.useState(false);
 
   const handleSendMessage = async (content: string, files: File[]) => {
@@ -62,6 +71,30 @@ export function ChatInterface() {
     setFeedbackPolarity(polarity);
   };
 
+  const handleSubmitFeedback = (
+    permissions: "private" | "public",
+    email: string,
+  ) => {
+    const feedback: Feedback = {
+      version: FEEDBACK_VERSION,
+      feedback: feedbackPolarity,
+      email,
+      permissions,
+      token: getToken(),
+      trajectory: removeApiKey(removeUnwantedKeys(events)),
+    };
+
+    const formData = new FormData();
+    formData.append("feedback", JSON.stringify(feedback));
+
+    fetcher.submit(formData, {
+      action: "/submit-feedback",
+      method: "POST",
+    });
+
+    setFeedbackShared(messages.length);
+  };
+
   return (
     <div className="h-full flex flex-col justify-between">
       <div
@@ -97,14 +130,16 @@ export function ChatInterface() {
 
       <div className="flex flex-col gap-[6px] px-4 pb-4">
         <div className="flex justify-between relative">
-          <FeedbackActions
-            onPositiveFeedback={() =>
-              onClickShareFeedbackActionButton("positive")
-            }
-            onNegativeFeedback={() =>
-              onClickShareFeedbackActionButton("negative")
-            }
-          />
+          {feedbackShared !== messages.length && messages.length > 3 && (
+            <FeedbackActions
+              onPositiveFeedback={() =>
+                onClickShareFeedbackActionButton("positive")
+              }
+              onNegativeFeedback={() =>
+                onClickShareFeedbackActionButton("negative")
+              }
+            />
+          )}
           <div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
             {messages.length > 2 &&
               curAgentState === AgentState.AWAITING_USER_INPUT && (
@@ -128,8 +163,9 @@ export function ChatInterface() {
 
       <FeedbackModal
         isOpen={feedbackModalIsOpen}
+        isSubmitting={fetcher.state === "submitting"}
         onClose={() => setFeedbackModalIsOpen(false)}
-        polarity={feedbackPolarity}
+        onSubmit={handleSubmitFeedback}
       />
     </div>
   );

frontend/src/components/feedback-form.tsx

@@ -1,81 +1,27 @@
-import React from "react";
-import hotToast from "react-hot-toast";
 import ModalButton from "./buttons/ModalButton";
-import { Feedback } from "#/api/open-hands.types";
-import OpenHands from "#/api/open-hands";
-
-const FEEDBACK_VERSION = "1.0";
-const VIEWER_PAGE = "https://www.all-hands.dev/share";
 
 interface FeedbackFormProps {
+  onSubmit: (permissions: "private" | "public", email: string) => void;
   onClose: () => void;
-  polarity: "positive" | "negative";
+  isSubmitting?: boolean;
 }
 
-export function FeedbackForm({ onClose, polarity }: FeedbackFormProps) {
-  const [isSubmitting, setIsSubmitting] = React.useState(false);
-
-  const copiedToClipboardToast = () => {
-    hotToast("Password copied to clipboard", {
-      icon: "📋",
-      position: "bottom-right",
-    });
-  };
-
-  const onPressToast = (password: string) => {
-    navigator.clipboard.writeText(password);
-    copiedToClipboardToast();
-  };
-
-  const shareFeedbackToast = (
-    message: string,
-    link: string,
-    password: string,
-  ) => {
-    hotToast(
-      <div className="flex flex-col gap-1">
-        <span>{message}</span>
-        <a
-          data-testid="toast-share-url"
-          className="text-blue-500 underline"
-          onClick={() => onPressToast(password)}
-          href={link}
-          target="_blank"
-          rel="noreferrer"
-        >
-          Go to shared feedback
-        </a>
-        <span onClick={() => onPressToast(password)} className="cursor-pointer">
-          Password: {password} <span className="text-gray-500">(copy)</span>
-        </span>
-      </div>,
-      { duration: 10000 },
-    );
-  };
-
-  const handleSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
+export function FeedbackForm({
+  onSubmit,
+  onClose,
+  isSubmitting,
+}: FeedbackFormProps) {
+  const handleSubmit = (event: React.FormEvent<HTMLFormElement>) => {
     event?.preventDefault();
     const formData = new FormData(event.currentTarget);
-    setIsSubmitting(true);
 
-    const email = formData.get("email")?.toString() || "";
-    const permissions = (formData.get("permissions")?.toString() ||
-      "private") as "private" | "public";
+    const email = formData.get("email")?.toString();
+    const permissions = formData.get("permissions")?.toString() as
+      | "private"
+      | "public"
+      | undefined;
 
-    const feedback: Feedback = {
-      version: FEEDBACK_VERSION,
-      email,
-      polarity,
-      permissions,
-      trajectory: [],
-      token: "",
-    };
-
-    const response = await OpenHands.submitFeedback(feedback);
-    const { message, feedback_id, password } = response.body; // eslint-disable-line
-    const link = `${VIEWER_PAGE}?share_id=${feedback_id}`;
-    shareFeedbackToast(message, link, password);
-    setIsSubmitting(false);
+    if (email) onSubmit(permissions || "private", email);
   };
 
   return (

frontend/src/components/feedback-modal.tsx

@@ -1,4 +1,6 @@
 import React from "react";
+import hotToast, { toast } from "react-hot-toast";
+import { useFetcher } from "@remix-run/react";
 import { FeedbackForm } from "./feedback-form";
 import {
   BaseModalTitle,
@@ -6,18 +8,82 @@ import {
 } from "./modals/confirmation-modals/BaseModal";
 import { ModalBackdrop } from "./modals/modal-backdrop";
 import ModalBody from "./modals/ModalBody";
+import { clientAction } from "#/routes/submit-feedback";
 
 interface FeedbackModalProps {
+  onSubmit: (permissions: "private" | "public", email: string) => void;
   onClose: () => void;
   isOpen: boolean;
-  polarity: "positive" | "negative";
+  isSubmitting?: boolean;
 }
 
 export function FeedbackModal({
+  onSubmit,
   onClose,
   isOpen,
-  polarity,
+  isSubmitting,
 }: FeedbackModalProps) {
+  const fetcher = useFetcher<typeof clientAction>({ key: "feedback" });
+  const isInitialRender = React.useRef(true);
+
+  const copiedToClipboardToast = () => {
+    hotToast("Password copied to clipboard", {
+      icon: "📋",
+      position: "bottom-right",
+    });
+  };
+
+  const onPressToast = (password: string) => {
+    navigator.clipboard.writeText(password);
+    copiedToClipboardToast();
+  };
+
+  const shareFeedbackToast = (
+    message: string,
+    link: string,
+    password: string,
+  ) => {
+    hotToast(
+      <div className="flex flex-col gap-1">
+        <span>{message}</span>
+        <a
+          data-testid="toast-share-url"
+          className="text-blue-500 underline"
+          onClick={() => onPressToast(password)}
+          href={link}
+          target="_blank"
+          rel="noreferrer"
+        >
+          Go to shared feedback
+        </a>
+        <span onClick={() => onPressToast(password)} className="cursor-pointer">
+          Password: {password} <span className="text-gray-500">(copy)</span>
+        </span>
+      </div>,
+      { duration: 5000 },
+    );
+  };
+
+  React.useEffect(() => {
+    if (isInitialRender.current) {
+      isInitialRender.current = false;
+      return;
+    }
+
+    // Handle feedback submission
+    if (fetcher.state === "idle" && fetcher.data) {
+      if (!fetcher.data.success) {
+        toast.error("Error submitting feedback");
+      } else if (fetcher.data.data) {
+        const { data } = fetcher.data;
+        const { message, link, password } = data;
+        shareFeedbackToast(message, link, password);
+      }
+
+      onClose();
+    }
+  }, [fetcher.state, fetcher.data?.success]);
+
   if (!isOpen) return null;
 
   return (
@@ -25,7 +91,11 @@ export function FeedbackModal({
       <ModalBody>
         <BaseModalTitle title="Feedback" />
         <BaseModalDescription description="To help us improve, we collect feedback from your interactions to improve our prompts. By submitting this form, you consent to us collecting this data." />
-        <FeedbackForm onClose={onClose} polarity={polarity} />
+        <FeedbackForm
+          onSubmit={onSubmit}
+          onClose={onClose}
+          isSubmitting={isSubmitting}
+        />
       </ModalBody>
     </ModalBackdrop>
   );

frontend/src/components/file-explorer/FileExplorer.tsx

@@ -91,15 +91,14 @@ function ExplorerActions({
 }
 
 interface FileExplorerProps {
-  isOpen: boolean;
-  onToggle: () => void;
   error: string | null;
 }
 
-function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
+function FileExplorer({ error }: FileExplorerProps) {
   const { revalidate } = useRevalidator();
 
   const { paths, setPaths } = useFiles();
+  const [isHidden, setIsHidden] = React.useState(false);
   const [isDragging, setIsDragging] = React.useState(false);
 
   const { curAgentState } = useSelector((state: RootState) => state.agent);
@@ -118,47 +117,52 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
       return;
     }
     dispatch(setRefreshID(Math.random()));
-    OpenHands.getFiles().then(setPaths);
+    // TODO: Get token from data loader
+    const token = localStorage.getItem("token");
+    if (token) OpenHands.getFiles(token).then(setPaths);
     revalidate();
   };
 
   const uploadFileData = async (files: FileList) => {
     try {
-      const result = await OpenHands.uploadFiles(Array.from(files));
+      const token = localStorage.getItem("token");
+      if (token) {
+        const result = await OpenHands.uploadFiles(token, Array.from(files));
 
-      if (isOpenHandsErrorResponse(result)) {
-        // Handle error response
-        toast.error(
-          `upload-error-${new Date().getTime()}`,
-          result.error || t(I18nKey.EXPLORER$UPLOAD_ERROR_MESSAGE),
-        );
-        return;
+        if (isOpenHandsErrorResponse(result)) {
+          // Handle error response
+          toast.error(
+            `upload-error-${new Date().getTime()}`,
+            result.error || t(I18nKey.EXPLORER$UPLOAD_ERROR_MESSAGE),
+          );
+          return;
+        }
+
+        const uploadedCount = result.uploaded_files.length;
+        const skippedCount = result.skipped_files.length;
+
+        if (uploadedCount > 0) {
+          toast.success(
+            `upload-success-${new Date().getTime()}`,
+            t(I18nKey.EXPLORER$UPLOAD_SUCCESS_MESSAGE, {
+              count: uploadedCount,
+            }),
+          );
+        }
+
+        if (skippedCount > 0) {
+          const message = t(I18nKey.EXPLORER$UPLOAD_PARTIAL_SUCCESS_MESSAGE, {
+            count: skippedCount,
+          });
+          toast.info(message);
+        }
+
+        if (uploadedCount === 0 && skippedCount === 0) {
+          toast.info(t(I18nKey.EXPLORER$NO_FILES_UPLOADED_MESSAGE));
+        }
+
+        refreshWorkspace();
       }
-
-      const uploadedCount = result.uploaded_files.length;
-      const skippedCount = result.skipped_files.length;
-
-      if (uploadedCount > 0) {
-        toast.success(
-          `upload-success-${new Date().getTime()}`,
-          t(I18nKey.EXPLORER$UPLOAD_SUCCESS_MESSAGE, {
-            count: uploadedCount,
-          }),
-        );
-      }
-
-      if (skippedCount > 0) {
-        const message = t(I18nKey.EXPLORER$UPLOAD_PARTIAL_SUCCESS_MESSAGE, {
-          count: skippedCount,
-        });
-        toast.info(message);
-      }
-
-      if (uploadedCount === 0 && skippedCount === 0) {
-        toast.info(t(I18nKey.EXPLORER$NO_FILES_UPLOADED_MESSAGE));
-      }
-
-      refreshWorkspace();
     } catch (e) {
       // Handle unexpected errors (network issues, etc.)
       toast.error(
@@ -207,7 +211,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
       <div
         className={twMerge(
           "bg-neutral-800 h-full border-r-1 border-r-neutral-600 flex flex-col",
-          !isOpen ? "w-12" : "w-60",
+          isHidden ? "w-12" : "w-60",
         )}
       >
         <div className="flex flex-col relative h-full px-3 py-2">
@@ -215,17 +219,17 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
             <div
               className={twMerge(
                 "flex items-center",
-                !isOpen ? "justify-center" : "justify-between",
+                isHidden ? "justify-center" : "justify-between",
               )}
             >
-              {isOpen && (
+              {!isHidden && (
                 <div className="text-neutral-300 font-bold text-sm">
                   {t(I18nKey.EXPLORER$LABEL_WORKSPACE)}
                 </div>
               )}
               <ExplorerActions
-                isHidden={!isOpen}
-                toggleHidden={onToggle}
+                isHidden={isHidden}
+                toggleHidden={() => setIsHidden((prev) => !prev)}
                 onRefresh={refreshWorkspace}
                 onUpload={selectFileInput}
               />
@@ -233,7 +237,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
           </div>
           {!error && (
             <div className="overflow-auto flex-grow">
-              <div style={{ display: !isOpen ? "none" : "block" }}>
+              <div style={{ display: isHidden ? "none" : "block" }}>
                 <ExplorerTree files={paths} />
               </div>
             </div>

frontend/src/components/file-explorer/TreeNode.tsx

@@ -59,11 +59,14 @@ function TreeNode({ path, defaultOpen = false }: TreeNodeProps) {
       return;
     }
 
-    try {
-      const newChildren = await OpenHands.getFiles(path);
-      setChildren(newChildren);
-    } catch (error) {
-      toast.error("Failed to fetch files");
+    const token = localStorage.getItem("token");
+    if (token) {
+      try {
+        const newChildren = await OpenHands.getFiles(token, path);
+        setChildren(newChildren);
+      } catch (error) {
+        toast.error("Failed to fetch files");
+      }
     }
   };
 
@@ -74,13 +77,15 @@ function TreeNode({ path, defaultOpen = false }: TreeNodeProps) {
   }, [refreshID, isOpen]);
 
   const handleClick = async () => {
+    const token = localStorage.getItem("token");
+
     if (isDirectory) {
       setIsOpen((prev) => !prev);
-    } else {
+    } else if (token) {
       const code = modifiedFiles[path] || files[path];
 
       try {
-        const fetchedCode = await OpenHands.getFile(path);
+        const fetchedCode = await OpenHands.getFile(token, path);
         setSelectedPath(path);
         if (!code || fetchedCode !== files[path]) {
           setFileContent(path, fetchedCode);

frontend/src/components/modals/ModalBody.tsx

@@ -1,5 +1,5 @@
+import clsx from "clsx";
 import React from "react";
-import { cn } from "#/utils/utils";
 
 interface ModalBodyProps {
   testID?: string;
@@ -11,7 +11,7 @@ function ModalBody({ testID, children, className }: ModalBodyProps) {
   return (
     <div
       data-testid={testID}
-      className={cn(
+      className={clsx(
         "bg-root-primary flex flex-col gap-6 items-center w-[384px] p-6 rounded-xl",
         className,
       )}

frontend/src/components/waitlist-modal.tsx

@@ -1,69 +0,0 @@
-import ModalButton from "./buttons/ModalButton";
-import { ModalBackdrop } from "./modals/modal-backdrop";
-import GitHubLogo from "#/assets/branding/github-logo.svg?react";
-import AllHandsLogo from "#/assets/branding/all-hands-logo.svg?react";
-import ModalBody from "./modals/ModalBody";
-
-interface WaitlistModalProps {
-  ghToken: string | null;
-  githubAuthUrl: string | null;
-}
-
-export function WaitlistModal({ ghToken, githubAuthUrl }: WaitlistModalProps) {
-  return (
-    <ModalBackdrop>
-      <ModalBody>
-        <AllHandsLogo width={68} height={46} />
-        <div className="flex flex-col gap-2 w-full items-center text-center">
-          <h1 className="text-2xl font-bold">
-            {ghToken ? "Just a little longer!" : "Sign in with GitHub"}
-          </h1>
-          {!ghToken && (
-            <p>
-              or{" "}
-              <a
-                href="https://www.all-hands.dev/join-waitlist"
-                target="_blank"
-                rel="noreferrer noopener"
-                className="text-blue-500 hover:underline underline-offset-2"
-              >
-                join the waitlist
-              </a>{" "}
-              if you haven&apos;t already
-            </p>
-          )}
-          {ghToken && (
-            <p className="text-sm">
-              Thanks for your patience! We&apos;re accepting new members
-              progressively. If you haven&apos;t joined the waitlist yet,
-              now&apos;s the time!
-            </p>
-          )}
-        </div>
-
-        {!ghToken && (
-          <ModalButton
-            text="Connect to GitHub"
-            icon={<GitHubLogo width={20} height={20} />}
-            className="bg-[#791B80] w-full"
-            onClick={() => {
-              if (githubAuthUrl) {
-                window.location.href = githubAuthUrl;
-              }
-            }}
-          />
-        )}
-        {ghToken && (
-          <a
-            href="https://www.all-hands.dev/join-waitlist"
-            target="_blank"
-            rel="noreferrer"
-            className="rounded bg-[#FFE165] text-black text-sm font-bold py-[10px] w-full text-center hover:opacity-80"
-          >
-            Join Waitlist
-          </a>
-        )}
-      </ModalBody>
-    </ModalBackdrop>
-  );
-}

frontend/src/context/socket.tsx

@@ -1,6 +1,7 @@
 import React from "react";
 import { Data } from "ws";
 import EventLogger from "#/utils/event-logger";
+import { getValidFallbackHost } from "#/utils/get-valid-fallback-host";
 
 interface WebSocketClientOptions {
   token: string | null;
@@ -45,17 +46,12 @@ function SocketProvider({ children }: SocketProviderProps) {
       );
     }
 
-    const baseUrl =
-      import.meta.env.VITE_BACKEND_BASE_URL || window?.location.host;
+    const fallback = getValidFallbackHost();
+    const baseUrl = import.meta.env.VITE_BACKEND_BASE_URL || fallback;
     const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
-    const sessionToken = options?.token || "NO_JWT"; // not allowed to be empty or duplicated
-    const ghToken = localStorage.getItem("ghToken") || "NO_GITHUB";
-
-    const ws = new WebSocket(`${protocol}//${baseUrl}/ws`, [
-      "openhands",
-      sessionToken,
-      ghToken,
-    ]);
+    const ws = new WebSocket(
+      `${protocol}//${baseUrl}/ws${options?.token ? `?token=${options.token}` : ""}`,
+    );
 
     ws.addEventListener("open", (event) => {
       setIsConnected(true);

frontend/src/routes/_oh._index/route.tsx

@@ -23,7 +23,6 @@ import store from "#/store";
 import { setInitialQuery } from "#/state/initial-query-slice";
 import { clientLoader as rootClientLoader } from "#/routes/_oh";
 import OpenHands from "#/api/open-hands";
-import { generateGitHubAuthUrl } from "#/utils/generate-github-auth-url";
 
 interface GitHubAuthProps {
   onConnectToGitHub: () => void;
@@ -63,10 +62,10 @@ export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
     githubClientId = null;
   }
 
-  const ghToken = localStorage.getItem("ghToken");
   const token = localStorage.getItem("token");
   if (token) return redirect("/app");
 
+  const ghToken = localStorage.getItem("ghToken");
   let repositories: GitHubRepository[] = [];
   if (ghToken) {
     const data = await retrieveAllGitHubUserRepositories(ghToken);
@@ -76,9 +75,10 @@ export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
   }
 
   let githubAuthUrl: string | null = null;
-  if (isSaas && githubClientId) {
+  if (isSaas) {
     const requestUrl = new URL(request.url);
-    githubAuthUrl = generateGitHubAuthUrl(githubClientId, requestUrl);
+    const redirectUri = `${requestUrl.origin}/oauth/github/callback`;
+    githubAuthUrl = `https://github.com/login/oauth/authorize?client_id=${githubClientId}&redirect_uri=${encodeURIComponent(redirectUri)}&scope=repo,user,workflow`;
   }
 
   return json({ repositories, githubAuthUrl });

frontend/src/routes/_oh.app._index/code-editor-component.tsx

@@ -1,21 +1,18 @@
-import { Editor, EditorProps } from "@monaco-editor/react";
+import { Editor, Monaco } from "@monaco-editor/react";
 import React from "react";
 import { useTranslation } from "react-i18next";
 import { VscCode } from "react-icons/vsc";
+import { type editor } from "monaco-editor";
 import toast from "react-hot-toast";
 import { I18nKey } from "#/i18n/declaration";
 import { useFiles } from "#/context/files";
 import OpenHands from "#/api/open-hands";
 
 interface CodeEditorCompoonentProps {
-  onMount: EditorProps["onMount"];
   isReadOnly: boolean;
 }
 
-function CodeEditorCompoonent({
-  onMount,
-  isReadOnly,
-}: CodeEditorCompoonentProps) {
+function CodeEditorCompoonent({ isReadOnly }: CodeEditorCompoonentProps) {
   const { t } = useTranslation();
   const {
     files,
@@ -25,6 +22,22 @@ function CodeEditorCompoonent({
     saveFileContent: saveNewFileContent,
   } = useFiles();
 
+  const handleEditorDidMount = React.useCallback(
+    (editor: editor.IStandaloneCodeEditor, monaco: Monaco): void => {
+      monaco.editor.defineTheme("my-theme", {
+        base: "vs-dark",
+        inherit: true,
+        rules: [],
+        colors: {
+          "editor.background": "#171717",
+        },
+      });
+
+      monaco.editor.setTheme("my-theme");
+    },
+    [],
+  );
+
   const handleEditorChange = (value: string | undefined) => {
     if (selectedPath && value) modifyFileContent(selectedPath, value);
   };
@@ -36,7 +49,8 @@ function CodeEditorCompoonent({
 
         if (content) {
           try {
-            await OpenHands.saveFile(selectedPath, content);
+            const token = localStorage.getItem("token")?.toString();
+            if (token) await OpenHands.saveFile(token, selectedPath, content);
           } catch (error) {
             toast.error("Failed to save file");
           }
@@ -54,7 +68,7 @@ function CodeEditorCompoonent({
     return (
       <div
         data-testid="code-editor-empty-message"
-        className="flex flex-col h-full items-center justify-center text-neutral-400"
+        className="flex flex-col items-center text-neutral-400"
       >
         <VscCode size={100} />
         {t(I18nKey.CODE_EDITOR$EMPTY_MESSAGE)}
@@ -65,6 +79,7 @@ function CodeEditorCompoonent({
   return (
     <Editor
       data-testid="code-editor"
+      height="100%"
       path={selectedPath ?? undefined}
       defaultValue=""
       value={
@@ -72,7 +87,7 @@ function CodeEditorCompoonent({
           ? modifiedFiles[selectedPath] || files[selectedPath]
           : undefined
       }
-      onMount={onMount}
+      onMount={handleEditorDidMount}
       onChange={handleEditorChange}
       options={{ readOnly: isReadOnly }}
     />

frontend/src/routes/_oh.app._index/route.tsx

@@ -1,13 +1,12 @@
 import React from "react";
 import { useSelector } from "react-redux";
-import { json, useRouteError } from "@remix-run/react";
+import { json, useLoaderData, useRouteError } from "@remix-run/react";
 import toast from "react-hot-toast";
-import { editor } from "monaco-editor";
-import { EditorProps } from "@monaco-editor/react";
 import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import FileExplorer from "#/components/file-explorer/FileExplorer";
 import OpenHands from "#/api/open-hands";
+import { useSocket } from "#/context/socket";
 import CodeEditorCompoonent from "./code-editor-component";
 import { useFiles } from "#/context/files";
 import { EditorActions } from "#/components/editor-actions";
@@ -29,7 +28,8 @@ export function ErrorBoundary() {
 }
 
 function CodeEditor() {
-  const { curAgentState } = useSelector((state: RootState) => state.agent);
+  const { token } = useLoaderData<typeof clientLoader>();
+  const { runtimeActive } = useSocket();
   const {
     setPaths,
     selectedPath,
@@ -37,27 +37,6 @@ function CodeEditor() {
     saveFileContent: saveNewFileContent,
     discardChanges,
   } = useFiles();
-  const [fileExplorerIsOpen, setFileExplorerIsOpen] = React.useState(true);
-  const editorRef = React.useRef<editor.IStandaloneCodeEditor | null>(null);
-
-  const toggleFileExplorer = () => {
-    setFileExplorerIsOpen((prev) => !prev);
-    editorRef.current?.layout({ width: 0, height: 0 });
-  };
-
-  const handleEditorDidMount: EditorProps["onMount"] = (e, monaco) => {
-    editorRef.current = e;
-
-    monaco.editor.defineTheme("oh-dark", {
-      base: "vs-dark",
-      inherit: true,
-      rules: [],
-      colors: {
-        "editor.background": "#171717",
-      },
-    });
-    monaco.editor.setTheme("oh-dark");
-  };
 
   const [errors, setErrors] = React.useState<{ getFiles: string | null }>({
     getFiles: null,
@@ -68,14 +47,15 @@ function CodeEditor() {
   );
 
   React.useEffect(() => {
-    if (curAgentState === AgentState.INIT) {
-      OpenHands.getFiles()
+    // only retrieve files if connected to WS to prevent requesting before runtime is ready
+    if (runtimeActive && token) {
+      OpenHands.getFiles(token)
         .then(setPaths)
         .catch(() => {
           setErrors({ getFiles: "Failed to retrieve files" });
         });
     }
-  }, [curAgentState]);
+  }, [runtimeActive, token]);
 
   // Code editing is only allowed when the agent is paused, finished, or awaiting user input (server rules)
   const isEditingAllowed = React.useMemo(
@@ -89,9 +69,9 @@ function CodeEditor() {
   const handleSave = async () => {
     if (selectedPath) {
       const content = modifiedFiles[selectedPath];
-      if (content) {
+      if (content && token) {
         try {
-          await OpenHands.saveFile(selectedPath, content);
+          await OpenHands.saveFile(token, selectedPath, content);
           saveNewFileContent(selectedPath);
         } catch (error) {
           toast.error("Failed to save file");
@@ -105,13 +85,9 @@ function CodeEditor() {
   };
 
   return (
-    <div className="flex h-full bg-neutral-900 relative">
-      <FileExplorer
-        isOpen={fileExplorerIsOpen}
-        onToggle={toggleFileExplorer}
-        error={errors.getFiles}
-      />
-      <div className="w-full">
+    <div className="flex h-full w-full bg-neutral-900 relative">
+      <FileExplorer error={errors.getFiles} />
+      <div className="flex flex-col min-h-0 w-full">
         {selectedPath && (
           <div className="flex w-full items-center justify-between self-end p-2">
             <span className="text-sm text-neutral-500">{selectedPath}</span>
@@ -122,10 +98,9 @@ function CodeEditor() {
             />
           </div>
         )}
-        <CodeEditorCompoonent
-          onMount={handleEditorDidMount}
-          isReadOnly={!isEditingAllowed}
-        />
+        <div className="flex grow items-center justify-center">
+          <CodeEditorCompoonent isReadOnly={!isEditingAllowed} />
+        </div>
       </div>
     </div>
   );

frontend/src/routes/_oh.app.tsx

@@ -72,15 +72,11 @@ const isAgentStateChange = (
 
 export const clientLoader = async () => {
   const ghToken = localStorage.getItem("ghToken");
-  try {
-    const isAuthed = await userIsAuthenticated();
-    if (!isAuthed) {
-      clearSession();
-      return redirect("/");
-    }
-  } catch (error) {
+
+  const isAuthed = await userIsAuthenticated(ghToken);
+  if (!isAuthed) {
     clearSession();
-    return redirect("/");
+    return redirect("/waitlist");
   }
 
   const q = store.getState().initalQuery.initialQuery;
@@ -289,21 +285,21 @@ function App() {
 
   React.useEffect(() => {
     (async () => {
-      if (runtimeActive && importedProjectZip) {
+      if (runtimeActive && token && importedProjectZip) {
         // upload files action
         try {
           const blob = base64ToBlob(importedProjectZip);
           const file = new File([blob], "imported-project.zip", {
             type: blob.type,
           });
-          await OpenHands.uploadFiles([file]);
+          await OpenHands.uploadFiles(token, [file]);
           dispatch(setImportedProjectZip(null));
         } catch (error) {
           toast.error("Failed to upload project files.");
         }
       }
     })();
-  }, [runtimeActive, importedProjectZip]);
+  }, [runtimeActive, token, importedProjectZip]);
 
   const {
     isOpen: securityModalIsOpen,
@@ -314,7 +310,7 @@ function App() {
   return (
     <div className="flex flex-col h-full gap-3">
       <div className="flex h-full overflow-auto gap-3">
-        <Container className="w-[390px] max-h-full">
+        <Container className="w-[375px] max-h-full">
           <ChatInterface />
         </Container>
 

frontend/src/routes/_oh.tsx

@@ -8,9 +8,7 @@ import {
   useLoaderData,
   useFetcher,
   Outlet,
-  ClientLoaderFunctionArgs,
 } from "@remix-run/react";
-import { useDispatch } from "react-redux";
 import { retrieveGitHubUser, isGitHubErrorReponse } from "#/api/github";
 import OpenHands from "#/api/open-hands";
 import CogTooth from "#/assets/cog-tooth";
@@ -26,13 +24,8 @@ import { getSettings, settingsAreUpToDate } from "#/services/settings";
 import AllHandsLogo from "#/assets/branding/all-hands-logo.svg?react";
 import NewProjectIcon from "#/assets/new-project.svg?react";
 import DocsIcon from "#/assets/docs.svg?react";
-import { userIsAuthenticated } from "#/utils/user-is-authenticated";
-import { generateGitHubAuthUrl } from "#/utils/generate-github-auth-url";
-import { WaitlistModal } from "#/components/waitlist-modal";
-import { setCurrentAgentState } from "#/state/agentSlice";
-import AgentState from "#/types/AgentState";
 
-export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
+export const clientLoader = async () => {
   try {
     const config = await OpenHands.getConfig();
     window.__APP_MODE__ = config.APP_MODE;
@@ -45,23 +38,6 @@ export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
   let token = localStorage.getItem("token");
   const ghToken = localStorage.getItem("ghToken");
 
-  let isAuthed: boolean = false;
-  let githubAuthUrl: string | null = null;
-
-  try {
-    isAuthed = await userIsAuthenticated();
-    if (!isAuthed && window.__GITHUB_CLIENT_ID__) {
-      const requestUrl = new URL(request.url);
-      githubAuthUrl = generateGitHubAuthUrl(
-        window.__GITHUB_CLIENT_ID__,
-        requestUrl,
-      );
-    }
-  } catch (error) {
-    isAuthed = false;
-    githubAuthUrl = null;
-  }
-
   let user: GitHubUser | GitHubErrorReponse | null = null;
   if (ghToken) user = await retrieveGitHubUser(ghToken);
 
@@ -77,8 +53,6 @@ export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
   return defer({
     token,
     ghToken,
-    isAuthed,
-    githubAuthUrl,
     user,
     settingsIsUpdated,
     settings,
@@ -127,18 +101,10 @@ export default function MainApp() {
   const { stop, isConnected } = useSocket();
   const navigation = useNavigation();
   const location = useLocation();
-  const {
-    token,
-    ghToken,
-    user,
-    isAuthed,
-    githubAuthUrl,
-    settingsIsUpdated,
-    settings,
-  } = useLoaderData<typeof clientLoader>();
+  const { token, user, settingsIsUpdated, settings } =
+    useLoaderData<typeof clientLoader>();
   const logoutFetcher = useFetcher({ key: "logout" });
   const endSessionFetcher = useFetcher({ key: "end-session" });
-  const dispatch = useDispatch();
 
   const [accountSettingsModalOpen, setAccountSettingsModalOpen] =
     React.useState(false);
@@ -208,7 +174,6 @@ export default function MainApp() {
 
   const handleEndSession = () => {
     setStartNewProjectModalIsOpen(false);
-    dispatch(setCurrentAgentState(AgentState.LOADING));
     // call new session action and redirect to '/'
     endSessionFetcher.submit(new FormData(), {
       method: "POST",
@@ -226,7 +191,7 @@ export default function MainApp() {
               type="button"
               aria-label="All Hands Logo"
               onClick={() => {
-                if (location.pathname === "/app")
+                if (location.pathname !== "/")
                   setStartNewProjectModalIsOpen(true);
               }}
             >
@@ -274,65 +239,62 @@ export default function MainApp() {
       </aside>
       <div className="h-full w-full relative">
         <Outlet />
-      </div>
-
-      {isAuthed && (!settingsIsUpdated || settingsModalIsOpen) && (
-        <ModalBackdrop onClose={() => setSettingsModalIsOpen(false)}>
-          <div className="bg-root-primary w-[384px] p-6 rounded-xl flex flex-col gap-2">
-            {settingsFormError && (
-              <p className="text-danger text-xs">{settingsFormError}</p>
-            )}
-            <span className="text-xl leading-6 font-semibold -tracking-[0.01em">
-              AI Provider Configuration
-            </span>
-            <p className="text-xs text-[#A3A3A3]">
-              To continue, connect an OpenAI, Anthropic, or other LLM account
-            </p>
-            {isConnected && (
-              <p className="text-xs text-danger">
-                Changing settings during an active session will end the session
+        {(!settingsIsUpdated || settingsModalIsOpen) && (
+          <ModalBackdrop onClose={() => setSettingsModalIsOpen(false)}>
+            <div className="bg-root-primary w-[384px] p-6 rounded-xl flex flex-col gap-2">
+              {settingsFormError && (
+                <p className="text-danger text-xs">{settingsFormError}</p>
+              )}
+              <span className="text-xl leading-6 font-semibold -tracking-[0.01em">
+                AI Provider Configuration
+              </span>
+              <p className="text-xs text-[#A3A3A3]">
+                To continue, connect an OpenAI, Anthropic, or other LLM account
               </p>
-            )}
-            <SettingsForm
-              settings={settings}
-              models={settingsFormData.models}
-              agents={settingsFormData.agents}
-              securityAnalyzers={settingsFormData.securityAnalyzers}
-              onClose={() => setSettingsModalIsOpen(false)}
+              {isConnected && (
+                <p className="text-xs text-danger">
+                  Changing settings during an active session will end the
+                  session
+                </p>
+              )}
+              <SettingsForm
+                settings={settings}
+                models={settingsFormData.models}
+                agents={settingsFormData.agents}
+                securityAnalyzers={settingsFormData.securityAnalyzers}
+                onClose={() => setSettingsModalIsOpen(false)}
+              />
+            </div>
+          </ModalBackdrop>
+        )}
+        {accountSettingsModalOpen && (
+          <ModalBackdrop onClose={handleAccountSettingsModalClose}>
+            <AccountSettingsModal
+              onClose={handleAccountSettingsModalClose}
+              selectedLanguage={settings.LANGUAGE}
+              gitHubError={isGitHubErrorReponse(user)}
             />
-          </div>
-        </ModalBackdrop>
-      )}
-      {accountSettingsModalOpen && (
-        <ModalBackdrop onClose={handleAccountSettingsModalClose}>
-          <AccountSettingsModal
-            onClose={handleAccountSettingsModalClose}
-            selectedLanguage={settings.LANGUAGE}
-            gitHubError={isGitHubErrorReponse(user)}
-          />
-        </ModalBackdrop>
-      )}
-      {startNewProjectModalIsOpen && (
-        <ModalBackdrop onClose={() => setStartNewProjectModalIsOpen(false)}>
-          <DangerModal
-            title="Are you sure you want to exit?"
-            description="You will lose any unsaved information."
-            buttons={{
-              danger: {
-                text: "Exit Project",
-                onClick: handleEndSession,
-              },
-              cancel: {
-                text: "Cancel",
-                onClick: () => setStartNewProjectModalIsOpen(false),
-              },
-            }}
-          />
-        </ModalBackdrop>
-      )}
-      {!isAuthed && (
-        <WaitlistModal ghToken={ghToken} githubAuthUrl={githubAuthUrl} />
-      )}
+          </ModalBackdrop>
+        )}
+        {startNewProjectModalIsOpen && (
+          <ModalBackdrop onClose={() => setStartNewProjectModalIsOpen(false)}>
+            <DangerModal
+              title="Are you sure you want to exit?"
+              description="You will lose any unsaved information."
+              buttons={{
+                danger: {
+                  text: "Exit Project",
+                  onClick: handleEndSession,
+                },
+                cancel: {
+                  text: "Cancel",
+                  onClick: () => setStartNewProjectModalIsOpen(false),
+                },
+              }}
+            />
+          </ModalBackdrop>
+        )}
+      </div>
     </div>
   );
 }

frontend/src/routes/_oh.waitlist.tsx

@@ -0,0 +1,39 @@
+import { Link } from "@remix-run/react";
+import Clipboard from "#/assets/clipboard.svg?react";
+
+function Waitlist() {
+  return (
+    <div className="bg-neutral-800 h-full flex items-center justify-center rounded-xl">
+      <div className="w-[384px] flex flex-col gap-6 bg-neutral-900 rounded-xl p-6">
+        <Clipboard className="w-14 self-center" />
+
+        <div className="flex flex-col gap-2">
+          <h1 className="text-[20px] leading-6 -tracking-[0.01em] font-semibold">
+            You&apos;re not in the waitlist yet!
+          </h1>
+          <p className="text-neutral-400 text-xs">
+            Please click{" "}
+            <a
+              href="https://www.all-hands.dev/join-waitlist"
+              target="_blank"
+              rel="noreferrer noopener"
+              className="text-blue-500"
+            >
+              here
+            </a>{" "}
+            to join the waitlist.
+          </p>
+        </div>
+
+        <Link
+          to="/"
+          className="text-white text-sm py-[10px] bg-neutral-500 rounded text-center"
+        >
+          Go back to home
+        </Link>
+      </div>
+    </div>
+  );
+}
+
+export default Waitlist;

frontend/src/routes/oauth.github.callback.tsx

@@ -11,11 +11,11 @@ export const clientLoader = async ({ request }: ClientLoaderFunctionArgs) => {
   const code = url.searchParams.get("code");
 
   if (code) {
+    // request to the server to exchange the code for a token
     const { access_token: accessToken } =
       await OpenHands.getGitHubAccessToken(code);
-
+    // set the token in local storage
     localStorage.setItem("ghToken", accessToken);
-
     return redirect("/");
   }
 

frontend/src/routes/submit-feedback.ts

@@ -0,0 +1,47 @@
+import { ClientActionFunctionArgs, json } from "@remix-run/react";
+import { Feedback } from "#/api/open-hands.types";
+import OpenHands from "#/api/open-hands";
+
+const VIEWER_PAGE = "https://www.all-hands.dev/share";
+
+const isFeedback = (feedback: unknown): feedback is Feedback => {
+  if (typeof feedback !== "object" || feedback === null) {
+    return false;
+  }
+
+  return (
+    "version" in feedback &&
+    "email" in feedback &&
+    "token" in feedback &&
+    "feedback" in feedback &&
+    "permissions" in feedback &&
+    "trajectory" in feedback
+  );
+};
+
+export const clientAction = async ({ request }: ClientActionFunctionArgs) => {
+  const formData = await request.formData();
+  const feedback = formData.get("feedback")?.toString();
+  const token = localStorage.getItem("token");
+
+  if (token && feedback) {
+    const parsed = JSON.parse(feedback);
+    if (isFeedback(parsed)) {
+      try {
+        const response = await OpenHands.sendFeedback(token, parsed);
+        if (response.statusCode === 200) {
+          const { message, feedback_id: feedbackId, password } = response.body;
+          const link = `${VIEWER_PAGE}?share_id=${feedbackId}`;
+          return json({
+            success: true,
+            data: { message, link, password },
+          });
+        }
+      } catch (error) {
+        return json({ success: false, data: null });
+      }
+    }
+  }
+
+  return json({ success: false, data: null });
+};

frontend/src/services/api.ts

@@ -1,26 +1,14 @@
-import { getToken, getGitHubToken } from "./auth";
+import { getToken } from "./auth";
 import toast from "#/utils/toast";
 
 const WAIT_FOR_AUTH_DELAY_MS = 500;
 
-const UNAUTHED_ROUTE_PREFIXES = [
-  "/api/authenticate",
-  "/api/options/",
-  "/config.json",
-  "/api/github/callback",
-];
-
 export async function request(
   url: string,
   options: RequestInit = {},
   disableToast: boolean = false,
-  returnResponse: boolean = false,
-  maxRetries: number = 3,
   /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
 ): Promise<any> {
-  if (maxRetries < 0) {
-    throw new Error("Max retries exceeded");
-  }
   const onFail = (msg: string) => {
     if (!disableToast) {
       toast.error("api", msg);
@@ -28,17 +16,12 @@ export async function request(
     throw new Error(msg);
   };
 
-  const needsAuth = !UNAUTHED_ROUTE_PREFIXES.some((prefix) =>
-    url.startsWith(prefix),
-  );
+  const needsAuth = !url.startsWith("/api/options/");
   const token = getToken();
-  const githubToken = getGitHubToken();
   if (!token && needsAuth) {
     return new Promise((resolve) => {
       setTimeout(() => {
-        resolve(
-          request(url, options, disableToast, returnResponse, maxRetries - 1),
-        );
+        resolve(request(url, options, disableToast));
       }, WAIT_FOR_AUTH_DELAY_MS);
     });
   }
@@ -49,13 +32,6 @@ export async function request(
       Authorization: `Bearer ${token}`,
     };
   }
-  if (githubToken) {
-    // eslint-disable-next-line no-param-reassign
-    options.headers = {
-      ...(options.headers || {}),
-      "X-GitHub-Token": githubToken,
-    };
-  }
 
   let response = null;
   try {
@@ -72,10 +48,6 @@ export async function request(
     onFail(`Error fetching ${url}: ${response?.statusText}`);
   }
 
-  if (returnResponse) {
-    return response;
-  }
-
   try {
     return await (response && response.json());
   } catch (e) {

frontend/src/services/auth.ts

@@ -1,5 +1,4 @@
 const TOKEN_KEY = "token";
-const GITHUB_TOKEN_KEY = "ghToken";
 
 const getToken = (): string => localStorage.getItem(TOKEN_KEY) ?? "";
 
@@ -11,22 +10,4 @@ const setToken = (token: string): void => {
   localStorage.setItem(TOKEN_KEY, token);
 };
 
-const getGitHubToken = (): string =>
-  localStorage.getItem(GITHUB_TOKEN_KEY) ?? "";
-
-const setGitHubToken = (token: string): void => {
-  localStorage.setItem(GITHUB_TOKEN_KEY, token);
-};
-
-const clearGitHubToken = (): void => {
-  localStorage.removeItem(GITHUB_TOKEN_KEY);
-};
-
-export {
-  getToken,
-  setToken,
-  clearToken,
-  getGitHubToken,
-  setGitHubToken,
-  clearGitHubToken,
-};
+export { getToken, setToken, clearToken };

frontend/src/utils/download-workspace.ts

@@ -4,7 +4,12 @@ import OpenHands from "#/api/open-hands";
  * Downloads the current workspace as a .zip file.
  */
 export const downloadWorkspace = async () => {
-  const blob = await OpenHands.getWorkspaceZip();
+  const token = localStorage.getItem("token");
+  if (!token) {
+    throw new Error("No token found");
+  }
+
+  const blob = await OpenHands.getWorkspaceZip(token);
 
   const url = URL.createObjectURL(blob);
   const link = document.createElement("a");

frontend/src/utils/generate-github-auth-url.ts

@@ -1,10 +0,0 @@
-/**
- * Generates a URL to redirect to for GitHub OAuth
- * @param clientId The GitHub OAuth client ID
- * @param requestUrl The URL of the request
- * @returns The URL to redirect to for GitHub OAuth
- */
-export const generateGitHubAuthUrl = (clientId: string, requestUrl: URL) => {
-  const redirectUri = `${requestUrl.origin}/oauth/github/callback`;
-  return `https://github.com/login/oauth/authorize?client_id=${clientId}&redirect_uri=${encodeURIComponent(redirectUri)}&scope=repo,user,workflow`;
-};

frontend/src/utils/get-valid-fallback-host.ts

@@ -0,0 +1,19 @@
+/**
+ * Get the valid fallback host. Returns the host unless it is localhost, in which case it returns localhost:3000
+ * @returns Valid fallback host
+ *
+ * @example
+ * // If the host is localhost (e.g., localhost:5173), it returns localhost:3000
+ * const host = getValidFallbackHost(); // localhost:3000
+ *
+ * // If the host is not localhost, it returns the host
+ * const host = getValidFallbackHost(); // sub.example.com
+ */
+export const getValidFallbackHost = () => {
+  if (typeof window !== "undefined") {
+    return window.location.host;
+  }
+
+  // Fallback is localhost:3000 because that is the default port for the server
+  return "localhost:3000";
+};

frontend/src/utils/user-is-authenticated.ts

@@ -1,10 +1,16 @@
+import { retrieveGitHubUser, isGitHubErrorReponse } from "#/api/github";
 import OpenHands from "#/api/open-hands";
 
-export const userIsAuthenticated = async () => {
-  try {
-    await OpenHands.authenticate();
-    return true;
-  } catch (error) {
-    return false;
+export const userIsAuthenticated = async (ghToken: string | null) => {
+  if (window.__APP_MODE__ !== "saas") return true;
+
+  let user: GitHubUser | GitHubErrorReponse | null = null;
+  if (ghToken) user = await retrieveGitHubUser(ghToken);
+
+  if (user && !isGitHubErrorReponse(user)) {
+    const isAuthed = await OpenHands.isAuthenticated(user.login);
+    return isAuthed;
   }
+
+  return false;
 };

frontend/src/utils/utils.ts

@@ -37,29 +37,21 @@ export const removeUnwantedKeys = (
     "focused_element_bid",
   ];
 
-  return data
-    .filter((item) => {
-      // Skip items that have a status key
-      if ("status" in item) {
-        return false;
-      }
-      return true;
-    })
-    .map((item) => {
-      // Create a shallow copy of item
-      const newItem = { ...item };
+  return data.map((item) => {
+    // Create a shallow copy of item
+    const newItem = { ...item };
 
-      // Check if extras exists and delete it from a new extras object
-      if (newItem.extras) {
-        const newExtras = { ...newItem.extras };
-        UNDESIRED_KEYS.forEach((key) => {
-          delete newExtras[key as keyof typeof newExtras];
-        });
-        newItem.extras = newExtras;
-      }
+    // Check if extras exists and delete it from a new extras object
+    if (newItem.extras) {
+      const newExtras = { ...newItem.extras };
+      UNDESIRED_KEYS.forEach((key) => {
+        delete newExtras[key as keyof typeof newExtras];
+      });
+      newItem.extras = newExtras;
+    }
 
-      return newItem;
-    });
+    return newItem;
+  });
 };
 
 export const removeApiKey = (

frontend/src/utils/verified-models.ts

@@ -1,10 +1,6 @@
 // Here are the list of verified models and providers that we know work well with OpenHands.
 export const VERIFIED_PROVIDERS = ["openai", "azure", "anthropic"];
-export const VERIFIED_MODELS = [
-  "gpt-4o",
-  "claude-3-5-sonnet-20240620",
-  "claude-3-5-sonnet-20241022",
-];
+export const VERIFIED_MODELS = ["gpt-4o", "claude-3-5-sonnet-20240620"];
 
 // LiteLLM does not return OpenAI models with the provider, so we list them here to set them ourselves for consistency
 // (e.g., they return `gpt-4o` instead of `openai/gpt-4o`)
@@ -19,7 +15,7 @@ export const VERIFIED_OPENAI_MODELS = [
 ];
 
 // LiteLLM does not return the compatible Anthropic models with the provider, so we list them here to set them ourselves
-// (e.g., they return `claude-3-5-sonnet-20241022` instead of `anthropic/claude-3-5-sonnet-20241022`)
+// (e.g., they return `claude-3-5-sonnet-20240620` instead of `anthropic/claude-3-5-sonnet-20240620`)
 export const VERIFIED_ANTHROPIC_MODELS = [
   "claude-2",
   "claude-2.1",

openhands/agenthub/codeact_agent/codeact_agent.py

@@ -93,23 +93,24 @@ class CodeActAgent(Agent):
             if config.micro_agent_name
             else None
         )
-
-        self.function_calling_active = self.config.function_calling
-        if self.function_calling_active and not self.llm.is_function_calling_active():
+        if (
+            self.config.function_calling
+            and not self.llm.config.supports_function_calling
+        ):
             logger.warning(
                 f'Function calling not supported for model {self.llm.config.model}. '
                 'Disabling function calling.'
             )
-            self.function_calling_active = False
+            self.config.function_calling = False
 
-        if self.function_calling_active:
+        if self.config.function_calling:
             # Function calling mode
             self.tools = codeact_function_calling.get_tools(
                 codeact_enable_browsing_delegate=self.config.codeact_enable_browsing_delegate,
                 codeact_enable_jupyter=self.config.codeact_enable_jupyter,
                 codeact_enable_llm_editor=self.config.codeact_enable_llm_editor,
             )
-            logger.debug(
+            logger.info(
                 f'TOOLS loaded for CodeActAgent: {json.dumps(self.tools, indent=2)}'
             )
             self.system_prompt = codeact_function_calling.SYSTEM_PROMPT
@@ -171,7 +172,7 @@ class CodeActAgent(Agent):
                 FileEditAction,
             ),
         ) or (isinstance(action, AgentFinishAction) and action.source == 'agent'):
-            if self.function_calling_active:
+            if self.config.function_calling:
                 tool_metadata = action.tool_call_metadata
                 assert tool_metadata is not None, (
                     'Tool call metadata should NOT be None when function calling is enabled. Action: '
@@ -185,7 +186,7 @@ class CodeActAgent(Agent):
                 pending_tool_call_action_messages[llm_response.id] = Message(
                     role=assistant_msg.role,
                     # tool call content SHOULD BE a string
-                    content=[TextContent(text=assistant_msg.content or '')]
+                    content=[TextContent(text=assistant_msg.content)]
                     if assistant_msg.content is not None
                     else [],
                     tool_calls=assistant_msg.tool_calls,
@@ -201,7 +202,7 @@ class CodeActAgent(Agent):
                 ]
         elif isinstance(action, MessageAction):
             role = 'user' if action.source == 'user' else 'assistant'
-            content = [TextContent(text=action.content or '')]
+            content = [TextContent(text=action.content)]
             if self.llm.vision_is_active() and action.images_urls:
                 content.append(ImageContent(image_urls=action.images_urls))
             return [
@@ -285,7 +286,7 @@ class CodeActAgent(Agent):
             # when the LLM tries to return the next message
             raise ValueError(f'Unknown observation type: {type(obs)}')
 
-        if self.function_calling_active:
+        if self.config.function_calling:
             # Update the message as tool response properly
             if (tool_call_metadata := obs.tool_call_metadata) is not None:
                 tool_call_id_to_message[tool_call_metadata.tool_call_id] = Message(
@@ -333,7 +334,7 @@ class CodeActAgent(Agent):
         params: dict = {
             'messages': self.llm.format_messages_for_llm(messages),
         }
-        if self.function_calling_active:
+        if self.config.function_calling:
             params['tools'] = self.tools
         else:
             params['stop'] = [
@@ -344,7 +345,7 @@ class CodeActAgent(Agent):
             ]
         response = self.llm.completion(**params)
 
-        if self.function_calling_active:
+        if self.config.function_calling:
             actions = codeact_function_calling.response_to_actions(response)
             for action in actions:
                 self.pending_actions.append(action)
@@ -478,7 +479,7 @@ class CodeActAgent(Agent):
                     else:
                         break
 
-        if not self.function_calling_active:
+        if not self.config.function_calling:
             # The latest user message is important:
             # we want to remind the agent of the environment constraints
             latest_user_message = next(

openhands/controller/agent_controller.py

@@ -156,7 +156,7 @@ class AgentController:
         if exception is not None and isinstance(exception, litellm.AuthenticationError):
             detail = 'Please check your credentials. Is your API key correct?'
         self.event_stream.add_event(
-            ErrorObservation(f'{message}:{detail}'), EventSource.ENVIRONMENT
+            ErrorObservation(f'{message}:{detail}'), EventSource.USER
         )
 
     async def start_step_loop(self):
@@ -346,8 +346,7 @@ class AgentController:
 
         self.state.agent_state = new_state
         self.event_stream.add_event(
-            AgentStateChangedObservation('', self.state.agent_state),
-            EventSource.ENVIRONMENT,
+            AgentStateChangedObservation('', self.state.agent_state), EventSource.AGENT
         )
 
         if new_state == AgentState.INIT and self.state.resume_state:
@@ -424,8 +423,7 @@ class AgentController:
         if self._is_stuck():
             # This need to go BEFORE report_error to sync metrics
             self.event_stream.add_event(
-                FatalErrorObservation('Agent got stuck in a loop'),
-                EventSource.ENVIRONMENT,
+                FatalErrorObservation('Agent got stuck in a loop'), EventSource.USER
             )
             return
 

openhands/core/cli.py

@@ -61,7 +61,7 @@ def display_event(event: Event):
         if hasattr(event, 'thought'):
             display_message(event.thought)
     if isinstance(event, MessageAction):
-        if event.source == EventSource.AGENT:
+        if event.source != EventSource.USER:
             display_message(event.content)
     if isinstance(event, CmdRunAction):
         display_command(event.command)
@@ -131,7 +131,7 @@ async def main():
         next_message = input('How can I help? >> ')
         if next_message == 'exit':
             event_stream.add_event(
-                ChangeAgentStateAction(AgentState.STOPPED), EventSource.ENVIRONMENT
+                ChangeAgentStateAction(AgentState.STOPPED), EventSource.USER
             )
             return
         action = MessageAction(content=next_message)

openhands/core/config/__init__.py

@@ -3,6 +3,7 @@ from openhands.core.config.app_config import AppConfig
 from openhands.core.config.config_utils import (
     OH_DEFAULT_AGENT,
     OH_MAX_ITERATIONS,
+    UndefinedString,
     get_field_info,
 )
 from openhands.core.config.llm_config import LLMConfig
@@ -21,6 +22,7 @@ from openhands.core.config.utils import (
 __all__ = [
     'OH_DEFAULT_AGENT',
     'OH_MAX_ITERATIONS',
+    'UndefinedString',
     'AgentConfig',
     'AppConfig',
     'LLMConfig',

openhands/core/config/agent_config.py

@@ -19,9 +19,9 @@ class AgentConfig:
     """
 
     function_calling: bool = True
-    codeact_enable_browsing_delegate: bool = True
+    codeact_enable_browsing_delegate: bool = False
     codeact_enable_llm_editor: bool = False
-    codeact_enable_jupyter: bool = True
+    codeact_enable_jupyter: bool = False
     micro_agent_name: str | None = None
     memory_enabled: bool = False
     memory_max_threads: int = 3

openhands/core/config/app_config.py

@@ -1,3 +1,4 @@
+import os
 import uuid
 from dataclasses import dataclass, field, fields, is_dataclass
 from typing import ClassVar
@@ -7,6 +8,7 @@ from openhands.core.config.agent_config import AgentConfig
 from openhands.core.config.config_utils import (
     OH_DEFAULT_AGENT,
     OH_MAX_ITERATIONS,
+    UndefinedString,
     get_field_info,
 )
 from openhands.core.config.llm_config import LLMConfig
@@ -53,8 +55,11 @@ class AppConfig:
     file_store: str = 'memory'
     file_store_path: str = '/tmp/file_store'
     trajectories_path: str | None = None
-    workspace_base: str | None = None
-    workspace_mount_path: str | None = None
+    # TODO: clean up workspace path after the removal of ServerRuntime
+    workspace_base: str = os.path.join(os.getcwd(), 'workspace')
+    workspace_mount_path: str | None = (
+        UndefinedString.UNDEFINED  # this path should always be set when config is fully loaded
+    )  # when set to None, do not mount the workspace
     workspace_mount_path_in_sandbox: str = '/workspace'
     workspace_mount_rewrite: str | None = None
     cache_dir: str = '/tmp/cache'

openhands/core/config/config_utils.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from types import UnionType
 from typing import get_args, get_origin
 
@@ -5,6 +6,10 @@ OH_DEFAULT_AGENT = 'CodeActAgent'
 OH_MAX_ITERATIONS = 100
 
 
+class UndefinedString(str, Enum):
+    UNDEFINED = 'UNDEFINED'
+
+
 def get_field_info(f):
     """Extract information about a dataclass field: type, optional, and default.
 

openhands/core/config/llm_config.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, fields
 from typing import Optional
 
 from openhands.core.config.config_utils import get_field_info
-from openhands.core.logger import LOG_DIR
 
 LLM_SENSITIVE_FIELDS = ['api_key', 'aws_access_key_id', 'aws_secret_access_key']
 
@@ -43,6 +42,7 @@ class LLMConfig:
         log_completions: Whether to log LLM completions to the state.
         log_completions_folder: The folder to log LLM completions to. Required if log_completions is True.
         draft_editor: A more efficient LLM to use for file editing. Introduced in [PR 3985](https://github.com/All-Hands-AI/OpenHands/pull/3985).
+        supports_function_calling: Whether the model supports function calling.
     """
 
     model: str = 'claude-3-5-sonnet-20241022'
@@ -75,8 +75,9 @@ class LLMConfig:
     disable_vision: bool | None = None
     caching_prompt: bool = True
     log_completions: bool = False
-    log_completions_folder: str = os.path.join(LOG_DIR, 'completions')
+    log_completions_folder: str | None = None
     draft_editor: Optional['LLMConfig'] = None
+    supports_function_calling: bool = False
 
     def defaults_to_dict(self) -> dict:
         """Serialize fields to a dict for the frontend, including type hints, defaults, and whether it's optional."""

openhands/core/config/utils.py

@@ -15,6 +15,7 @@ from openhands.core.config.app_config import AppConfig
 from openhands.core.config.config_utils import (
     OH_DEFAULT_AGENT,
     OH_MAX_ITERATIONS,
+    UndefinedString,
 )
 from openhands.core.config.llm_config import LLMConfig
 from openhands.core.config.sandbox_config import SandboxConfig
@@ -190,19 +191,18 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
 
 def finalize_config(cfg: AppConfig):
     """More tweaks to the config after it's been loaded."""
-    if cfg.workspace_base is not None:
-        cfg.workspace_base = os.path.abspath(cfg.workspace_base)
-        if cfg.workspace_mount_path is None:
-            cfg.workspace_mount_path = cfg.workspace_base
+    cfg.workspace_base = os.path.abspath(cfg.workspace_base)
+    # Set workspace_mount_path if not set by the user
+    if cfg.workspace_mount_path is UndefinedString.UNDEFINED:
+        cfg.workspace_mount_path = cfg.workspace_base
 
-        if cfg.workspace_mount_rewrite:
-            base = cfg.workspace_base or os.getcwd()
-            parts = cfg.workspace_mount_rewrite.split(':')
-            cfg.workspace_mount_path = base.replace(parts[0], parts[1])
+    if cfg.workspace_mount_rewrite:  # and not config.workspace_mount_path:
+        # TODO why do we need to check if workspace_mount_path is None?
+        base = cfg.workspace_base or os.getcwd()
+        parts = cfg.workspace_mount_rewrite.split(':')
+        cfg.workspace_mount_path = base.replace(parts[0], parts[1])
 
-    # make sure log_completions_folder is an absolute path
     for llm in cfg.llms.values():
-        llm.log_completions_folder = os.path.abspath(llm.log_completions_folder)
         if llm.embedding_base_url is None:
             llm.embedding_base_url = llm.base_url
 

openhands/core/logger.py

@@ -12,6 +12,9 @@ LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO').upper()
 DEBUG = os.getenv('DEBUG', 'False').lower() in ['true', '1', 'yes']
 if DEBUG:
     LOG_LEVEL = 'DEBUG'
+    import litellm
+
+    litellm.set_verbose = True
 
 LOG_TO_FILE = os.getenv('LOG_TO_FILE', 'False').lower() in ['true', '1', 'yes']
 DISABLE_COLOR_PRINTING = False

openhands/core/message.py

@@ -49,8 +49,6 @@ class ImageContent(Content):
 
 
 class Message(BaseModel):
-    # NOTE: this is not the same as EventSource
-    # These are the roles in the LLM's APIs
     role: Literal['user', 'system', 'assistant', 'tool']
     content: list[TextContent | ImageContent] = Field(default_factory=list)
     cache_enabled: bool = False
@@ -68,21 +66,6 @@ class Message(BaseModel):
 
     @model_serializer
     def serialize_model(self) -> dict:
-        # We need two kinds of serializations:
-        # - into a single string: for providers that don't support list of content items (e.g. no vision, no tool calls)
-        # - into a list of content items: the new APIs of providers with vision/prompt caching/tool calls
-        # NOTE: remove this when litellm or providers support the new API
-        if self.cache_enabled or self.vision_enabled or self.tool_call_id is not None:
-            return self._list_serializer()
-        return self._string_serializer()
-
-    def _string_serializer(self):
-        content = '\n'.join(
-            item.text for item in self.content if isinstance(item, TextContent)
-        )
-        return {'content': content, 'role': self.role}
-
-    def _list_serializer(self):
         content: list[dict] = []
         role_tool_with_prompt_caching = False
         for item in self.content:

openhands/events/event.py

@@ -9,7 +9,6 @@ from openhands.llm.metrics import Metrics
 class EventSource(str, Enum):
     AGENT = 'agent'
     USER = 'user'
-    ENVIRONMENT = 'environment'
 
 
 @dataclass

openhands/events/stream.py

@@ -71,15 +71,7 @@ class EventStream:
         end_id=None,
         reverse=False,
         filter_out_type: tuple[type[Event], ...] | None = None,
-        filter_hidden=False,
     ) -> Iterable[Event]:
-        def should_filter(event: Event):
-            if filter_hidden and hasattr(event, 'hidden') and event.hidden:
-                return True
-            if filter_out_type is not None and isinstance(event, filter_out_type):
-                return True
-            return False
-
         if reverse:
             if end_id is None:
                 end_id = self._cur_id - 1
@@ -87,7 +79,9 @@ class EventStream:
             while event_id >= start_id:
                 try:
                     event = self.get_event(event_id)
-                    if not should_filter(event):
+                    if filter_out_type is None or not isinstance(
+                        event, filter_out_type
+                    ):
                         yield event
                 except FileNotFoundError:
                     logger.debug(f'No event found for ID {event_id}')
@@ -99,7 +93,9 @@ class EventStream:
                     break
                 try:
                     event = self.get_event(event_id)
-                    if not should_filter(event):
+                    if filter_out_type is None or not isinstance(
+                        event, filter_out_type
+                    ):
                         yield event
                 except FileNotFoundError:
                     break

openhands/llm/llm.py

@@ -47,20 +47,12 @@ LLM_RETRY_EXCEPTIONS: tuple[type[Exception], ...] = (
 # cache prompt supporting models
 # remove this when we gemini and deepseek are supported
 CACHE_PROMPT_SUPPORTED_MODELS = [
-    'claude-3-5-sonnet-20241022',
     'claude-3-5-sonnet-20240620',
+    'claude-3-5-sonnet-20241022',
     'claude-3-haiku-20240307',
     'claude-3-opus-20240229',
 ]
 
-# function calling supporting models
-FUNCTION_CALLING_SUPPORTED_MODELS = [
-    'claude-3-5-sonnet-20240620',
-    'claude-3-5-sonnet-20241022',
-    'gpt-4o',
-    'gpt-4o-mini',
-]
-
 
 class LLM(RetryMixin, DebugMixin):
     """The LLM class represents a Language Model instance.
@@ -91,13 +83,9 @@ class LLM(RetryMixin, DebugMixin):
         # litellm actually uses base Exception here for unknown model
         self.model_info: ModelInfo | None = None
 
-        try:
-            if self.config.model.startswith('openrouter'):
-                self.model_info = litellm.get_model_info(self.config.model)
-        except Exception as e:
-            logger.debug(f'Error getting model info: {e}')
-
-        if self.config.model.startswith('litellm_proxy/'):
+        if self.config.model.startswith('openrouter'):
+            self.model_info = litellm.get_model_info(self.config.model)
+        elif self.config.model.startswith('litellm_proxy/'):
             # IF we are using LiteLLM proxy, get model info from LiteLLM proxy
             # GET {base_url}/v1/model/info with litellm_model_id as path param
             response = requests.get(
@@ -175,6 +163,11 @@ class LLM(RetryMixin, DebugMixin):
                 ):
                     self.config.max_output_tokens = self.model_info['max_tokens']
 
+        self.config.supports_function_calling = (
+            self.model_info is not None
+            and self.model_info.get('supports_function_calling', False)
+        )
+
         self._completion = partial(
             litellm_completion,
             model=self.config.model,
@@ -193,7 +186,7 @@ class LLM(RetryMixin, DebugMixin):
             logger.debug('LLM: model has vision enabled')
         if self.is_caching_prompt_active():
             logger.debug('LLM: caching prompt enabled')
-        if self.is_function_calling_active():
+        if self.config.supports_function_calling:
             logger.debug('LLM: model supports function calling')
 
         completion_unwrapped = self._completion
@@ -324,27 +317,14 @@ class LLM(RetryMixin, DebugMixin):
         Returns:
             boolean: True if prompt caching is supported and enabled for the given model.
         """
-        return self.config.caching_prompt is True and (
-            (
+        return (
+            self.config.caching_prompt is True
+            and self.model_info is not None
+            and self.model_info.get('supports_prompt_caching', False)
+            and (
                 self.config.model in CACHE_PROMPT_SUPPORTED_MODELS
                 or self.config.model.split('/')[-1] in CACHE_PROMPT_SUPPORTED_MODELS
             )
-            or (
-                self.model_info is not None
-                and self.model_info.get('supports_prompt_caching', False)
-            )
-        )
-
-    def is_function_calling_active(self) -> bool:
-        # Check if model name is in supported list before checking model_info
-        model_name_supported = (
-            self.config.model in FUNCTION_CALLING_SUPPORTED_MODELS
-            or self.config.model.split('/')[-1] in FUNCTION_CALLING_SUPPORTED_MODELS
-            or any(m in self.config.model for m in FUNCTION_CALLING_SUPPORTED_MODELS)
-        )
-        return model_name_supported and (
-            self.model_info is not None
-            and self.model_info.get('supports_function_calling', False)
         )
 
     def _post_completion(self, response: ModelResponse) -> None:

openhands/runtime/base.py

@@ -95,7 +95,7 @@ class Runtime(FileEditRuntimeMixin):
 
     def log(self, level: str, message: str) -> None:
         message = f'[runtime {self.sid}] {message}'
-        getattr(logger, level)(message, stacklevel=2)
+        getattr(logger, level)(message)
 
     # ====================================================================
 
@@ -136,8 +136,6 @@ class Runtime(FileEditRuntimeMixin):
             )
             observation._cause = event.id  # type: ignore[attr-defined]
             observation.tool_call_metadata = event.tool_call_metadata
-
-            # this might be unnecessary, since source should be set by the event stream when we're here
             source = event.source if event.source else EventSource.AGENT
             await self.event_stream.async_add_event(observation, source)  # type: ignore[arg-type]
 

openhands/runtime/impl/e2b/sandbox.py

@@ -4,7 +4,9 @@ import tarfile
 from glob import glob
 
 from e2b import Sandbox as E2BSandbox
-from e2b.sandbox.exception import TimeoutException
+from e2b.sandbox.exception import (
+    TimeoutException,
+)
 
 from openhands.core.config import SandboxConfig
 from openhands.core.logger import openhands_logger as logger

openhands/runtime/impl/eventstream/eventstream_runtime.py

@@ -42,7 +42,7 @@ from openhands.utils.tenacity_stop import stop_if_should_exit
 
 
 class LogBuffer:
-    """Synchronous buffer for Docker container logs with proper shutdown handling.
+    """Synchronous buffer for Docker container logs.
 
     This class provides a thread-safe way to collect, store, and retrieve logs
     from a Docker container. It uses a list to store log lines and provides methods
@@ -51,94 +51,53 @@ class LogBuffer:
 
     def __init__(self, container: docker.models.containers.Container, logFn: Callable):
         self.init_msg = 'Runtime client initialized.'
-        self.container = container
+
         self.buffer: list[str] = []
         self.lock = threading.Lock()
         self._stop_event = threading.Event()
-        self._closed = False
-
-        self.log_generator = container.logs(stream=True, follow=True, tail=100)
-
-        self.log = logFn
+        self.log_generator = container.logs(stream=True, follow=True)
         self.log_stream_thread = threading.Thread(target=self.stream_logs)
         self.log_stream_thread.daemon = True
         self.log_stream_thread.start()
+        self.log = logFn
 
     def append(self, log_line: str):
-        """Thread-safe append to log buffer"""
-        if self._closed:
-            return
         with self.lock:
             self.buffer.append(log_line)
 
     def get_and_clear(self) -> list[str]:
-        """Thread-safe get and clear of log buffer"""
         with self.lock:
             logs = list(self.buffer)
             self.buffer.clear()
             return logs
 
     def stream_logs(self):
-        """Stream logs from the Docker container in a separate thread with error handling.
+        """Stream logs from the Docker container in a separate thread.
 
         This method runs in its own thread to handle the blocking
         operation of reading log lines from the Docker SDK's synchronous generator.
         """
-        if not self.log_generator:
-            return
-
         try:
-            while not self._stop_event.is_set():
-                try:
-                    # Use a timeout when reading from generator
-                    log_line = next(self.log_generator, None)
-                    if log_line is None:
-                        break
-                    if log_line:
-                        decoded_line = log_line.decode('utf-8').rstrip()
-                        self.append(decoded_line)
-                except StopIteration:
-                    break
-                except Exception as e:
-                    if not self._stop_event.is_set():
-                        self.log('error', f'Error reading docker logs: {e}')
+            for log_line in self.log_generator:
+                if self._stop_event.is_set():
                     break
+                if log_line:
+                    decoded_line = log_line.decode('utf-8').rstrip()
+                    self.append(decoded_line)
         except Exception as e:
-            if not self._stop_event.is_set():
-                self.log('error', f'Error in log stream thread: {e}')
-        finally:
-            self._closed = True
+            self.log('error', f'Error streaming docker logs: {e}')
 
     def __del__(self):
-        """Ensure proper cleanup on deletion"""
-        if not self._closed and hasattr(self, 'log_stream_thread') and self.log_stream_thread.is_alive():
+        if self.log_stream_thread.is_alive():
             self.log(
                 'warn',
                 "LogBuffer was not properly closed. Use 'log_buffer.close()' for clean shutdown.",
             )
-            self.close(timeout=2)
+            self.close(timeout=5)
 
     def close(self, timeout: float = 5.0):
-        """Close the log buffer with proper cleanup
-
-        Args:
-            timeout (float): Maximum time to wait for thread shutdown
-        """
-        if self._closed:
-            return
-
         self._stop_event.set()
-        self._closed = True
-
-        if hasattr(self, 'log_stream_thread') and self.log_stream_thread.is_alive():
-            try:
-                self.log_stream_thread.join(timeout)
-            except Exception as e:
-                self.log('error', f'Error joining log thread: {e}')
-            
-            # Force kill thread if it's still alive
-            if self.log_stream_thread.is_alive():
-                self.log('warn', 'Log thread did not shut down cleanly')
+        self.log_stream_thread.join(timeout)
 
 
 class EventStreamRuntime(Runtime):
@@ -246,7 +205,11 @@ class EventStreamRuntime(Runtime):
             self.log(
                 'info', f'Starting runtime with image: {self.runtime_container_image}'
             )
-            self._init_container()
+            self._init_container(
+                sandbox_workspace_dir=self.config.workspace_mount_path_in_sandbox,  # e.g. /workspace
+                mount_dir=self.config.workspace_mount_path,  # e.g. /opt/openhands/_test_workspace
+                plugins=self.plugins,
+            )
             self.log('info', f'Container started: {self.container_name}')
 
         else:
@@ -283,14 +246,19 @@ class EventStreamRuntime(Runtime):
         stop=tenacity.stop_after_attempt(5) | stop_if_should_exit(),
         wait=tenacity.wait_exponential(multiplier=1, min=4, max=60),
     )
-    def _init_container(self):
+    def _init_container(
+        self,
+        sandbox_workspace_dir: str,
+        mount_dir: str | None = None,
+        plugins: list[PluginRequirement] | None = None,
+    ):
         try:
             self.log('debug', 'Preparing to start container...')
             self.send_status_message('STATUS$PREPARING_CONTAINER')
             plugin_arg = ''
-            if self.plugins is not None and len(self.plugins) > 0:
+            if plugins is not None and len(plugins) > 0:
                 plugin_arg = (
-                    f'--plugins {" ".join([plugin.name for plugin in self.plugins])} '
+                    f'--plugins {" ".join([plugin.name for plugin in plugins])} '
                 )
 
             self._host_port = self._find_available_port()
@@ -326,27 +294,17 @@ class EventStreamRuntime(Runtime):
                 environment['DEBUG'] = 'true'
 
             self.log('debug', f'Workspace Base: {self.config.workspace_base}')
-            if (
-                self.config.workspace_mount_path is not None
-                and self.config.workspace_mount_path_in_sandbox is not None
-            ):
+            if mount_dir is not None and sandbox_workspace_dir is not None:
                 # e.g. result would be: {"/home/user/openhands/workspace": {'bind': "/workspace", 'mode': 'rw'}}
-                volumes = {
-                    self.config.workspace_mount_path: {
-                        'bind': self.config.workspace_mount_path_in_sandbox,
-                        'mode': 'rw',
-                    }
-                }
-                logger.debug(f'Mount dir: {self.config.workspace_mount_path}')
+                volumes = {mount_dir: {'bind': sandbox_workspace_dir, 'mode': 'rw'}}
+                self.log('debug', f'Mount dir: {mount_dir}')
             else:
-                logger.debug(
-                    'Mount dir is not set, will not mount the workspace directory to the container'
+                self.log(
+                    'warn',
+                    'Warning: Mount dir is not set, will not mount the workspace directory to the container!\n',
                 )
                 volumes = None
-            self.log(
-                'debug',
-                f'Sandbox workspace: {self.config.workspace_mount_path_in_sandbox}'
-            )
+            self.log('debug', f'Sandbox workspace: {sandbox_workspace_dir}')
 
             if self.config.sandbox.browsergym_eval_env is not None:
                 browsergym_arg = (
@@ -361,7 +319,7 @@ class EventStreamRuntime(Runtime):
                     f'/openhands/micromamba/bin/micromamba run -n openhands '
                     f'poetry run '
                     f'python -u -m openhands.runtime.action_execution_server {self._container_port} '
-                    f'--working-dir "{self.config.workspace_mount_path_in_sandbox}" '
+                    f'--working-dir "{sandbox_workspace_dir}" '
                     f'{plugin_arg}'
                     f'--username {"openhands" if self.config.run_as_openhands else "root"} '
                     f'--user-id {self.config.sandbox.user_id} '
@@ -424,25 +382,22 @@ class EventStreamRuntime(Runtime):
             )
 
     @tenacity.retry(
-        stop=tenacity.stop_after_delay(120) | stop_if_should_exit(),  # Reduced timeout
-        wait=tenacity.wait_exponential(multiplier=1, min=1, max=5),  # Faster retries
+        stop=tenacity.stop_after_delay(120) | stop_if_should_exit(),
+        wait=tenacity.wait_exponential(multiplier=2, min=1, max=20),
         reraise=(ConnectionRefusedError,),
     )
     def _wait_until_alive(self):
-        """Wait for runtime to be ready with proper error handling and timeouts"""
         self._refresh_logs()
         if not self.log_buffer:
             raise RuntimeError('Runtime client is not ready.')
 
-        # Use a shorter timeout for individual requests
         response = send_request_with_retry(
             self.session,
             'GET',
             f'{self.api_url}/alive',
             retry_exceptions=[ConnectionRefusedError],
-            timeout=10,  # Shorter timeout per request
+            timeout=300,  # 5 minutes gives the container time to be alive 🧟‍♂️
         )
-
         if response.status_code == 200:
             return
         else:
@@ -450,55 +405,22 @@ class EventStreamRuntime(Runtime):
             self.log('error', msg)
             raise RuntimeError(msg)
 
-
     def close(self, rm_all_containers: bool = True):
-        """Closes the EventStreamRuntime and associated objects with proper cleanup
+        """Closes the EventStreamRuntime and associated objects
 
         Parameters:
         - rm_all_containers (bool): Whether to remove all containers with the 'openhands-sandbox-' prefix
         """
-        print("CLOSE RUNTIME")
-        # First stop any ongoing requests
-        if self.session:
-            try:
-                self.session.close()
-            except Exception as e:
-                self.log('error', f'Error closing session: {e}')
-            self.session = None
 
-        # Then close log buffer
         if self.log_buffer:
-            try:
-                self.log_buffer.close(timeout=2)  # Short timeout for log buffer
-            except Exception as e:
-                self.log('error', f'Error closing log buffer: {e}')
-            self.log_buffer = None
+            self.log_buffer.close()
+
+        if self.session:
+            self.session.close()
 
-        # Skip container cleanup if we're attached
         if self.attach_to_existing:
             return
 
-        # Clean up container
-        if self.container:
-            try:
-                # Try to stop container gracefully first
-                self.container.stop(timeout=5)
-            except Exception as e:
-                self.log('error', f'Error stopping container: {e}')
-                try:
-                    # Force kill if graceful stop fails
-                    self.container.kill()
-                except Exception as e2:
-                    self.log('error', f'Error killing container: {e2}')
-            
-            try:
-                # Remove the container
-                self.container.remove(force=True)
-            except Exception as e:
-                self.log('error', f'Error removing container: {e}')
-            
-            self.container = None
-
         try:
             containers = self.docker_client.containers.list(all=True)
             for container in containers:

openhands/runtime/plugins/agent_skills/file_editor/impl.py

@@ -46,13 +46,13 @@ class EditTool:
         if command == 'view':
             return self.view(_path, view_range)
         elif command == 'create':
-            if file_text is None:
+            if not file_text:
                 raise ToolError('Parameter `file_text` is required for command: create')
             self.write_file(_path, file_text)
             self._file_history[_path].append(file_text)
             return ToolResult(output=f'File created successfully at: {_path}')
         elif command == 'str_replace':
-            if old_str is None:
+            if not old_str:
                 raise ToolError(
                     'Parameter `old_str` is required for command: str_replace'
                 )
@@ -62,7 +62,7 @@ class EditTool:
                 raise ToolError(
                     'Parameter `insert_line` is required for command: insert'
                 )
-            if new_str is None:
+            if not new_str:
                 raise ToolError('Parameter `new_str` is required for command: insert')
             return self.insert(_path, insert_line, new_str)
         elif command == 'undo_edit':

openhands/runtime/utils/runtime_templates/Dockerfile.j2

@@ -71,9 +71,7 @@ RUN /openhands/micromamba/bin/micromamba create -n openhands -y && \
 RUN \
     if [ -d /openhands/code ]; then rm -rf /openhands/code; fi && \
     mkdir -p /openhands/code/openhands && \
-    touch /openhands/code/openhands/__init__.py && \
-    chmod a+rwx /openhands/code/openhands/__init__.py
-
+    touch /openhands/code/openhands/__init__.py
 COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/
 
 {{ install_dependencies() }}

openhands/security/invariant/analyzer.py

@@ -147,7 +147,6 @@ class InvariantAnalyzer(SecurityAnalyzer):
         new_event = action_from_dict(
             {'action': 'change_agent_state', 'args': {'agent_state': 'user_confirmed'}}
         )
-        # we should confirm only on agent actions
         event_source = event.source if event.source else EventSource.AGENT
         await call_sync_from_async(self.event_stream.add_event, new_event, event_source)
 

openhands/server/data_models/feedback.py

@@ -1,5 +1,5 @@
 import json
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import requests
 from pydantic import BaseModel
@@ -10,12 +10,10 @@ from openhands.core.logger import openhands_logger as logger
 class FeedbackDataModel(BaseModel):
     version: str
     email: str
-    polarity: Literal['positive', 'negative']
-    feedback: Literal[
-        'positive', 'negative'
-    ]  # TODO: remove this, its here for backward compatibility
+    token: str
+    feedback: Literal['positive', 'negative']
     permissions: Literal['public', 'private']
-    trajectory: Optional[list[dict[str, Any]]]
+    trajectory: list[dict[str, Any]]
 
 
 FEEDBACK_URL = 'https://share-od-trajectory-3u9bw9tx.uc.gateway.dev/share_od_trajectory'
@@ -23,7 +21,6 @@ FEEDBACK_URL = 'https://share-od-trajectory-3u9bw9tx.uc.gateway.dev/share_od_tra
 
 def store_feedback(feedback: FeedbackDataModel) -> dict[str, str]:
     # Start logging
-    feedback.feedback = feedback.polarity
     display_feedback = feedback.model_dump()
     if 'trajectory' in display_feedback:
         display_feedback['trajectory'] = (

openhands/server/github.py

@@ -1,128 +0,0 @@
-import os
-
-import httpx
-
-from openhands.core.logger import openhands_logger as logger
-from openhands.server.sheets_client import GoogleSheetsClient
-
-GITHUB_CLIENT_ID = os.getenv('GITHUB_CLIENT_ID', '').strip()
-GITHUB_CLIENT_SECRET = os.getenv('GITHUB_CLIENT_SECRET', '').strip()
-
-
-class UserVerifier:
-    def __init__(self) -> None:
-        logger.info('Initializing UserVerifier')
-        self.file_users: list[str] | None = None
-        self.sheets_client: GoogleSheetsClient | None = None
-        self.spreadsheet_id: str | None = None
-
-        # Initialize from environment variables
-        self._init_file_users()
-        self._init_sheets_client()
-
-    def _init_file_users(self) -> None:
-        """Load users from text file if configured"""
-        waitlist = os.getenv('GITHUB_USER_LIST_FILE')
-        if not waitlist:
-            logger.info('GITHUB_USER_LIST_FILE not configured')
-            return
-
-        if not os.path.exists(waitlist):
-            logger.error(f'User list file not found: {waitlist}')
-            raise FileNotFoundError(f'User list file not found: {waitlist}')
-
-        try:
-            with open(waitlist, 'r') as f:
-                self.file_users = [line.strip() for line in f if line.strip()]
-            logger.info(
-                f'Successfully loaded {len(self.file_users)} users from {waitlist}'
-            )
-        except Exception as e:
-            logger.error(f'Error reading user list file {waitlist}: {str(e)}')
-
-    def _init_sheets_client(self) -> None:
-        """Initialize Google Sheets client if configured"""
-        sheet_id = os.getenv('GITHUB_USERS_SHEET_ID')
-
-        if not sheet_id:
-            logger.info('GITHUB_USERS_SHEET_ID not configured')
-            return
-
-        logger.info('Initializing Google Sheets integration')
-        self.sheets_client = GoogleSheetsClient()
-        self.spreadsheet_id = sheet_id
-
-    def is_active(self) -> bool:
-        return bool(self.file_users or (self.sheets_client and self.spreadsheet_id))
-
-    def is_user_allowed(self, username: str) -> bool:
-        """Check if user is allowed based on file and/or sheet configuration"""
-        if not self.is_active():
-            return True
-
-        logger.info(f'Checking if GitHub user {username} is allowed')
-        if self.file_users:
-            if username in self.file_users:
-                logger.info(f'User {username} found in text file allowlist')
-                return True
-            logger.debug(f'User {username} not found in text file allowlist')
-
-        if self.sheets_client and self.spreadsheet_id:
-            sheet_users = self.sheets_client.get_usernames(self.spreadsheet_id)
-            if username in sheet_users:
-                logger.info(f'User {username} found in Google Sheets allowlist')
-                return True
-            logger.debug(f'User {username} not found in Google Sheets allowlist')
-
-        logger.info(f'User {username} not found in any allowlist')
-        return False
-
-
-async def authenticate_github_user(auth_token) -> bool:
-    user_verifier = UserVerifier()
-
-    if not user_verifier.is_active():
-        logger.info('No user verification sources configured - allowing all users')
-        return True
-
-    logger.info('Checking GitHub token')
-
-    if not auth_token:
-        logger.warning('No GitHub token provided')
-        return False
-
-    login = await get_github_user(auth_token)
-
-    if not user_verifier.is_user_allowed(login):
-        logger.warning(f'GitHub user {login} not in allow list')
-        return False
-
-    logger.info(f'GitHub user {login} authenticated')
-    return True
-
-
-async def get_github_user(token: str) -> str:
-    """Get GitHub user info from token.
-
-    Args:
-        token: GitHub access token
-
-    Returns:
-        Tuple of (login, error_message)
-        If successful, error_message is None
-        If failed, login is None and error_message contains the error
-    """
-    logger.info('Fetching GitHub user info from token')
-    headers = {
-        'Accept': 'application/vnd.github+json',
-        'Authorization': f'Bearer {token}',
-        'X-GitHub-Api-Version': '2022-11-28',
-    }
-    async with httpx.AsyncClient() as client:
-        logger.debug('Making request to GitHub API')
-        response = await client.get('https://api.github.com/user', headers=headers)
-        response.raise_for_status()
-        user_data = response.json()
-        login = user_data.get('login')
-        logger.info(f'Successfully retrieved GitHub user: {login}')
-        return login

openhands/server/listen.py

@@ -13,11 +13,6 @@ from pathspec.patterns import GitWildMatchPattern
 
 from openhands.security.options import SecurityAnalyzers
 from openhands.server.data_models.feedback import FeedbackDataModel, store_feedback
-from openhands.server.github import (
-    GITHUB_CLIENT_ID,
-    GITHUB_CLIENT_SECRET,
-    authenticate_github_user,
-)
 from openhands.storage import get_file_store
 from openhands.utils.async_utils import call_sync_from_async
 
@@ -34,10 +29,12 @@ from fastapi import (
     WebSocket,
     status,
 )
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, StreamingResponse
 from fastapi.security import HTTPBearer
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
+from starlette.middleware.base import BaseHTTPMiddleware
 
 import openhands.agenthub  # noqa F401 (we import this to get the agents registered)
 from openhands.controller.agent import Agent
@@ -60,7 +57,6 @@ from openhands.events.serialization import event_to_dict
 from openhands.llm import bedrock
 from openhands.runtime.base import Runtime
 from openhands.server.auth import get_sid_from_token, sign_token
-from openhands.server.middleware import LocalhostCORSMiddleware, NoCacheMiddleware
 from openhands.server.session import SessionManager
 
 load_dotenv()
@@ -69,6 +65,9 @@ config = load_app_config()
 file_store = get_file_store(config.file_store, config.file_store_path)
 session_manager = SessionManager(config, file_store)
 
+GITHUB_CLIENT_ID = os.getenv('GITHUB_CLIENT_ID', '').strip()
+GITHUB_CLIENT_SECRET = os.getenv('GITHUB_CLIENT_SECRET', '').strip()
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -79,13 +78,30 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(lifespan=lifespan)
 app.add_middleware(
-    LocalhostCORSMiddleware,
+    CORSMiddleware,
+    allow_origins=['http://localhost:3001', 'http://127.0.0.1:3001'],
     allow_credentials=True,
     allow_methods=['*'],
     allow_headers=['*'],
 )
 
 
+class NoCacheMiddleware(BaseHTTPMiddleware):
+    """
+    Middleware to disable caching for all routes by adding appropriate headers
+    """
+
+    async def dispatch(self, request, call_next):
+        response = await call_next(request)
+        if not request.url.path.startswith('/assets'):
+            response.headers['Cache-Control'] = (
+                'no-cache, no-store, must-revalidate, max-age=0'
+            )
+            response.headers['Pragma'] = 'no-cache'
+            response.headers['Expires'] = '0'
+        return response
+
+
 app.add_middleware(NoCacheMiddleware)
 
 security_scheme = HTTPBearer()
@@ -203,13 +219,7 @@ async def attach_session(request: Request, call_next):
         response = await call_next(request)
         return response
 
-    github_token = request.headers.get('X-GitHub-Token')
-    if not await authenticate_github_user(github_token):
-        return JSONResponse(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            content={'error': 'Not authenticated'},
-        )
-
+    # For all other methods, validate the Authorization header
     if not request.headers.get('Authorization'):
         logger.warning('Missing Authorization header')
         return JSONResponse(
@@ -301,100 +311,43 @@ async def websocket_endpoint(websocket: WebSocket):
         {"action": "finish", "args": {}}
         ```
     """
-    session = None
-    try:
-        # Get protocols from Sec-WebSocket-Protocol header
-        protocols = websocket.headers.get('sec-websocket-protocol', '').split(', ')
+    await asyncio.wait_for(websocket.accept(), 10)
 
-        # The first protocol should be our real protocol (e.g. 'openhands')
-        # The second protocol should contain our auth token
-        if len(protocols) < 3:
-            logger.error('Expected 3 websocket protocols, got %d', len(protocols))
-            await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
+    if websocket.query_params.get('token'):
+        token = websocket.query_params.get('token')
+        sid = get_sid_from_token(token, config.jwt_secret)
+
+        if sid == '':
+            await websocket.send_json({'error': 'Invalid token', 'error_code': 401})
+            await websocket.close()
             return
+    else:
+        sid = str(uuid.uuid4())
+        token = sign_token({'sid': sid}, config.jwt_secret)
 
-        real_protocol = protocols[0]
-        jwt_token = protocols[1] if protocols[1] != 'NO_JWT' else ''
-        github_token = protocols[2] if protocols[2] != 'NO_GITHUB' else ''
+    logger.info(f'New session: {sid}')
+    session = session_manager.add_or_restart_session(sid, websocket)
+    await websocket.send_json({'token': token, 'status': 'ok'})
 
-        if not await authenticate_github_user(github_token):
-            await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
-            return
+    latest_event_id = -1
+    if websocket.query_params.get('latest_event_id'):
+        latest_event_id = int(websocket.query_params.get('latest_event_id'))
+    for event in session.agent_session.event_stream.get_events(
+        start_id=latest_event_id + 1
+    ):
+        if isinstance(
+            event,
+            (
+                NullAction,
+                NullObservation,
+                ChangeAgentStateAction,
+                AgentStateChangedObservation,
+            ),
+        ):
+            continue
+        await websocket.send_json(event_to_dict(event))
 
-        try:
-            await asyncio.wait_for(websocket.accept(subprotocol=real_protocol), timeout=10)
-        except asyncio.TimeoutError:
-            logger.error("WebSocket accept timed out")
-            await websocket.close(code=status.WS_1408_REQUEST_TIMEOUT)
-            return
-
-        if jwt_token:
-            sid = get_sid_from_token(jwt_token, config.jwt_secret)
-
-            if sid == '':
-                await websocket.send_json({'error': 'Invalid token', 'error_code': 401})
-                await websocket.close()
-                return
-        else:
-            sid = str(uuid.uuid4())
-            jwt_token = sign_token({'sid': sid}, config.jwt_secret)
-
-        logger.info(f'New session: {sid}')
-        session = session_manager.add_or_restart_session(sid, websocket)
-        
-        try:
-            await asyncio.wait_for(
-                websocket.send_json({'token': jwt_token, 'status': 'ok'}),
-                timeout=5
-            )
-        except asyncio.TimeoutError:
-            logger.error("Failed to send initial response")
-            await session.close()
-            await websocket.close(code=status.WS_1408_REQUEST_TIMEOUT)
-            return
-
-        latest_event_id = -1
-        if websocket.query_params.get('latest_event_id'):
-            latest_event_id = int(websocket.query_params.get('latest_event_id'))
-        
-        # Send historical events with timeout
-        try:
-            async with asyncio.timeout(30):  # 30 second timeout for historical events
-                for event in session.agent_session.event_stream.get_events(
-                    start_id=latest_event_id + 1
-                ):
-                    if isinstance(
-                        event,
-                        (
-                            NullAction,
-                            NullObservation,
-                            ChangeAgentStateAction,
-                            AgentStateChangedObservation,
-                        ),
-                    ):
-                        continue
-                    await websocket.send_json(event_to_dict(event))
-        except asyncio.TimeoutError:
-            logger.error("Timeout sending historical events")
-            await session.close()
-            await websocket.close(code=status.WS_1408_REQUEST_TIMEOUT)
-            return
-
-        # Start the main receive loop with heartbeat
-        await session.loop_recv()
-
-    except WebSocketDisconnect:
-        logger.info("WebSocket disconnected normally")
-    except Exception as e:
-        logger.exception("Error in websocket handler: %s", str(e))
-    finally:
-        if session:
-            try:
-                await asyncio.wait_for(session.close(), timeout=5)
-            except asyncio.TimeoutError:
-                logger.error("Timeout during session cleanup")
-            except Exception as e:
-                logger.exception("Error during session cleanup: %s", str(e))
+    await session.loop_recv()
 
 
 @app.get('/api/options/models')
@@ -684,14 +637,14 @@ async def upload_file(request: Request, files: list[UploadFile]):
 
 
 @app.post('/api/submit-feedback')
-async def submit_feedback(request: Request):
+async def submit_feedback(request: Request, feedback: FeedbackDataModel):
     """Submit user feedback.
 
     This function stores the provided feedback data.
 
     To submit feedback:
     ```sh
-    curl -X POST -d '{"email": "test@example.com"}' -H "Authorization:"
+    curl -X POST -F "email=test@example.com" -F "token=abc" -F "feedback=positive" -F "permissions=private" -F "trajectory={}" http://localhost:3000/api/submit-feedback
     ```
 
     Args:
@@ -706,19 +659,6 @@ async def submit_feedback(request: Request):
     """
     # Assuming the storage service is already configured in the backend
     # and there is a function to handle the storage.
-    body = await request.json()
-    events = request.state.conversation.event_stream.get_events(filter_hidden=True)
-    trajectory = []
-    for event in events:
-        trajectory.append(event_to_dict(event))
-    feedback = FeedbackDataModel(
-        email=body.get('email', ''),
-        version=body.get('version', ''),
-        permissions=body.get('permissions', 'private'),
-        polarity=body.get('polarity', ''),
-        feedback=body.get('polarity', ''),
-        trajectory=trajectory,
-    )
     try:
         feedback_data = store_feedback(feedback)
         return JSONResponse(status_code=200, content=feedback_data)
@@ -890,21 +830,34 @@ def github_callback(auth_code: AuthCode):
     )
 
 
-@app.post('/api/authenticate')
-async def authenticate(request: Request):
-    token = request.headers.get('X-GitHub-Token')
-    if not await authenticate_github_user(token):
-        return JSONResponse(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            content={'error': 'Not authorized via GitHub waitlist'},
-        )
+class User(BaseModel):
+    login: str  # GitHub login handle
 
-    response = JSONResponse(
+
+@app.post('/api/authenticate')
+def authenticate(user: User | None = None):
+    waitlist = os.getenv('GITHUB_USER_LIST_FILE')
+
+    # Only check if waitlist is provided
+    if waitlist is not None:
+        try:
+            with open(waitlist, 'r') as f:
+                users = f.read().splitlines()
+                if user is None or user.login not in users:
+                    return JSONResponse(
+                        status_code=status.HTTP_403_FORBIDDEN,
+                        content={'error': 'User not on waitlist'},
+                    )
+        except FileNotFoundError:
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={'error': 'Waitlist file not found'},
+            )
+
+    return JSONResponse(
         status_code=status.HTTP_200_OK, content={'message': 'User authenticated'}
     )
 
-    return response
-
 
 class SPAStaticFiles(StaticFiles):
     async def get_response(self, path: str, scope):

openhands/server/middleware.py

@@ -1,43 +0,0 @@
-from urllib.parse import urlparse
-
-from fastapi.middleware.cors import CORSMiddleware
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.types import ASGIApp
-
-
-class LocalhostCORSMiddleware(CORSMiddleware):
-    """
-    Custom CORS middleware that allows any request from localhost/127.0.0.1 domains,
-    while using standard CORS rules for other origins.
-    """
-
-    def __init__(self, app: ASGIApp, **kwargs) -> None:
-        super().__init__(app, **kwargs)
-
-    def is_allowed_origin(self, origin: str) -> bool:
-        if origin:
-            parsed = urlparse(origin)
-            hostname = parsed.hostname or ''
-
-            # Allow any localhost/127.0.0.1 origin regardless of port
-            if hostname in ['localhost', '127.0.0.1']:
-                return True
-
-        # For missing origin or other origins, use the parent class's logic
-        return super().is_allowed_origin(origin)
-
-
-class NoCacheMiddleware(BaseHTTPMiddleware):
-    """
-    Middleware to disable caching for all routes by adding appropriate headers
-    """
-
-    async def dispatch(self, request, call_next):
-        response = await call_next(request)
-        if not request.url.path.startswith('/assets'):
-            response.headers['Cache-Control'] = (
-                'no-cache, no-store, must-revalidate, max-age=0'
-            )
-            response.headers['Pragma'] = 'no-cache'
-            response.headers['Expires'] = '0'
-        return response

openhands/server/session/agent_session.py

@@ -55,39 +55,22 @@ class AgentSession:
         agent_configs: dict[str, AgentConfig] | None = None,
         status_message_callback: Optional[Callable] = None,
     ):
-        """Starts the Agent session with proper error handling and timeouts
+        """Starts the Agent session
         Parameters:
         - runtime_name: The name of the runtime associated with the session
-        - config: Application configuration
-        - agent: Agent instance to use
-        - max_iterations: Maximum number of iterations
-        - max_budget_per_task: Maximum budget per task
-        - agent_to_llm_config: LLM configurations for different agents
-        - agent_configs: Agent configurations
-        - status_message_callback: Callback for status updates
+        - config:
+        - agent:
+        - max_iterations:
+        - max_budget_per_task:
+        - agent_to_llm_config:
+        - agent_configs:
         """
         if self.controller or self.runtime:
             raise RuntimeError(
                 'Session already started. You need to close this session and start a new one.'
             )
 
-        # Create a future to track the start operation
-        start_future = asyncio.Future()
-        
-        def start_callback(future):
-            try:
-                exc = future.exception()
-                if exc:
-                    start_future.set_exception(exc)
-                else:
-                    start_future.set_result(None)
-            except asyncio.CancelledError:
-                start_future.cancel()
-            except Exception as e:
-                start_future.set_exception(e)
-
-        # Start the agent in a thread pool with proper error propagation
-        task = asyncio.get_event_loop().run_in_executor(
+        asyncio.get_event_loop().run_in_executor(
             None,
             self._start_thread,
             runtime_name,
@@ -99,49 +82,13 @@ class AgentSession:
             agent_configs,
             status_message_callback,
         )
-        task.add_done_callback(start_callback)
-
-        try:
-            # Wait for start with timeout
-            await asyncio.wait_for(start_future, timeout=120)
-        except asyncio.TimeoutError:
-            logger.error("Agent session start timed out")
-            # Cleanup if start times out
-            await self.close()
-            raise RuntimeError("Agent session start timed out")
-        except Exception as e:
-            logger.exception("Error starting agent session")
-            await self.close()
-            raise
 
     def _start_thread(self, *args):
-        """Start the agent in a separate thread with proper error handling"""
         try:
-            # Create new event loop for this thread
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            
-            try:
-                # Run with timeout
-                loop.run_until_complete(
-                    asyncio.wait_for(self._start(*args), timeout=25)
-                )
-            except asyncio.TimeoutError:
-                logger.error("Timeout in agent start")
-                raise RuntimeError("Timeout in agent start")
-            except Exception as e:
-                logger.exception("Error in agent start")
-                raise
-            finally:
-                try:
-                    # Clean up the loop
-                    loop.run_until_complete(loop.shutdown_asyncgens())
-                    loop.close()
-                except Exception as e:
-                    logger.error(f"Error cleaning up thread loop: {e}")
-        except Exception as e:
-            logger.error(f"Fatal error in start thread: {e}")
-            raise
+            asyncio.run(self._start(*args), debug=True)
+        except RuntimeError:
+            logger.error(f'Error starting session: {RuntimeError}', exc_info=True)
+            logger.debug('Session Finished')
 
     async def _start(
         self,
@@ -154,6 +101,7 @@ class AgentSession:
         agent_configs: dict[str, AgentConfig] | None = None,
         status_message_callback: Optional[Callable] = None,
     ):
+        self.loop = asyncio.get_running_loop()
         self._create_security_analyzer(config.security.security_analyzer)
         await self._create_runtime(
             runtime_name=runtime_name,
@@ -170,67 +118,30 @@ class AgentSession:
             agent_configs=agent_configs,
         )
         self.event_stream.add_event(
-            ChangeAgentStateAction(AgentState.INIT), EventSource.ENVIRONMENT
+            ChangeAgentStateAction(AgentState.INIT), EventSource.USER
         )
         if self.controller:
             self.controller.agent_task = self.controller.start_step_loop()
             await self.controller.agent_task  # type: ignore
 
     async def close(self):
-        """Closes the Agent session with proper cleanup and timeouts"""
+        """Closes the Agent session"""
+
         if self._closed:
             return
+        if self.controller is not None:
+            end_state = self.controller.get_state()
+            end_state.save_to_session(self.sid, self.file_store)
+            await self.controller.close()
+        if self.runtime is not None:
+            self.runtime.close()
+        if self.security_analyzer is not None:
+            await self.security_analyzer.close()
 
-        try:
-            # Set closed flag early to prevent multiple close attempts
-            self._closed = True
+        if self.loop:
+            self.loop.stop()
 
-            # Save state with timeout
-            if self.controller is not None:
-                try:
-                    async with asyncio.timeout(5):
-                        end_state = self.controller.get_state()
-                        end_state.save_to_session(self.sid, self.file_store)
-                except asyncio.TimeoutError:
-                    logger.error("Timeout saving agent state")
-                except Exception as e:
-                    logger.error(f"Error saving agent state: {e}")
-
-            # Close controller with timeout
-            if self.controller is not None:
-                try:
-                    async with asyncio.timeout(5):
-                        await self.controller.close()
-                except asyncio.TimeoutError:
-                    logger.error("Timeout closing controller")
-                except Exception as e:
-                    logger.error(f"Error closing controller: {e}")
-                self.controller = None
-
-            # Close runtime (this is synchronous but should be quick)
-            if self.runtime is not None:
-                try:
-                    self.runtime.close()
-                except Exception as e:
-                    logger.error(f"Error closing runtime: {e}")
-                self.runtime = None
-
-            # Close security analyzer with timeout
-            if self.security_analyzer is not None:
-                try:
-                    async with asyncio.timeout(5):
-                        await self.security_analyzer.close()
-                except asyncio.TimeoutError:
-                    logger.error("Timeout closing security analyzer")
-                except Exception as e:
-                    logger.error(f"Error closing security analyzer: {e}")
-                self.security_analyzer = None
-
-        except Exception as e:
-            logger.exception(f"Unexpected error during session cleanup: {e}")
-        finally:
-            # Ensure closed flag is set even if cleanup fails
-            self._closed = True
+        self._closed = True
 
     def _create_security_analyzer(self, security_analyzer: str | None):
         """Creates a SecurityAnalyzer instance that will be used to analyze the agent actions