Update openhands/runtime/client/client.py

Signed-off-by: niliy <WannaTen@users.noreply.github.com>

.devcontainer/README.MD

@@ -1 +1 @@
-The files in this directory configure a development container for GitHub Codespaces.
+The files in this directory configure a development container for GitHub Codespaces.

.github/pull_request_template.md

@@ -1,4 +1,4 @@
-**What is the problem that this fixes or functionality that this introduces? Does it fix any open issues?**
+**Short description of the problem this fixes or functionality that this introduces. This may be used for the CHANGELOG**
 
 
 
@@ -8,4 +8,4 @@
 
 
 ---
-**Other references**
+**Link of any specific issues this addresses**

.github/workflows/ghcr_app.yml

@@ -1,7 +1,6 @@
 # Workflow that builds, tests and then pushes the app docker images to the ghcr.io repository
 name: Build and Publish App Image
 
-
 # Always run on "main"
 # Always run on tags
 # Always run on PRs
@@ -56,6 +55,11 @@ jobs:
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
-      - name: Build and export image
-        id: build
-        run: ./containers/build.sh openhands ${{ github.repository_owner }} --push
+      - name: Build and push app image
+        if: "!github.event.pull_request.head.repo.fork"
+        run: |
+          ./containers/build.sh openhands ${{ github.repository_owner }} --push
+      - name: Build app image
+        if: "github.event.pull_request.head.repo.fork"
+        run: |
+          ./containers/build.sh openhands image ${{ github.repository_owner }}

.github/workflows/ghcr_runtime.yml

@@ -1,5 +1,5 @@
 # Workflow that builds, tests and then pushes the runtime docker images to the ghcr.io repository
-name: Build, Test and Publish Runtime Image
+name: Build, Test and Publish RT Image
 
 # Only run one workflow of the same group at a time.
 # There can be at most one running and one pending job in a concurrency group at any time.
@@ -7,6 +7,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
+# Always run on "main"
+# Always run on tags
+# Always run on PRs
+# Can also be triggered manually
 on:
   push:
     branches:
@@ -31,7 +35,9 @@ jobs:
       packages: write
     strategy:
       matrix:
-        base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm']
+        base_image:
+          - image: 'nikolaik/python-nodejs:python3.11-nodejs22'
+            tag: nikolaik
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -60,62 +66,164 @@ jobs:
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
-      - name: Install poetry via pipx
-        run: pipx install poetry
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-          cache: 'poetry'
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      - name: Install poetry via pipx
+        run: pipx install poetry
       - name: Install Python dependencies using Poetry
         run: make install-python-dependencies
       - name: Create source distribution and Dockerfile
-        run: poetry run python3 openhands/runtime/utils/runtime_build.py --base_image ${{ matrix.base_image }} --build_folder containers/runtime --force_rebuild
-      - name: Build and export image
-        id: build
+        run: poetry run python3 openhands/runtime/utils/runtime_build.py --base_image ${{ matrix.base_image.image }} --build_folder containers/runtime --force_rebuild
+      - name: Build and push runtime image ${{ matrix.base_image.image }}
+        if: github.event.pull_request.head.repo.fork != true
         run: |
-          suffix=$(echo "${{ matrix.base_image }}" | cut -d ':' -f 1 | cut -d '/' -f 1)
-          ./containers/build.sh runtime ${{ github.repository_owner }} --push $suffix
+          ./containers/build.sh runtime ${{ github.repository_owner }} --push ${{ matrix.base_image.tag }}
+      # Forked repos can't push to GHCR, so we need to upload the image as an artifact
+      - name: Build runtime image ${{ matrix.base_image.image }} for fork
+        if: github.event.pull_request.head.repo.fork
+        uses: docker/build-push-action@v6
+        with:
+          tags: ghcr.io/all-hands-ai/runtime:${{ github.sha }}-${{ matrix.base_image.tag }}
+          outputs: type=docker,dest=/tmp/runtime-${{ matrix.base_image.tag }}.tar
+          context: containers/runtime
+      - name: Upload runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtime-${{ matrix.base_image.tag }}
+          path: /tmp/runtime-${{ matrix.base_image.tag }}.tar
 
-  # Run unit tests with the EventStream runtime Docker images
-  test_runtime:
-    name: Test Runtime
-    runs-on: ubuntu-latest
+  # Run unit tests with the EventStream runtime Docker images as root
+  test_runtime_root:
+    name: RT Unit Tests (Root)
     needs: [ghcr_build_runtime]
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        base_image: ['nikolaik', 'python', 'node']
+        base_image: ['nikolaik']
     steps:
       - uses: actions/checkout@v4
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
+      # Forked repos can't push to GHCR, so we need to download the image as an artifact
+      - name: Download runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        uses: actions/download-artifact@v4
         with:
-          tool-cache: true
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          swap-storage: true
-      - name: Install poetry via pipx
-        run: pipx install poetry
+          name: runtime-${{ matrix.base_image }}
+          path: /tmp
+      - name: Load runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        run: |
+          docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-          cache: 'poetry'
+      - name: Install poetry via pipx
+        run: pipx install poetry
       - name: Install Python dependencies using Poetry
         run: make install-python-dependencies
       - name: Run runtime tests
         run: |
-          git_hash=$(git rev-parse --short "$GITHUB_SHA")
-          image_name=ghcr.io/${{ github.repository_owner }}/runtime:$git_hash-${{ matrix.base_image }}
+          # We install pytest-xdist in order to run tests across CPUs. However, tests start to fail when we run
+          # then across more than 2 CPUs for some reason
+          poetry run pip install pytest-xdist
+
+          # Install to be able to retry on failures for flaky tests
+          poetry run pip install pytest-rerunfailures
+
+          image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
+          SKIP_CONTAINER_LOGS=true \
           TEST_RUNTIME=eventstream \
           SANDBOX_USER_ID=$(id -u) \
           SANDBOX_BASE_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
-          poetry run pytest --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
+          RUN_AS_OPENHANDS=false \
+          poetry run pytest -n 3 --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  # Run unit tests with the EventStream runtime Docker images as openhands user
+  test_runtime_oh:
+    name: RT Unit Tests (openhands)
+    runs-on: ubuntu-latest
+    needs: [ghcr_build_runtime]
+    strategy:
+      matrix:
+        base_image: ['nikolaik']
+    steps:
+      - uses: actions/checkout@v4
+      # Forked repos can't push to GHCR, so we need to download the image as an artifact
+      - name: Download runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        uses: actions/download-artifact@v4
+        with:
+          name: runtime-${{ matrix.base_image }}
+          path: /tmp
+      - name: Load runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        run: |
+          docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install poetry via pipx
+        run: pipx install poetry
+      - name: Install Python dependencies using Poetry
+        run: make install-python-dependencies
+      - name: Run runtime tests
+        run: |
+          # We install pytest-xdist in order to run tests across CPUs. However, tests start to fail when we run
+          # then across more than 2 CPUs for some reason
+          poetry run pip install pytest-xdist
+
+          # Install to be able to retry on failures for flaky tests
+          poetry run pip install pytest-rerunfailures
+
+          image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
+          image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
+
+          SKIP_CONTAINER_LOGS=true \
+          TEST_RUNTIME=eventstream \
+          SANDBOX_USER_ID=$(id -u) \
+          SANDBOX_BASE_CONTAINER_IMAGE=$image_name \
+          TEST_IN_CI=true \
+          RUN_AS_OPENHANDS=true \
+          poetry run pytest -n 3 --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
@@ -123,28 +231,46 @@ jobs:
 
   # Run integration tests with the eventstream runtime Docker image
   runtime_integration_tests_on_linux:
-    name: Runtime Integration Tests on Linux
+    name: RT Integration Tests (Linux)
     runs-on: ubuntu-latest
     needs: [ghcr_build_runtime]
     strategy:
       fail-fast: false
       matrix:
-        base_image: ['nikolaik', 'python', 'node']
+        base_image: ['nikolaik']
     steps:
       - uses: actions/checkout@v4
-      - name: Install poetry via pipx
-        run: pipx install poetry
+      # Forked repos can't push to GHCR, so we need to download the image as an artifact
+      - name: Download runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        uses: actions/download-artifact@v4
+        with:
+          name: runtime-${{ matrix.base_image }}
+          path: /tmp
+      - name: Load runtime image for fork
+        if: github.event.pull_request.head.repo.fork
+        run: |
+          docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-          cache: 'poetry'
+      - name: Install poetry via pipx
+        run: pipx install poetry
       - name: Install Python dependencies using Poetry
         run: make install-python-dependencies
       - name: Run integration tests
         run: |
-          git_hash=$(git rev-parse --short "$GITHUB_SHA")
-          image_name=ghcr.io/${{ github.repository_owner }}/runtime:$git_hash-${{ matrix.base_image }}
+          image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
           TEST_RUNTIME=eventstream \
@@ -158,11 +284,26 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-  # Checks that all runtime tests have passed
-  all_runtime_tests_passed:
+  # The two following jobs (named identically) are to check whether all the runtime tests have passed as the
+  # "All Runtime Tests Passed" is a required job for PRs to merge
+  # Due to this bug: https://github.com/actions/runner/issues/2566, we want to create a job that runs when the
+  # prerequisites have been cancelled or failed so merging is disallowed, otherwise Github considers "skipped" as "success"
+  runtime_tests_check_success:
     name: All Runtime Tests Passed
+    if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
     runs-on: ubuntu-latest
-    needs: [test_runtime, runtime_integration_tests_on_linux]
+    needs: [test_runtime_root, test_runtime_oh, runtime_integration_tests_on_linux]
     steps:
       - name: All tests passed
         run: echo "All runtime tests have passed successfully!"
+
+  runtime_tests_check_fail:
+    name: All Runtime Tests Passed
+    if: ${{ cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+    runs-on: ubuntu-latest
+    needs: [test_runtime_root, test_runtime_oh, runtime_integration_tests_on_linux]
+    steps:
+      - name: Some tests failed
+        run: |
+          echo "Some runtime tests failed or were cancelled"
+          exit 1

.github/workflows/py-unit-tests.yml

@@ -22,13 +22,21 @@ jobs:
         python-version: ['3.11']
     steps:
       - uses: actions/checkout@v4
-      - name: Install poetry via pipx
-        run: pipx install poetry
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      - name: Install poetry via pipx
+        run: pipx install poetry
       - name: Install Python dependencies using Poetry
         run: poetry install --without evaluation,llama-index
       - name: Install & Start Docker
@@ -111,7 +119,7 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Tests
-        run: poetry run pytest --forked --cov=agenthub --cov=openhands --cov-report=xml ./tests/unit
+        run: poetry run pytest --forked --cov=agenthub --cov=openhands --cov-report=xml -svv ./tests/unit
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:

.github/workflows/pypi-release.yml

@@ -0,0 +1,31 @@
+# Publishes the OpenHands PyPi package
+name: Publish PyPi Package
+
+# Triggered manually
+on:
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: 'Reason for manual trigger'
+        required: true
+        default: ''
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.4.1
+        with:
+          virtualenvs-in-project: true
+          virtualenvs-path: ~/.virtualenvs
+      - name: Install Poetry Dependencies
+        run: poetry install --no-interaction --no-root
+      - name: Build poetry project
+        run: poetry build -v
+      - name: publish
+        run: poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }}

.github/workflows/regenerate_integration_tests.yml

@@ -0,0 +1,70 @@
+
+name: Regenerate Integration Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      debug:
+        description: 'Enable debug mode'
+        type: boolean
+        default: true
+      log_to_file:
+        description: 'Enable logging to file'
+        type: boolean
+        default: true
+      force_regenerate_tests:
+        description: 'Force regeneration of tests'
+        type: boolean
+        default: false
+      force_use_llm:
+        description: 'Force use of LLM'
+        type: boolean
+        default: false
+
+jobs:
+  regenerate_integration_tests:
+    if: github.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/pypoetry
+          ~/.virtualenvs
+        key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+    - name: Install poetry via pipx
+      run: pipx install poetry
+    - name: Install Python dependencies using Poetry
+      run: make install-python-dependencies
+    - name: Build Environment
+      run: make build
+    - name: Regenerate integration tests
+      run: |
+        DEBUG=${{ inputs.debug }} \
+        LOG_TO_FILE=${{ inputs.log_to_file }} \
+        FORCE_REGENERATE_TESTS=${{ inputs.force_regenerate_tests }} \
+        FORCE_USE_LLM=${{ inputs.force_use_llm }} \
+        ./tests/integration/regenerate.sh
+    - name: Commit changes
+      run: |
+        if git diff --quiet --exit-code; then
+          echo "No changes to commit"
+          exit 0
+        fi
+
+        git config --global user.name 'github-actions[bot]'
+        git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+        git add .
+        # run it twice in case pre-commit makes changes
+        git commit -am "Regenerate integration tests" || git commit -am "Regenerate integration tests"
+        git push

.github/workflows/stale.yml

@@ -15,6 +15,7 @@ jobs:
           stale-issue-message: 'This issue is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
           days-before-stale: 30
+          exempt-issue-labels: 'tracked'
           close-issue-message: 'This issue was closed because it has been stalled for over 30 days with no activity.'
           close-pr-message: 'This PR was closed because it has been stalled for over 30 days with no activity.'
           days-before-close: 7

Development.md

@@ -97,3 +97,28 @@ Please refer to [this README](./tests/integration/README.md) for details.
 ### 9. Add or update dependency
 1. Add your dependency in `pyproject.toml` or use `poetry add xxx`
 2. Update the poetry.lock file via `poetry lock --no-update`
+
+## Develop inside Docker container
+
+TL;DR
+
+```bash
+make docker-dev
+```
+
+See more details [here](./containers/dev/README.md)
+
+If you are just interested in running `OpenHands` without installing all the required tools on your host.
+
+```bash
+make docker-run
+```
+
+If you do not have `make` on your host, run:
+
+```bash
+cd ./containers/dev
+./dev.sh
+```
+
+You do need [Docker](https://docs.docker.com/engine/install/) installed on your host though.

Makefile

@@ -2,8 +2,9 @@ SHELL=/bin/bash
 # Makefile for OpenHands project
 
 # Variables
+BACKEND_HOST ?= "127.0.0.1"
 BACKEND_PORT = 3000
-BACKEND_HOST = "127.0.0.1:$(BACKEND_PORT)"
+BACKEND_HOST_PORT = "$(BACKEND_HOST):$(BACKEND_PORT)"
 FRONTEND_PORT = 3001
 DEFAULT_WORKSPACE_DIR = "./workspace"
 DEFAULT_MODEL = "gpt-4o"
@@ -189,12 +190,12 @@ build-frontend:
 # Start backend
 start-backend:
 	@echo "$(YELLOW)Starting backend...$(RESET)"
-	@poetry run uvicorn openhands.server.listen:app --port $(BACKEND_PORT) --reload --reload-exclude "workspace/*"
+	@poetry run uvicorn openhands.server.listen:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) --reload --reload-exclude "workspace/*"
 
 # Start frontend
 start-frontend:
 	@echo "$(YELLOW)Starting frontend...$(RESET)"
-	@cd frontend && VITE_BACKEND_HOST=$(BACKEND_HOST) VITE_FRONTEND_PORT=$(FRONTEND_PORT) npm run start
+	@cd frontend && VITE_BACKEND_HOST=$(BACKEND_HOST_PORT) VITE_FRONTEND_PORT=$(FRONTEND_PORT) npm run start
 
 # Common setup for running the app (non-callable)
 _run_setup:
@@ -204,7 +205,7 @@ _run_setup:
 	fi
 	@mkdir -p logs
 	@echo "$(YELLOW)Starting backend server...$(RESET)"
-	@poetry run uvicorn openhands.server.listen:app --port $(BACKEND_PORT) &
+	@poetry run uvicorn openhands.server.listen:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) &
 	@echo "$(YELLOW)Waiting for the backend to start...$(RESET)"
 	@until nc -z localhost $(BACKEND_PORT); do sleep 0.1; done
 	@echo "$(GREEN)Backend started successfully.$(RESET)"
@@ -216,6 +217,20 @@ run:
 	@cd frontend && echo "$(BLUE)Starting frontend with npm...$(RESET)" && npm run start -- --port $(FRONTEND_PORT)
 	@echo "$(GREEN)Application started successfully.$(RESET)"
 
+# Run the app (in docker)
+docker-run: WORKSPACE_BASE ?= $(PWD)/workspace
+docker-run:
+	@if [ -f /.dockerenv ]; then \
+		echo "Running inside a Docker container. Exiting..."; \
+		exit 0; \
+	else \
+		echo "$(YELLOW)Running the app in Docker $(OPTIONS)...$(RESET)"; \
+		export WORKSPACE_BASE=${WORKSPACE_BASE}; \
+		export SANDBOX_USER_ID=$(shell id -u); \
+		export DATE=$(shell date +%Y%m%d%H%M%S); \
+		docker compose up $(OPTIONS); \
+	fi
+
 # Run the app (WSL mode)
 run-wsl:
 	@echo "$(YELLOW)Running the app in WSL mode...$(RESET)"
@@ -260,6 +275,10 @@ setup-config-prompts:
 		echo "    - nomic-embed-text"; \
 		echo "    - all-minilm"; \
 		echo "    - stable-code"; \
+		echo "    - bge-m3"; \
+		echo "    - bge-large"; \
+		echo "    - paraphrase-multilingual"; \
+		echo "    - snowflake-arctic-embed"; \
 		echo "  - Leave blank to default to 'BAAI/bge-small-en-v1.5' via huggingface"; \
 		read -p "> " llm_embedding_model; \
 		echo "embedding_model=\"$$llm_embedding_model\"" >> $(CONFIG_FILE).tmp; \
@@ -276,6 +295,16 @@ setup-config-prompts:
 		fi
 
 
+# Develop in container
+docker-dev:
+	@if [ -f /.dockerenv ]; then \
+		echo "Running inside a Docker container. Exiting..."; \
+		exit 0; \
+	else \
+		echo "$(YELLOW)Build and run in Docker $(OPTIONS)...$(RESET)"; \
+		./containers/dev/dev.sh $(OPTIONS); \
+	fi
+
 # Clean up all caches
 clean:
 	@echo "$(YELLOW)Cleaning up caches...$(RESET)"
@@ -294,7 +323,10 @@ help:
 	@echo "  $(GREEN)start-frontend$(RESET)      - Start the frontend server for the OpenHands project."
 	@echo "  $(GREEN)run$(RESET)                 - Run the OpenHands application, starting both backend and frontend servers."
 	@echo "                        Backend Log file will be stored in the 'logs' directory."
+	@echo "  $(GREEN)docker-dev$(RESET)          - Build and run the OpenHands application in Docker."
+	@echo "  $(GREEN)docker-run$(RESET)          - Run the OpenHands application, starting both backend and frontend servers in Docker."
 	@echo "  $(GREEN)help$(RESET)                - Display this help message, providing information on available targets."
 
 # Phony targets
 .PHONY: build check-dependencies check-python check-npm check-docker check-poetry install-python-dependencies install-frontend-dependencies install-pre-commit-hooks lint start-backend start-frontend run run-wsl setup-config setup-config-prompts help
+.PHONY: docker-dev docker-run

README.md

@@ -1,65 +1,49 @@
 <a name="readme-top"></a>
 
-<!--
-*** Thanks for checking out the Best-README-Template. If you have a suggestion
-*** that would make this better, please fork the repo and create a pull request
-*** or simply open an issue with the tag "enhancement".
-*** Don't forget to give the project a star!
-*** Thanks again! Now go create something AMAZING! :D
--->
+<div align="center">
+  <img src="./docs/static/img/logo.png" alt="Logo" width="200">
+  <h1 align="center">OpenHands: Code Less, Make More</h1>
+</div>
 
-<!-- PROJECT SHIELDS -->
-<!--
-*** I'm using markdown "reference style" links for readability.
-*** Reference links are enclosed in brackets [ ] instead of parentheses ( ).
-*** See the bottom of this document for the declaration of the reference variables
-*** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
-*** https://www.markdownguide.org/basic-syntax/#reference-style-links
--->
 
 <div align="center">
   <a href="https://github.com/All-Hands-AI/OpenHands/graphs/contributors"><img src="https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Contributors"></a>
-  <a href="https://github.com/All-Hands-AI/OpenHands/network/members"><img src="https://img.shields.io/github/forks/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Forks"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/stargazers"><img src="https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Stargazers"></a>
-  <a href="https://github.com/All-Hands-AI/OpenHands/issues"><img src="https://img.shields.io/github/issues/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Issues"></a>
+  <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
-  <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=blue" alt="Credits"></a>
   <br/>
   <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
-  <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge"></a>
-</div>
-
-<!-- PROJECT LOGO -->
-<div align="center">
-  <img src="./docs/static/img/logo.png" alt="Logo" width="200" height="200">
-  <h1 align="center">OpenHands: Code Less, Make More</h1>
-  <a href="https://docs.all-hands.dev/modules/usage/intro"><img src="https://img.shields.io/badge/Documentation-OpenHands-blue?logo=googledocs&logoColor=white&style=for-the-badge" alt="Check out the documentation"></a>
-  <a href="https://arxiv.org/abs/2407.16741"><img src="https://img.shields.io/badge/Paper-%20on%20Arxiv-red?logo=arxiv&style=for-the-badge" alt="Paper on Arxiv"></a>
+  <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
   <br/>
-  <a href="https://huggingface.co/spaces/OpenHands/evaluation"><img src="https://img.shields.io/badge/Evaluation-Benchmark%20on%20HF%20Space-green?logo=huggingface&style=for-the-badge" alt="Evaluation Benchmark"></a>
+  <a href="https://docs.all-hands.dev/modules/usage/getting-started"><img src="https://img.shields.io/badge/Documentation-000?logo=googledocs&logoColor=FFE165&style=for-the-badge" alt="Check out the documentation"></a>
+  <a href="https://arxiv.org/abs/2407.16741"><img src="https://img.shields.io/badge/Paper%20on%20Arxiv-000?logoColor=FFE165&logo=arxiv&style=for-the-badge" alt="Paper on Arxiv"></a>
+  <a href="https://huggingface.co/spaces/OpenHands/evaluation"><img src="https://img.shields.io/badge/Benchmark%20score-000?logoColor=FFE165&logo=huggingface&style=for-the-badge" alt="Evaluation Benchmark Score"></a>
+  <hr>
 </div>
-<hr>
 
-Welcome to OpenHands, a platform for autonomous software engineers, powered by AI and LLMs (previously called "OpenDevin").
+Welcome to OpenHands (formerly OpenDevin), a platform for software development agents powered by AI.
 
-OpenHands agents collaborate with human developers to write code, fix bugs, and ship features.
+OpenHands agents can do anything a human developer can: modify code, run commands, browse the web,
+call APIs, and yes—even copy code snippets from StackOverflow.
+
+Learn more at [docs.all-hands.dev](https://docs.all-hands.dev), or jump to the [Quick Start](#-quick-start).
 
 ![App screenshot](./docs/static/img/screenshot.png)
 
-## ⚡ Getting Started
-OpenHands works best with Docker version 26.0.0+ (Docker Desktop 4.31.0+).
-You must be using Linux, Mac OS, or WSL on Windows.
+## ⚡ Quick Start
 
-To start OpenHands in a docker container, run the following commands in your terminal:
+The easiest way to run OpenHands is in Docker. You can change `WORKSPACE_BASE` below to
+point OpenHands to existing code that you'd like to modify.
 
-> [!WARNING]
-> When you run the following command, files in `./workspace` may be modified or deleted.
+See the [Getting Started](https://docs.all-hands.dev/modules/usage/getting-started) guide for
+system requirements and more information.
 
 ```bash
-WORKSPACE_BASE=$(pwd)/workspace
-docker run -it \
-    --pull=always \
+export WORKSPACE_BASE=$(pwd)/workspace
+
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.9-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -v $WORKSPACE_BASE:/opt/workspace_base \
@@ -70,29 +54,23 @@ docker run -it \
     ghcr.io/all-hands-ai/openhands:0.9
 ```
 
-> [!NOTE]
-> This command pulls the `0.9` tag, which represents the most recent stable release of OpenHands. You have other options as well:
-> - For a specific release version, use `ghcr.io/all-hands-ai/openhands:<OpenHands_version>` (replace <OpenHands_version> with the desired version number).
-> - For the most up-to-date development version, use `ghcr.io/all-hands-ai/openhands:main`. This version may be **(unstable!)** and is recommended for testing or development purposes only.
->
-> Choose the tag that best suits your needs based on stability requirements and desired features.
+You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
 
-You'll find OpenHands running at [http://localhost:3000](http://localhost:3000) with access to `./workspace`. To have OpenHands operate on your code, place it in `./workspace`.
-OpenHands will only have access to this workspace folder. The rest of your system will not be affected as it runs in a secured docker sandbox.
+You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
+or as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode).
 
-Upon opening OpenHands, you must select the appropriate `Model` and enter the `API Key` within the settings that should pop up automatically. These can be set at any time by selecting
-the `Settings` button (gear icon) in the UI. If the required `Model` does not exist in the list, you can manually enter it in the text box.
+Visit [Getting Started](https://docs.all-hands.dev/modules/usage/getting-started) for more information and setup instructions.
 
-For the development workflow, see [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md).
+If you want to modify the OpenHands source code, check out [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md).
 
-Are you having trouble? Check out our [Troubleshooting Guide](https://docs.all-hands.dev/modules/usage/troubleshooting).
+Having issues? The [Troubleshooting Guide](https://docs.all-hands.dev/modules/usage/troubleshooting) can help.
 
-## 🚀 Documentation
+## 📖 Documentation
 
 To learn more about the project, and for tips on using OpenHands,
-**check out our [documentation](https://docs.all-hands.dev/modules/usage/intro)**.
+**check out our [documentation](https://docs.all-hands.dev/modules/usage/getting-started)**.
 
-There you'll find resources on how to use different LLM providers (like ollama and Anthropic's Claude),
+There you'll find resources on how to use different LLM providers,
 troubleshooting resources, and advanced configuration options.
 
 ## 🤝 How to Contribute
@@ -127,17 +105,6 @@ Let's make software engineering better together!
 
 Distributed under the MIT License. See [`LICENSE`](./LICENSE) for more information.
 
-[contributors-shield]: https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge
-[contributors-url]: https://github.com/All-Hands-AI/OpenHands/graphs/contributors
-[forks-shield]: https://img.shields.io/github/forks/All-Hands-AI/OpenHands?style=for-the-badge
-[forks-url]: https://github.com/All-Hands-AI/OpenHands/network/members
-[stars-shield]: https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge
-[stars-url]: https://github.com/All-Hands-AI/OpenHands/stargazers
-[issues-shield]: https://img.shields.io/github/issues/All-Hands-AI/OpenHands?style=for-the-badge
-[issues-url]: https://github.com/All-Hands-AI/OpenHands/issues
-[license-shield]: https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge
-[license-url]: https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE
-
 ## 🙏 Acknowledgements
 
 OpenHands is built by a large number of contributors, and every contribution is greatly appreciated! We also build upon other open source projects, and we are deeply thankful for their work.

agenthub/browsing_agent/browsing_agent.py

@@ -65,10 +65,15 @@ In order to accomplish my goal I need to send the information asked back to the
 """
 
 
-def get_prompt(error_prefix: str, cur_axtree_txt: str, prev_action_str: str) -> str:
+def get_prompt(
+    error_prefix: str, cur_url: str, cur_axtree_txt: str, prev_action_str: str
+) -> str:
     prompt = f"""\
 {error_prefix}
 
+# Current Page URL:
+{cur_url}
+
 # Current Accessibility Tree:
 {cur_axtree_txt}
 
@@ -139,6 +144,7 @@ class BrowsingAgent(Agent):
         """
         messages: list[Message] = []
         prev_actions = []
+        cur_url = ''
         cur_axtree_txt = ''
         error_prefix = ''
         last_obs = None
@@ -179,6 +185,9 @@ class BrowsingAgent(Agent):
                 self.error_accumulator += 1
                 if self.error_accumulator > 5:
                     return MessageAction('Too many errors encountered. Task failed.')
+
+            cur_url = last_obs.url
+
             try:
                 cur_axtree_txt = flatten_axtree_to_str(
                     last_obs.axtree_object,
@@ -204,11 +213,11 @@ class BrowsingAgent(Agent):
 
         messages.append(Message(role='system', content=[TextContent(text=system_msg)]))
 
-        prompt = get_prompt(error_prefix, cur_axtree_txt, prev_action_str)
+        prompt = get_prompt(error_prefix, cur_url, cur_axtree_txt, prev_action_str)
         messages.append(Message(role='user', content=[TextContent(text=prompt)]))
-        logger.debug(prompt)
+
         response = self.llm.completion(
-            messages=[message.model_dump() for message in messages],
+            messages=self.llm.format_messages_for_llm(messages),
             temperature=0.0,
             stop=[')```', ')\n```'],
         )

agenthub/browsing_agent/prompt.py

@@ -57,7 +57,7 @@ class Flags:
 
     @classmethod
     def from_dict(self, flags_dict):
-        """Helper for JSON serializble requirement."""
+        """Helper for JSON serializable requirement."""
         if isinstance(flags_dict, Flags):
             return flags_dict
 
@@ -354,7 +354,7 @@ and executed by a program, make sure to follow the formatting instructions.
         self._prompt += '\n'.join(
             [
                 f"""\
- - [{msg['role']}] {msg['message']}"""
+ - [{msg['role']}], {msg['message']}"""
                 for msg in chat_messages
             ]
         )

agenthub/browsing_agent/response_parser.py

@@ -24,9 +24,9 @@ class BrowsingResponseParser(ResponseParser):
         if action_str is None:
             return ''
         action_str = action_str.strip()
-        if not action_str.endswith('```'):
+        if action_str and not action_str.endswith('```'):
             action_str = action_str + ')```'
-        logger.info(action_str)
+        logger.debug(action_str)
         return action_str
 
     def parse_action(self, action_str: str) -> Action:

agenthub/codeact_agent/README.md

@@ -15,7 +15,7 @@ The conceptual idea is illustrated below. At each turn, the agent can:
 
 To make the CodeAct agent more powerful with only access to `bash` action space, CodeAct agent leverages OpenHands's plugin system:
 - [Jupyter plugin](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/jupyter): for IPython execution via bash command
-- [SWE-agent tool plugin](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/swe_agent_commands): Powerful bash command line tools for software development tasks introduced by [swe-agent](https://github.com/princeton-nlp/swe-agent).
+- [Agent Skills plugin](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/agent_skills): Powerful bash command line tools for software development tasks introduced by [swe-agent](https://github.com/princeton-nlp/swe-agent).
 
 ## Demo
 

agenthub/codeact_agent/codeact_agent.py

@@ -1,9 +1,11 @@
 import os
+from itertools import islice
 
 from agenthub.codeact_agent.action_parser import CodeActResponseParser
 from openhands.controller.agent import Agent
 from openhands.controller.state.state import State
 from openhands.core.config import AgentConfig
+from openhands.core.logger import openhands_logger as logger
 from openhands.core.message import ImageContent, Message, TextContent
 from openhands.events.action import (
     Action,
@@ -17,6 +19,7 @@ from openhands.events.observation import (
     AgentDelegateObservation,
     CmdOutputObservation,
     IPythonRunCellObservation,
+    UserRejectObservation,
 )
 from openhands.events.observation.error import ErrorObservation
 from openhands.events.observation.observation import Observation
@@ -27,6 +30,7 @@ from openhands.runtime.plugins import (
     JupyterRequirement,
     PluginRequirement,
 )
+from openhands.utils.microagent import MicroAgent
 from openhands.utils.prompt import PromptManager
 
 
@@ -73,10 +77,21 @@ class CodeActAgent(Agent):
         """
         super().__init__(llm, config)
         self.reset()
+
+        self.micro_agent = (
+            MicroAgent(
+                os.path.join(
+                    os.path.dirname(__file__), 'micro', f'{config.micro_agent_name}.md'
+                )
+            )
+            if config.micro_agent_name
+            else None
+        )
+
         self.prompt_manager = PromptManager(
             prompt_dir=os.path.join(os.path.dirname(__file__)),
             agent_skills_docs=AgentSkillsRequirement.documentation,
-            micro_agent_name=None,  # TODO: implement micro-agent
+            micro_agent=self.micro_agent,
         )
 
     def action_to_str(self, action: Action) -> str:
@@ -104,7 +119,11 @@ class CodeActAgent(Agent):
         ):
             content = [TextContent(text=self.action_to_str(action))]
 
-            if isinstance(action, MessageAction) and action.images_urls:
+            if (
+                self.llm.vision_is_active()
+                and isinstance(action, MessageAction)
+                and action.images_urls
+            ):
                 content.append(ImageContent(image_urls=action.images_urls))
 
             return Message(
@@ -114,14 +133,15 @@ class CodeActAgent(Agent):
 
     def get_observation_message(self, obs: Observation) -> Message | None:
         max_message_chars = self.llm.config.max_message_chars
+        obs_prefix = 'OBSERVATION:\n'
         if isinstance(obs, CmdOutputObservation):
-            text = 'OBSERVATION:\n' + truncate_content(obs.content, max_message_chars)
+            text = obs_prefix + truncate_content(obs.content, max_message_chars)
             text += (
                 f'\n[Command {obs.command_id} finished with exit code {obs.exit_code}]'
             )
             return Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, IPythonRunCellObservation):
-            text = 'OBSERVATION:\n' + obs.content
+            text = obs_prefix + obs.content
             # replace base64 images with a placeholder
             splitted = text.split('\n')
             for i, line in enumerate(splitted):
@@ -133,14 +153,16 @@ class CodeActAgent(Agent):
             text = truncate_content(text, max_message_chars)
             return Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, AgentDelegateObservation):
-            text = 'OBSERVATION:\n' + truncate_content(
-                str(obs.outputs), max_message_chars
-            )
+            text = obs_prefix + truncate_content(str(obs.outputs), max_message_chars)
             return Message(role='user', content=[TextContent(text=text)])
         elif isinstance(obs, ErrorObservation):
-            text = 'OBSERVATION:\n' + truncate_content(obs.content, max_message_chars)
+            text = obs_prefix + truncate_content(obs.content, max_message_chars)
             text += '\n[Error occurred in processing last action]'
             return Message(role='user', content=[TextContent(text=text)])
+        elif isinstance(obs, UserRejectObservation):
+            text = 'OBSERVATION:\n' + truncate_content(obs.content, max_message_chars)
+            text += '\n[Last action has been rejected by the user]'
+            return Message(role='user', content=[TextContent(text=text)])
         else:
             # If an observation message is not returned, it will cause an error
             # when the LLM tries to return the next message
@@ -171,27 +193,51 @@ class CodeActAgent(Agent):
 
         # prepare what we want to send to the LLM
         messages = self._get_messages(state)
-
-        response = self.llm.completion(
-            messages=[message.model_dump() for message in messages],
-            stop=[
+        params = {
+            'messages': self.llm.format_messages_for_llm(messages),
+            'stop': [
                 '</execute_ipython>',
                 '</execute_bash>',
                 '</execute_browse>',
             ],
-            temperature=0.0,
-        )
+            'temperature': 0.0,
+        }
+
+        if self.llm.is_caching_prompt_active():
+            params['extra_headers'] = {
+                'anthropic-beta': 'prompt-caching-2024-07-31',
+            }
+
+        try:
+            response = self.llm.completion(**params)
+        except Exception as e:
+            logger.error(f'{e}')
+            error_message = '{}: {}'.format(type(e).__name__, str(e).split('\n')[0])
+            return AgentFinishAction(
+                thought=f'Agent encountered an error while processing the last action.\nError: {error_message}\nPlease try again.'
+            )
+
         return self.action_parser.parse(response)
 
     def _get_messages(self, state: State) -> list[Message]:
         messages: list[Message] = [
             Message(
                 role='system',
-                content=[TextContent(text=self.prompt_manager.system_message)],
+                content=[
+                    TextContent(
+                        text=self.prompt_manager.system_message,
+                        cache_prompt=self.llm.is_caching_prompt_active(),  # Cache system prompt
+                    )
+                ],
             ),
             Message(
                 role='user',
-                content=[TextContent(text=self.prompt_manager.initial_user_message)],
+                content=[
+                    TextContent(
+                        text=self.prompt_manager.initial_user_message,
+                        cache_prompt=self.llm.is_caching_prompt_active(),  # if the user asks the same query,
+                    )
+                ],
             ),
         ]
 
@@ -208,42 +254,38 @@ class CodeActAgent(Agent):
             if message:
                 # handle error if the message is the SAME role as the previous message
                 # litellm.exceptions.BadRequestError: litellm.BadRequestError: OpenAIException - Error code: 400 - {'detail': 'Only supports u/a/u/a/u...'}
-                # there should not have two consecutive messages from the same role
+                # there shouldn't be two consecutive messages from the same role
                 if messages and messages[-1].role == message.role:
                     messages[-1].content.extend(message.content)
                 else:
                     messages.append(message)
 
-        # the latest user message is important:
+        # Add caching to the last 2 user messages
+        if self.llm.is_caching_prompt_active():
+            user_turns_processed = 0
+            for message in reversed(messages):
+                if message.role == 'user' and user_turns_processed < 2:
+                    message.content[
+                        -1
+                    ].cache_prompt = True  # Last item inside the message content
+                    user_turns_processed += 1
+
+        # The latest user message is important:
         # we want to remind the agent of the environment constraints
         latest_user_message = next(
-            (
-                m
-                for m in reversed(messages)
-                if m.role == 'user'
-                and any(isinstance(c, TextContent) for c in m.content)
+            islice(
+                (
+                    m
+                    for m in reversed(messages)
+                    if m.role == 'user'
+                    and any(isinstance(c, TextContent) for c in m.content)
+                ),
+                1,
             ),
             None,
         )
-
-        # Get the last user text inside content
         if latest_user_message:
-            latest_user_message_text = next(
-                (
-                    t
-                    for t in reversed(latest_user_message.content)
-                    if isinstance(t, TextContent)
-                )
-            )
-            # add a reminder to the prompt
             reminder_text = f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task. When finished reply with <finish></finish>.'
-
-            if latest_user_message_text:
-                latest_user_message_text.text = (
-                    latest_user_message_text.text + reminder_text
-                )
-            else:
-                latest_user_message_text = TextContent(text=reminder_text)
-                latest_user_message.content.append(latest_user_message_text)
+            latest_user_message.content.append(TextContent(text=reminder_text))
 
         return messages

agenthub/codeact_agent/micro/github.md

@@ -0,0 +1,69 @@
+---
+name: github
+agent: CodeActAgent
+require_env_var:
+    SANDBOX_ENV_GITHUB_TOKEN: "Create a GitHub Personal Access Token (https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) and set it as SANDBOX_GITHUB_TOKEN in your environment variables."
+---
+
+# How to Interact with Github
+
+## Environment Variable Available
+
+- `GITHUB_TOKEN`: A read-only token for Github.
+
+## Using GitHub's RESTful API
+
+Use `curl` with the `GITHUB_TOKEN` to interact with GitHub's API. Here are some common operations:
+
+Here's a template for API calls:
+
+```sh
+curl -H "Authorization: token $GITHUB_TOKEN" \
+    "https://api.github.com/{endpoint}"
+```
+
+First replace `{endpoint}` with the specific API path. Common operations:
+
+1. View an issue or pull request:
+   - Issues: `/repos/{owner}/{repo}/issues/{issue_number}`
+   - Pull requests: `/repos/{owner}/{repo}/pulls/{pull_request_number}`
+
+2. List repository issues or pull requests:
+   - Issues: `/repos/{owner}/{repo}/issues`
+   - Pull requests: `/repos/{owner}/{repo}/pulls`
+
+3. Search issues or pull requests:
+   - `/search/issues?q=repo:{owner}/{repo}+is:{type}+{search_term}+state:{state}`
+   - Replace `{type}` with `issue` or `pr`
+
+4. List repository branches:
+   `/repos/{owner}/{repo}/branches`
+
+5. Get commit details:
+   `/repos/{owner}/{repo}/commits/{commit_sha}`
+
+6. Get repository details:
+   `/repos/{owner}/{repo}`
+
+7. Get user information:
+   `/user`
+
+8. Search repositories:
+   `/search/repositories?q={query}`
+
+9. Get rate limit status:
+   `/rate_limit`
+
+Replace `{owner}`, `{repo}`, `{commit_sha}`, `{issue_number}`, `{pull_request_number}`,
+`{search_term}`, `{state}`, and `{query}` with appropriate values.
+
+## Important Notes
+
+1. Always use the GitHub API for operations instead of a web browser.
+2. The `GITHUB_TOKEN` is read-only. Avoid operations that require write access.
+3. Git config (username and email) is pre-set. Do not modify.
+4. Edit and test code locally. Never push directly to remote.
+5. Verify correct branch before committing.
+6. Commit changes frequently.
+7. If the issue or task is ambiguous or lacks sufficient detail, always request clarification from the user before proceeding.
+8. You should avoid using command line tools like `sed` for file editing.

agenthub/codeact_agent/system_prompt.j2

@@ -1,16 +1,19 @@
 {% set MINIMAL_SYSTEM_PREFIX %}
-A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
-The assistant can use an interactive Python (Jupyter Notebook) environment, executing code with <execute_ipython>.
+A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed answers to the user's questions.
+The assistant can use a Python environment with <execute_ipython>, e.g.:
 <execute_ipython>
 print("Hello World!")
 </execute_ipython>
-The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
+The assistant can execute bash commands wrapped with <execute_bash>, e.g. <execute_bash> ls </execute_bash>.
+If a bash command returns exit code `-1`, this means the process is not yet finished.
+The assistant must then send a second <execute_bash>. The second <execute_bash> can be empty
+(which will retrieve any additional logs), or it can contain text to be sent to STDIN of the running process,
+or it can contain the text `ctrl+c` to interrupt the process.
 
-For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
-Important, however: do not run interactive commands. You do not have access to stdin.
-Also, you need to handle commands that may run indefinitely and not return a result. For such cases, you should redirect the output to a file and run the command in the background to avoid blocking the execution.
-For example, to run a Python script that might run indefinitely without returning immediately, you can use the following format: <execute_bash> python3 app.py > server.log 2>&1 & </execute_bash>
-Also, if a command execution result saying like: Command: "npm start" timed out. Sending SIGINT to the process, you should also retry with running the command in the background.
+For commands that may run indefinitely, the output should be redirected to a file and the command run
+in the background, e.g. <execute_bash> python3 app.py > server.log 2>&1 & </execute_bash>
+If a command execution result says "Command timed out. Sending SIGINT to the process",
+the assistant should retry running the command in the background.
 {% endset %}
 {% set BROWSING_PREFIX %}
 The assistant can browse the Internet with <execute_browse> and </execute_browse>.
@@ -24,7 +27,14 @@ The assistant can install Python packages using the %pip magic command in an IPy
 {% set COMMAND_DOCS %}
 Apart from the standard Python library, the assistant can also use the following functions (already imported) in <execute_ipython> environment:
 {{ agent_skills_docs }}
-Please note that THE `edit_file_by_replace`, `append_file` and `insert_content_at_line` FUNCTIONS REQUIRE PROPER INDENTATION. If the assistant would like to add the line '        print(x)', it must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run.
+IMPORTANT:
+- `open_file` only returns the first 100 lines of the file by default! The assistant MUST use `scroll_down` repeatedly to read the full file BEFORE making edits!
+- The assistant shall adhere to THE `edit_file_by_replace`, `append_file` and `insert_content_at_line` FUNCTIONS REQUIRING PROPER INDENTATION. If the assistant would like to add the line '        print(x)', it must fully write the line out, with all leading spaces before the code!
+- Indentation is important and code that is not indented correctly will fail and require fixing before it can be run.
+- Any code issued should be less than 50 lines to avoid context being cut off!
+- After EVERY `create_file` the method `append_file` shall be used to write the FIRST content!
+- For `edit_file_by_replace` NEVER provide empty parameters!
+- For `edit_file_by_replace` the file must be read fully before any replacements!
 {% endset %}
 {% set SYSTEM_SUFFIX %}
 Responses should be concise.
@@ -32,7 +42,8 @@ The assistant should attempt fewer things at a time instead of putting too many
 Include ONLY ONE <execute_ipython>, <execute_bash>, or <execute_browse> per response, unless the assistant is finished with the task or needs more input or action from the user in order to proceed.
 If the assistant is finished with the task you MUST include <finish></finish> in your response.
 IMPORTANT: Execute code using <execute_ipython>, <execute_bash>, or <execute_browse> whenever possible.
-The assistant should utilize full file paths and the 'pwd' command to prevent path-related errors. The assistant should refrain from excessive apologies in its responses.
+The assistant should utilize full file paths and the `pwd` command to prevent path-related errors.
+The assistant must avoid apologies and thanks in its responses.
 
 {% endset %}
 {# Combine all parts without newlines between them #}

agenthub/codeact_swe_agent/codeact_swe_agent.py

@@ -94,7 +94,11 @@ class CodeActSWEAgent(Agent):
         ):
             content = [TextContent(text=self.action_to_str(action))]
 
-            if isinstance(action, MessageAction) and action.images_urls:
+            if (
+                self.llm.vision_is_active()
+                and isinstance(action, MessageAction)
+                and action.images_urls
+            ):
                 content.append(ImageContent(image_urls=action.images_urls))
 
             return Message(
@@ -156,9 +160,8 @@ class CodeActSWEAgent(Agent):
 
         # prepare what we want to send to the LLM
         messages: list[Message] = self._get_messages(state)
-
         response = self.llm.completion(
-            messages=[message.model_dump() for message in messages],
+            messages=self.llm.format_messages_for_llm(messages),
             stop=[
                 '</execute_ipython>',
                 '</execute_bash>',

agenthub/micro/agent.py

@@ -73,10 +73,13 @@ class MicroAgent(Agent):
             latest_user_message=last_user_message,
         )
         content = [TextContent(text=prompt)]
-        if last_image_urls:
+        if self.llm.vision_is_active() and last_image_urls:
             content.append(ImageContent(image_urls=last_image_urls))
         message = Message(role='user', content=content)
-        resp = self.llm.completion(messages=[message.model_dump()])
+        resp = self.llm.completion(
+            messages=self.llm.format_messages_for_llm(message),
+            temperature=0.0,
+        )
         action_resp = resp['choices'][0]['message']['content']
         action = parse_response(action_resp)
         return action

agenthub/planner_agent/agent.py

@@ -46,8 +46,8 @@ class PlannerAgent(Agent):
             state, self.llm.config.max_message_chars
         )
         content = [TextContent(text=prompt)]
-        if image_urls:
+        if self.llm.vision_is_active() and image_urls:
             content.append(ImageContent(image_urls=image_urls))
         message = Message(role='user', content=content)
-        resp = self.llm.completion(messages=[message.model_dump()])
+        resp = self.llm.completion(messages=self.llm.format_messages_for_llm(message))
         return self.response_parser.parse(resp)

compose.yml

@@ -0,0 +1,22 @@
+#
+services:
+  openhands:
+    build:
+      context: ./
+      dockerfile: ./containers/app/Dockerfile
+    image: openhands:latest
+    container_name: openhands-app-${DATE:-}
+    environment:
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
+      - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
+    ports:
+      - "3000:3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${WORKSPACE_BASE:-$PWD/workspace}:/opt/workspace_base
+    pull_policy: build
+    stdin_open: true
+    tty: true

config.template.toml

@@ -64,6 +64,15 @@ workspace_base = "./workspace"
 # Name of the default agent
 #default_agent = "CodeActAgent"
 
+# JWT secret for authentication
+#jwt_secret = ""
+
+# Restrict file types for file uploads
+#file_uploads_restrict_file_types = false
+
+# List of allowed file extensions for uploads
+#file_uploads_allowed_extensions = [".*"]
+
 #################################### LLM #####################################
 # Configuration for LLM models (group name starts with 'llm')
 # use 'llm' for the default LLM config
@@ -117,14 +126,31 @@ embedding_model = ""
 # Model to use
 model = "gpt-4o"
 
-# Number of retries to attempt
-#num_retries = 5
+# Number of retries to attempt when an operation fails with the LLM.
+# Increase this value to allow more attempts before giving up
+#num_retries = 8
 
-# Retry maximum wait time
-#retry_max_wait = 60
+# Maximum wait time (in seconds) between retry attempts
+# This caps the exponential backoff to prevent excessively long
+#retry_max_wait = 120
 
-# Retry minimum wait time
-#retry_min_wait = 3
+# Minimum wait time (in seconds) between retry attempts
+# This sets the initial delay before the first retry
+#retry_min_wait = 15
+
+# Multiplier for exponential backoff calculation
+# The wait time increases by this factor after each failed attempt
+# A value of 2.0 means each retry waits twice as long as the previous one
+#retry_multiplier = 2.0
+
+# Drop any unmapped (unsupported) params without causing an exception
+#drop_params = false
+
+# Using the prompt caching feature provided by the LLM
+#caching_prompt = false
+
+# Base URL for the OLLAMA API
+#ollama_base_url = ""
 
 # Temperature for the API
 #temperature = 0.0
@@ -135,12 +161,15 @@ model = "gpt-4o"
 # Top p for the API
 #top_p = 0.5
 
-[llm.gpt3]
+# If model is vision capable, this option allows to disable image processing (useful for cost reduction).
+#disable_vision = true
+
+[llm.gpt4o-mini]
 # API key to use
 api_key = "your-api-key"
 
 # Model to use
-model = "gpt-3.5"
+model = "gpt-4o-mini"
 
 #################################### Agent ###################################
 # Configuration for agents (group name starts with 'agent')
@@ -149,6 +178,9 @@ model = "gpt-3.5"
 # agent.CodeActAgent
 ##############################################################################
 [agent]
+# Name of the micro agent to use for this agent
+#micro_agent_name = ""
+
 # Memory enabled
 #memory_enabled = false
 
@@ -182,6 +214,18 @@ llm_config = 'gpt3'
 # Enable auto linting after editing
 #enable_auto_lint = false
 
+# Whether to initialize plugins
+#initialize_plugins = true
+
+# Extra dependencies to install in the runtime image
+#runtime_extra_deps = ""
+
+# Environment variables to set at the launch of the runtime
+#runtime_startup_env_vars = {}
+
+# BrowserGym environment to use for evaluation
+#browsergym_eval_env = ""
+
 #################################### Security ###################################
 # Configuration for security features
 ##############################################################################

containers/build.sh

@@ -20,6 +20,7 @@ cache_tag="$cache_tag_base"
 if [[ -n $GITHUB_SHA ]]; then
   git_hash=$(git rev-parse --short "$GITHUB_SHA")
   tags+=("$git_hash")
+  tags+=("$GITHUB_SHA")
 fi
 
 if [[ -n $GITHUB_REF_NAME ]]; then

containers/dev/Dockerfile

@@ -0,0 +1,124 @@
+# syntax=docker/dockerfile:1
+
+###
+FROM ubuntu:22.04 AS dind
+
+# https://docs.docker.com/engine/install/ubuntu/
+RUN apt-get update && apt-get install -y \
+	ca-certificates \
+	curl \
+	&& install -m 0755 -d /etc/apt/keyrings \
+	&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
+	&& chmod a+r /etc/apt/keyrings/docker.asc \
+	&& echo \
+		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+		$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+RUN apt-get update && apt-get install -y \
+	docker-ce \
+	docker-ce-cli \
+	containerd.io \
+	docker-buildx-plugin \
+	docker-compose-plugin \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+###
+FROM dind AS openhands
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+#
+RUN apt-get update && apt-get install -y \
+	bash \
+    build-essential \
+    curl \
+	git \
+	git-lfs \
+	software-properties-common \
+	make \
+    netcat \
+    sudo \
+	wget \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+# https://github.com/cli/cli/blob/trunk/docs/install_linux.md
+RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+	&& apt-get update && apt-get -y install \
+    gh \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get clean \
+  && apt-get autoremove -y
+
+# Python 3.11
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.11 python3.11-venv python3.11-dev python3-pip \
+    && ln -s /usr/bin/python3.11 /usr/bin/python
+
+# NodeJS >= 18.17.1
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
+    && apt-get install -y nodejs
+
+# Poetry >= 1.8
+RUN curl -fsSL https://install.python-poetry.org | python3.11 - \
+    && ln -s ~/.local/bin/poetry /usr/local/bin/poetry
+
+#
+RUN <<EOF
+#!/bin/bash
+printf "#!/bin/bash
+set +x
+uname -a
+docker --version
+gh --version | head -n 1
+git --version
+#
+python --version
+echo node `node --version`
+echo npm `npm --version`
+poetry --version
+netcat -h 2>&1 | head -n 1
+" > /version.sh
+chmod a+x /version.sh
+EOF
+
+###
+FROM openhands AS dev
+
+RUN apt-get update && apt-get install -y \
+	dnsutils \
+	file \
+	iproute2 \
+	jq \
+	lsof \
+	ripgrep \
+	silversearcher-ag \
+	vim \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& apt-get clean \
+	&& apt-get autoremove -y
+
+WORKDIR /app
+
+# cache build dependencies
+RUN \
+  --mount=type=bind,source=./,target=/app/ \
+  <<EOF
+#!/bin/bash
+make -s clean
+make -s check-dependencies
+make -s install-python-dependencies
+
+# NOTE
+# node_modules are .dockerignore-d therefore not mountable
+# make -s install-frontend-dependencies
+EOF
+
+#
+CMD ["bash"]

containers/dev/README.md

@@ -0,0 +1,54 @@
+# Develop in Docker
+
+Install [Docker](https://docs.docker.com/engine/install/) on your host machine and run:
+
+```bash
+make docker-dev
+# same as:
+cd ./containers/dev
+./dev.sh
+```
+
+It could take some time if you are running for the first time as Docker will pull all the  tools required for building OpenHands. The next time you run again, it should be instant.
+
+## Build and run
+
+If everything goes well, you should be inside a container after Docker finishes building the `openhands:dev` image similar to the following:
+
+```bash
+Build and run in Docker ...
+root@93fc0005fcd2:/app#
+```
+
+You may now proceed with the normal [build and run](../../Development.md) workflow as if you were on the host.
+
+## Make changes
+
+The source code on the host is mounted as `/app` inside docker. You may edit the files as usual either inside the Docker container or on your host with your favorite IDE/editors.
+
+The following are also mapped as readonly from your host:
+
+```yaml
+# host credentials
+- $HOME/.git-credentials:/root/.git-credentials:ro
+- $HOME/.gitconfig:/root/.gitconfig:ro
+- $HOME/.npmrc:/root/.npmrc:ro
+```
+
+## VSCode
+
+Alternatively, if you use VSCode, you could also [attach to the running container](https://code.visualstudio.com/docs/devcontainers/attach-container).
+
+See details for [developing in docker](https://code.visualstudio.com/docs/devcontainers/containers) or simply ask `OpenHands` ;-)
+
+## Rebuild dev image
+
+You could optionally pass additional options to the build script.
+
+```bash
+make docker-dev OPTIONS="--build"
+# or
+./containers/dev/dev.sh --build
+```
+
+See [docker compose run](https://docs.docker.com/reference/cli/docker/compose/run/) for more options.

containers/dev/compose.yml

@@ -0,0 +1,38 @@
+#
+services:
+  dev:
+    privileged: true
+    build:
+      context: ${OPENHANDS_WORKSPACE:-../../}
+      dockerfile: ./containers/dev/Dockerfile
+    image: openhands:dev
+    container_name: openhands-dev
+    environment:
+      - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
+      - SANDBOX_API_HOSTNAME=host.docker.internal
+      #
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
+      - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
+    ports:
+      - "3000:3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${WORKSPACE_BASE:-$PWD/workspace}:/opt/workspace_base
+      # source code
+      - ${OPENHANDS_WORKSPACE:-../../}:/app
+      # host credentials
+      - $HOME/.git-credentials:/root/.git-credentials:ro
+      - $HOME/.gitconfig:/root/.gitconfig:ro
+      - $HOME/.npmrc:/root/.npmrc:ro
+      # cache
+      - cache-data:/root/.cache
+    pull_policy: never
+    stdin_open: true
+    tty: true
+
+##
+volumes:
+  cache-data:

containers/dev/dev.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -o pipefail
+
+function get_docker() {
+    echo "Docker is required to build and run OpenHands."
+    echo "https://docs.docker.com/get-started/get-docker/"
+    exit 1
+}
+
+function check_tools() {
+	command -v docker &>/dev/null || get_docker
+}
+
+function exit_if_indocker() {
+    if [ -f /.dockerenv ]; then
+        echo "Running inside a Docker container. Exiting..."
+        exit 1
+    fi
+}
+
+#
+exit_if_indocker
+
+check_tools
+
+##
+OPENHANDS_WORKSPACE=$(git rev-parse --show-toplevel)
+
+cd "$OPENHANDS_WORKSPACE/containers/dev/" || exit 1
+
+##
+export BACKEND_HOST="0.0.0.0"
+#
+export SANDBOX_USER_ID=$(id -u)
+export WORKSPACE_BASE=${WORKSPACE_BASE:-$OPENHANDS_WORKSPACE/workspace}
+
+docker compose run --rm --service-ports "$@" dev
+
+##

docs/docusaurus.config.ts

@@ -4,8 +4,8 @@ import { themes as prismThemes } from "prism-react-renderer";
 
 const config: Config = {
   title: "OpenHands",
-  tagline: "An Open Platform for AI Software Developers as Generalist Agents",
-  favicon: "img/logo.png",
+  tagline: "Code Less, Make More",
+  favicon: "img/logo-square.png",
 
   // Set the production url of your site here
   url: "https://docs.all-hands.dev",
@@ -73,23 +73,28 @@ const config: Config = {
           type: "docSidebar",
           sidebarId: "docsSidebar",
           position: "left",
-          label: "Docs",
+          label: "User Guides",
         },
         {
           type: "docSidebar",
           sidebarId: "apiSidebar",
           position: "left",
-          label: "Codebase",
+          label: "Python API",
+        },
+        {
+          type: 'localeDropdown',
+          position: 'left',
+        },
+        {
+          href: "https://all-hands.dev",
+          label: "Company",
+          position: "right",
         },
         {
           href: "https://github.com/All-Hands-AI/OpenHands",
           label: "GitHub",
           position: "right",
         },
-        {
-          type: 'localeDropdown',
-          position: 'left',
-        },
       ],
     },
     prism: {

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/custom_sandbox_guide.md

@@ -59,10 +59,6 @@ Félicitations !
 
 ## Explication technique
 
-Le code pertinent est défini dans [ssh_box.py](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/ssh_box.py) et [image_agnostic_util.py](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py).
-
-En particulier, ssh_box.py vérifie l'objet config pour ```config.sandbox.base_container_image``` et ensuite tente de récupérer l'image à l'aide de [get_openhands_sandbox_image](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py#L72), qui est défini dans image_agnostic_util.py.
-
 Lorsqu'une image personnalisée est utilisée pour la première fois, elle ne sera pas trouvée et donc elle sera construite (à l'exécution ultérieure, l'image construite sera trouvée et renvoyée).
 
 L'image personnalisée est construite avec [_build_sandbox_image()](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py#L29), qui crée un fichier docker en utilisant votre image personnalisée comme base et configure ensuite l'environnement pour OpenHands, comme ceci:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -41,4 +41,4 @@ ne peut être aussi puissant que les modèles qui le pilotent -- heureusement, l
 
 Certains LLM ont des limites de taux et peuvent nécessiter des réessais. OpenHands réessaiera automatiquement les demandes s'il reçoit une erreur 429 ou une erreur de connexion API.
 Vous pouvez définir les variables d'environnement `LLM_NUM_RETRIES`, `LLM_RETRY_MIN_WAIT`, `LLM_RETRY_MAX_WAIT` pour contrôler le nombre de réessais et le temps entre les réessais.
-Par défaut, `LLM_NUM_RETRIES` est 5 et `LLM_RETRY_MIN_WAIT`, `LLM_RETRY_MAX_WAIT` sont respectivement de 3 secondes et 60 secondes.
+Par défaut, `LLM_NUM_RETRIES` est 8 et `LLM_RETRY_MIN_WAIT`, `LLM_RETRY_MAX_WAIT` sont respectivement de 15 secondes et 120 secondes.

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/custom_sandbox_guide.md

@@ -2,13 +2,30 @@
 
 默认的 OpenHands 沙箱包含一个[最小化 ubuntu 配置](https://github.com/All-Hands-AI/OpenHands/blob/main/containers/sandbox/Dockerfile)。您的应用场景可能需要在默认状态下安装额外的软件。本指南将教您如何通过使用自定义 Docker 映像来实现这一目标。
 
+目前提供两种实现方案：
+1. 从 Docker Hub 拉取已有镜像。例如，如果您想安装 `nodejs` ，您可以通过使用 `node:20` 镜像来实现。
+2. 创建并使用您自定义 Docker 镜像。
+
+若选择第一种方案，您可以直接略过 `Create Your Docker Image` 部分。
+
+为了获得功能更丰富的环境，您可能想要考虑使用预构建的镜像，比如 [nikolaik/python-nodejs](https://hub.docker.com/r/nikolaik/python-nodejs)，这个镜像预装了 Python 和 Node.js，同时还包含了许多其他有用的工具和库，比如：
+
+- Node.js: 22.x
+- npm: 10.x
+- yarn: stable
+- Python: latest
+- pip: latest
+- pipenv: latest
+- poetry: latest
+- uv: latest
+
 ## 环境设置
 
 确保您能够首先通过 [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) 运行 OpenHands。
 
 ## 创建您的 Docker 映像
 
-接下来，您必须创建一个自定义的 Docker 映像，该映像是基于 Debian 或 Ubuntu 的。例如，如果我们希望 OpenHands 能够访问 "node" 可执行文件，我们可以使用以下 `Dockerfile`:
+接下来，您可以开始创建一个自定义的 Docker 映像，该映像必须是基于 Debian 或 Ubuntu 的。例如，如果我们希望 OpenHands 能够访问 `node` 可执行文件，我们可以使用以下 `Dockerfile`:
 
 ```bash
 # 从最新版 ubuntu 开始
@@ -21,7 +38,7 @@ RUN apt-get update && apt-get install
 RUN apt-get install -y nodejs
 ```
 
-然后构建您选择的映像，例如“custom_image”。为此可以在目录中创建文件夹并将 `Dockerfile` 放入其中，并在该目录内运行以下命令：
+然后命名并构建您选择的映像，例如“custom_image”。为此可以创建一个文件夹并将 `Dockerfile` 放入其中，并在该文件夹内运行以下命令：
 
 ```bash
 docker build -t custom_image .
@@ -31,7 +48,7 @@ docker build -t custom_image .
 
 > 注意：在本文档描述的配置中，OpenHands 将在沙箱内部以“openhands”用户身份运行。因此，通过 Dockerfile 安装的所有包应可供系统上的所有用户使用，而不仅仅是 root 用户。
 
-> 使用 `apt-get` 上面安装的 node 是为所有用户安装的。
+> `Dockerfile`中，使用 `apt-get` 安装的 node 是为所有用户安装的。
 
 ## 在 config.toml 文件中指定自定义映像
 
@@ -41,46 +58,26 @@ docker build -t custom_image .
 [core]
 workspace_base="./workspace"
 run_as_openhands=true
-base_container_image="custom_image"
+sandbox_base_container_image="custom_image"
 ```
 
-> 确保 `sandbox_base_container_image` 设置为您前一步中自定义映像的名称。
+对于 `sandbox_base_container_image` 的值, 您可以选择以下任意一项：
+1. 在上一步中您构建的自定义镜像的名称（例如，`“custom_image”`）
+2. 从 Docker Hub 拉取的镜像（例如，`“node:20”`，如果你需要一个预装 `Node.js` 的沙箱环境）
 
 ## 运行
 
-通过运行 ```make run``` 在顶层目录下运行 OpenHands。
+在顶层目录下通过执行 ```make run``` 运行 OpenHands。
 
 导航至 ```localhost:3001``` 并检查所需依赖是否可用。
 
-在上述示例的情况下，终端中运行 `node -v` 会输出 `v18.19.1`。
+在上述示例的情况下，终端中运行 `node -v` 会输出 `v20.15.0`。
 
 恭喜您！
 
 ## 技术解释
 
-相关代码定义在 [ssh_box.py](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/ssh_box.py) 和 [image_agnostic_util.py](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py) 中。
-
-特别是 ssh_box.py 检查配置对象中的 ```config.sandbox.base_container_image```，然后尝试使用 [get_openhands_sandbox_image](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py#L72)，在 image_agnostic_util.py 定义中进行检索。
-
-初次使用自定义映像时，该映像将不会被找到，因此将被构建（在后续运行中已构建的映像将被查找并返回）。
-
-自定义映像是通过 `_build_sandbox_image()` 构建的，在 [image_agnostic_util.py](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py#L29) 中，使用您的 custom_image 作为基础，并为 OpenHands 配置环境。例如：
-
-```python
-dockerfile_content = (
-        f'FROM {base_image}\n'
-        'RUN apt update && apt install -y openssh-server wget sudo\n'
-        'RUN mkdir -p -m0755 /var/run/sshd\n'
-        'RUN mkdir -p /openhands && mkdir -p /openhands/logs && chmod 777 /openhands/logs\n'
-        'RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"\n'
-        'RUN bash Miniforge3-$(uname)-$(uname -m).sh -b -p /openhands/miniforge3\n'
-        'RUN bash -c ". /openhands/miniforge3/etc/profile.d/conda.sh && conda config --set changeps1 False && conda config --append channels conda-forge"\n'
-        'RUN echo "export PATH=/openhands/miniforge3/bin:$PATH" >> ~/.bashrc\n'
-        'RUN echo "export PATH=/openhands/miniforge3/bin:$PATH" >> /openhands/bash.bashrc\n'
-    ).strip()
-```
-
-> 注意：映像名称通过 [_get_new_image_name()](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/docker/image_agnostic_util.py#L63) 修改，并且是后续运行中搜索的修改后的名称。
+请参考[运行时文档中自定义 Docker 镜像的章节](https://docs.all-hands.dev/modules/usage/architecture/runtime#advanced-how-openhands-builds-and-maintains-od-runtime-images)获取更详细的解释。
 
 ## 故障排除 / 错误
 
@@ -98,8 +95,8 @@ sandbox_user_id="1001"
 
 ### 端口使用错误
 
-如果您看到关于端口被占用或不可用的错误，请尝试删除所有正在运行的 Docker 容器（通过运行 `docker ps` 和 `docker rm` 相关容器），然后重新运行 ```make run```。
+如果您遇到端口被占用或不可用的错误提示，可以尝试先用`docker ps`命令列出所有运行中的 Docker 容器，然后使用`docker rm`命令删除相关容器，最后再重新执行```make run```命令。
 
 ## 讨论
 
-对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) 或 [Discord](https://discord.gg/ESHStjSjD4)，并提问！
+对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) 或 [Discord](https://discord.gg/ESHStjSjD4) 提问！

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/llms/llms.md

@@ -43,4 +43,4 @@ OpenHands 将向你配置的 LLM 发出许多提示。大多数这些 LLM 都是
 
 一些 LLM 有速率限制，可能需要重试操作。OpenHands 会在收到 429 错误或 API 连接错误时自动重试请求。
 你可以设置 `LLM_NUM_RETRIES`，`LLM_RETRY_MIN_WAIT`，`LLM_RETRY_MAX_WAIT` 环境变量来控制重试次数和重试之间的时间。
-默认情况下，`LLM_NUM_RETRIES` 为 5，`LLM_RETRY_MIN_WAIT` 和 `LLM_RETRY_MAX_WAIT` 分别为 3 秒和 60 秒。
+默认情况下，`LLM_NUM_RETRIES` 为 8，`LLM_RETRY_MIN_WAIT` 和 `LLM_RETRY_MAX_WAIT` 分别为 15 秒和 120 秒。

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/llms/localLLMs.md

@@ -74,7 +74,7 @@ WORKSPACE_DIR="$(pwd)/workspace"
 
 如有需要，可以替换您选择的 `LLM_MODEL`。
 
-完成！现在您可以通过 `make run` 启动 Devin 而无需 Docker。现在您应该可以连接到 `http://localhost:3000/`
+完成！现在您可以通过 `make run` 启动 OpenHands 而无需 Docker。现在您应该可以连接到 `http://localhost:3000/`
 
 ## 选择您的模型
 

docs/modules/usage/about.md

@@ -8,22 +8,22 @@ sidebar_position: 8
 
 Achieving full replication of production-grade applications with LLMs is a complex endeavor. Our strategy involves:
 
-1. **Core Technical Research:** Focusing on foundational research to understand and improve the technical aspects of code generation and handling.
-2. **Specialist Abilities:** Enhancing the effectiveness of core components through data curation, training methods, and more.
-3. **Task Planning:** Developing capabilities for bug detection, codebase management, and optimization.
-4. **Evaluation:** Establishing comprehensive evaluation metrics to better understand and improve our models.
+1. **Core Technical Research:** Focusing on foundational research to understand and improve the technical aspects of code generation and handling
+2. **Specialist Abilities:** Enhancing the effectiveness of core components through data curation, training methods, and more
+3. **Task Planning:** Developing capabilities for bug detection, codebase management, and optimization
+4. **Evaluation:** Establishing comprehensive evaluation metrics to better understand and improve our models
 
 ## 🚧 Default Agent
 
-- Our default Agent is currently the CodeActAgent, which is capable of generating code and handling files. We're working on other Agent implementations, including [SWE Agent](https://swe-agent.com/). You can [read about our current set of agents here](./agents).
+Our default Agent is currently the [CodeActAgent](agents), which is capable of generating code and handling files.
 
 ## 🤝 How to Contribute
 
 OpenHands is a community-driven project, and we welcome contributions from everyone. Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of software engineering with AI, there are many ways to get involved:
 
-- **Code Contributions:** Help us develop the core functionalities, frontend interface, or sandboxing solutions.
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements.
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+- **Code Contributions:** Help us develop the core functionalities, frontend interface, or sandboxing solutions
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability
 
 For details, please check [this document](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md).
 

docs/modules/usage/agents.md

@@ -2,13 +2,14 @@
 sidebar_position: 3
 ---
 
-# 🧠 Agents and Capabilities
+# 🧠 Main Agent and Capabilities
 
-## CodeAct Agent
+## CodeActAgent
 
 ### Description
 
-This agent implements the CodeAct idea ([paper](https://arxiv.org/abs/2402.01030), [tweet](https://twitter.com/xingyaow_/status/1754556835703751087)) that consolidates LLM agents’ **act**ions into a unified **code** action space for both _simplicity_ and _performance_ (see paper for more details).
+This agent implements the CodeAct idea ([paper](https://arxiv.org/abs/2402.01030), [tweet](https://twitter.com/xingyaow_/status/1754556835703751087)) that consolidates LLM agents’ **act**ions into a
+unified **code** action space for both _simplicity_ and _performance_.
 
 The conceptual idea is illustrated below. At each turn, the agent can:
 
@@ -20,74 +21,8 @@ The conceptual idea is illustrated below. At each turn, the agent can:
 
 ![image](https://github.com/All-Hands-AI/OpenHands/assets/38853559/92b622e3-72ad-4a61-8f41-8c040b6d5fb3)
 
-### Plugin System
-
-To make the CodeAct agent more powerful with only access to `bash` action space, CodeAct agent leverages OpenHands's plugin system:
-
-- [Jupyter plugin](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/jupyter): for IPython execution via bash command
-- [SWE-agent tool plugin](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/swe_agent_commands): Powerful bash command line tools for software development tasks introduced by [swe-agent](https://github.com/princeton-nlp/swe-agent).
-
 ### Demo
 
 https://github.com/All-Hands-AI/OpenHands/assets/38853559/f592a192-e86c-4f48-ad31-d69282d5f6ac
 
-_Example of CodeActAgent with `gpt-4-turbo-2024-04-09` performing a data science task (linear regression)_
-
-### Actions
-
-`Action`,
-`CmdRunAction`,
-`IPythonRunCellAction`,
-`AgentEchoAction`,
-`AgentFinishAction`,
-`AgentTalkAction`
-
-### Observations
-
-`CmdOutputObservation`,
-`IPythonRunCellObservation`,
-`AgentMessageObservation`,
-`UserMessageObservation`
-
-### Methods
-
-| Method          | Description                                                                                                                                     |
-| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
-| `__init__`      | Initializes an agent with `llm` and a list of messages `list[Mapping[str, str]]`                                                                |
-| `step`          | Performs one step using the CodeAct Agent. This includes gathering info on previous steps and prompting the model to make a command to execute. |
-
-## Planner Agent
-
-### Description
-
-The planner agent utilizes a special prompting strategy to create long term plans for solving problems.
-The agent is given its previous action-observation pairs, current task, and hint based on last action taken at every step.
-
-### Actions
-
-`NullAction`,
-`CmdRunAction`,
-`BrowseURLAction`,
-`GithubPushAction`,
-`FileReadAction`,
-`FileWriteAction`,
-`AgentThinkAction`,
-`AgentFinishAction`,
-`AgentSummarizeAction`,
-`AddTaskAction`,
-`ModifyTaskAction`,
-
-### Observations
-
-`Observation`,
-`NullObservation`,
-`CmdOutputObservation`,
-`FileReadObservation`,
-`BrowserOutputObservation`
-
-### Methods
-
-| Method          | Description                                                                                                                                                                               |
-| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `__init__`      | Initializes an agent with `llm`                                                                                                                                                           |
-| `step`          | Checks to see if current step is completed, returns `AgentFinishAction` if True. Otherwise, creates a plan prompt and sends to model for inference, adding the result as the next action. |
+_Example of CodeActAgent with `gpt-4-turbo-2024-04-09` performing a data science task (linear regression)_.

docs/modules/usage/architecture/runtime.md

@@ -3,29 +3,24 @@
 The OpenHands EventStream Runtime is the core component that enables secure and flexible execution of AI agent's action.
 It creates a sandboxed environment using Docker, where arbitrary code can be run safely without risking the host system.
 
-
 ## Why do we need a sandboxed runtime?
 
 OpenHands needs to execute arbitrary code in a secure, isolated environment for several reasons:
 
-1. Security: Executing untrusted code can pose significant risks to the host system. A sandboxed environment prevents malicious code from accessing or modifying the host system's resources.
+1. Security: Executing untrusted code can pose significant risks to the host system. A sandboxed environment prevents malicious code from accessing or modifying the host system's resources
+2. Consistency: A sandboxed environment ensures that code execution is consistent across different machines and setups, eliminating "it works on my machine" issues
+3. Resource Control: Sandboxing allows for better control over resource allocation and usage, preventing runaway processes from affecting the host system
+4. Isolation: Different projects or users can work in isolated environments without interfering with each other or the host system
+5. Reproducibility: Sandboxed environments make it easier to reproduce bugs and issues, as the execution environment is consistent and controllable
 
-2. Consistency: A sandboxed environment ensures that code execution is consistent across different machines and setups, eliminating "it works on my machine" issues.
-
-3. Resource Control: Sandboxing allows for better control over resource allocation and usage, preventing runaway processes from affecting the host system.
-
-4. Isolation: Different projects or users can work in isolated environments without interfering with each other or the host system.
-
-5. Reproducibility: Sandboxed environments make it easier to reproduce bugs and issues, as the execution environment is consistent and controllable.
-
-## How does our Runtime work?
+## How does the Runtime work?
 
 The OpenHands Runtime system uses a client-server architecture implemented with Docker containers. Here's an overview of how it works:
 
 ```mermaid
 graph TD
     A[User-provided Custom Docker Image] --> B[OpenHands Backend]
-    B -->|Builds| C[OD Runtime Image]
+    B -->|Builds| C[OH Runtime Image]
     C -->|Launches| D[Runtime Client]
     D -->|Initializes| E[Browser]
     D -->|Initializes| F[Bash Shell]
@@ -51,94 +46,85 @@ graph TD
     end
 ```
 
-1. User Input: The user provides a custom base Docker image.
-
-2. Image Building: OpenHands builds a new Docker image (the "OD runtime image") based on the user-provided image. This new image includes OpenHands-specific code, primarily the "runtime client."
-
-3. Container Launch: When OpenHands starts, it launches a Docker container using the OD runtime image.
-
-4. Client Initialization: The runtime client initializes inside the container, setting up necessary components like a bash shell and loading any specified plugins.
-
-5. Communication: The OpenHands backend (`runtime.py`) communicates with the runtime client over RESTful API, sending actions and receiving observations.
-
-6. Action Execution: The runtime client receives actions from the backend, executes them in the sandboxed environment, and sends back observations.
-
-7. Observation Return: The client sends execution results back to the OpenHands backend as observations.
+1. User Input: The user provides a custom base Docker image
+2. Image Building: OpenHands builds a new Docker image (the "OH runtime image") based on the user-provided image. This new image includes OpenHands-specific code, primarily the "runtime client"
+3. Container Launch: When OpenHands starts, it launches a Docker container using the OH runtime image
+4. Client Initialization: The runtime client initializes inside the container, setting up necessary components like a bash shell and loading any specified plugins
+5. Communication: The OpenHands backend (`runtime.py`) communicates with the runtime client over RESTful API, sending actions and receiving observations
+6. Action Execution: The runtime client receives actions from the backend, executes them in the sandboxed environment, and sends back observations
+7. Observation Return: The client sends execution results back to the OpenHands backend as observations
 
 
-The role of the client is crucial:
-- It acts as an intermediary between the OpenHands backend and the sandboxed environment.
-- It executes various types of actions (shell commands, file operations, Python code, etc.) safely within the container.
-- It manages the state of the sandboxed environment, including the current working directory and loaded plugins.
-- It formats and returns observations to the backend, ensuring a consistent interface for processing results.
+The role of the client:
+- It acts as an intermediary between the OpenHands backend and the sandboxed environment
+- It executes various types of actions (shell commands, file operations, Python code, etc.) safely within the container
+- It manages the state of the sandboxed environment, including the current working directory and loaded plugins
+- It formats and returns observations to the backend, ensuring a consistent interface for processing results
 
 
-## Advanced: How OpenHands builds and maintains OD Runtime images
+## How OpenHands builds and maintains OH Runtime images
 
-OpenHands uses a sophisticated approach to build and manage runtime images. This process ensures efficiency, consistency, and flexibility in creating and maintaining Docker images for both production and development environments.
+OpenHands' approach to building and managing runtime images ensures efficiency, consistency, and flexibility in creating and maintaining Docker images for both production and development environments.
 
-Check out [relavant code](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/utils/runtime_build.py) if you are interested in more details.
+Check out the [relevant code](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/utils/runtime_build.py) if you are interested in more details.
 
 ### Image Tagging System
 
 OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility:
 
-1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
+1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`.
    Example: `runtime:abc123def456`
 
-   - This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile.
-   - Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile.
-   - This ensures reproducibility: the same hash always means the same image contents.
+   - This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile
+   - Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile
+   - This ensures reproducibility; the same hash always means the same image contents
 
-2. Generic tag: `{target_image_repo}:{target_image_tag}`
-   Example: `runtime:openhands_v0.8.3_ubuntu_tag_22.04`
+2. Generic tag: `{target_image_repo}:{target_image_tag}`.
+   Example: `runtime:oh_v0.9.3_ubuntu_tag_22.04`
 
-   - This tag follows the format: `runtime:openhands_v{OPENHANDS_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
-   - It represents the latest build for a particular base image and OpenHands version combination.
-   - This tag is updated whenever a new image is built from the same base image, even if the source code changes.
+   - This tag follows the format: `runtime:oh_v{OH_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
+   - It represents the latest build for a particular base image and OpenHands version combination
+   - This tag is updated whenever a new image is built from the same base image, even if the source code changes
 
-The hash-based tag ensures exact reproducibility, while the generic tag provides a stable reference to the latest version of a particular configuration. This dual-tagging approach allows OpenHands to efficiently manage both development and production environments.
+The hash-based tag ensures reproducibility, while the generic tag provides a stable reference to the latest version of a particular configuration. This dual-tagging approach allows OpenHands to efficiently manage both development and production environments.
 
 ### Build Process
 
 1. Image Naming Convention:
-   - Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
+   - Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`.
      Example: `runtime:abc123def456`
-   - Generic tag: `{target_image_repo}:{target_image_tag}`
-     Example: `runtime:openhands_v0.8.3_ubuntu_tag_22.04`
+   - Generic tag: `{target_image_repo}:{target_image_tag}`.
+     Example: `runtime:oh_v0.9.3_ubuntu_tag_22.04`
 
 2. Build Process:
-   - a. Convert the base image name to an OD runtime image name.
-      Example: `ubuntu:22.04` -> `runtime:openhands_v0.8.3_ubuntu_tag_22.04`
-   - b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash.
-   - c. Check for an existing image with the calculated hash.
-   - d. If not found, check for a recent compatible image to use as a base.
-   - e. If no compatible image exists, build from scratch using the original base image.
-   - f. Tag the new image with both hash-based and generic tags.
+   - a. Convert the base image name to an OH runtime image name
+      Example: `ubuntu:22.04` -> `runtime:oh_v0.9.3_ubuntu_tag_22.04`
+   - b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash
+   - c. Check for an existing image with the calculated hash
+   - d. If not found, check for a recent compatible image to use as a base
+   - e. If no compatible image exists, build from scratch using the original base image
+   - f. Tag the new image with both hash-based and generic tags
 
 3. Image Reuse and Rebuilding Logic:
    The system follows these steps to determine whether to build a new image or use an existing one from a user-provided (base) image (e.g., `ubuntu:22.04`):
-
-   a. If an image exists with the same hash (e.g., `runtime:abc123def456`), it will be reused as is.
-
-   b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `runtime:openhands_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
-
-   c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch.
+   - a. If an image exists with the same hash (e.g., `runtime:abc123def456`), it will be reused as is
+   - b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `runtime:oh_v0.9.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies
+   - c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch
 
 4. Caching and Efficiency:
-   - The system attempts to reuse existing images when possible to save build time.
-   - If an exact match (by hash) is found, it's used without rebuilding.
-   - If a compatible image is found, it's used as a base for rebuilding, saving time on dependency installation.
+   - The system attempts to reuse existing images when possible to save build time
+   - If an exact match (by hash) is found, it's used without rebuilding
+   - If a compatible image is found, it's used as a base for rebuilding, saving time on dependency installation
 
 Here's a flowchart illustrating the build process:
 
 ```mermaid
 flowchart TD
     A[Start] --> B{Convert base image name}
-    B --> |ubuntu:22.04 -> runtime:openhands_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
+    B --> |ubuntu:22.04 -> runtime:oh_v0.9.3_ubuntu_tag_22.04| C[Generate build context and hash]
     C --> D{Check for existing image with hash}
     D -->|Found runtime:abc123def456| E[Use existing image]
-    D -->|Not found| F{Check for runtime:openhands_v0.8.3_ubuntu_tag_22.04}
+    D -->|Not found| F{Check for runtime:oh_v0.9.3_ubuntu_tag_22.04}
     F -->|Found| G[Rebuild based on recent image]
     F -->|Not found| H[Build from scratch]
     G --> I[Tag with hash and generic tags]
@@ -149,29 +135,22 @@ flowchart TD
 
 This approach ensures that:
 
-1. Identical source code and Dockerfile always produce the same image (via hash-based tags).
-2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images).
-3. The generic tag (e.g., `runtime:openhands_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
+1. Identical source code and Dockerfile always produce the same image (via hash-based tags)
+2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images)
+3. The generic tag (e.g., `runtime:oh_v0.9.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination
 
-By using this method, OpenHands maintains an efficient and flexible system for building and managing runtime images, adapting to both development needs and production requirements.
-
-
-## Advanced: Runtime Plugin System
+## Runtime Plugin System
 
 The OpenHands Runtime supports a plugin system that allows for extending functionality and customizing the runtime environment. Plugins are initialized when the runtime client starts up.
 
-Check [an example of Jupyter plugin here](https://github.com/All-Hands-AI/OpenHands/blob/9c44d94cef32e6426ebd8deeeb52963153b2348a/openhands/runtime/plugins/jupyter/__init__.py#L30-L63) if you want to implement your own plugin.
+Check [an example of Jupyter plugin here](https://github.com/All-Hands-AI/OpenHands/blob/ecf4aed28b0cf7c18d4d8ff554883ba182fc6bdd/openhands/runtime/plugins/jupyter/__init__.py#L21-L55) if you want to implement your own plugin.
 
 *More details about the Plugin system are still under construction - contributions are welcomed!*
 
 Key aspects of the plugin system:
 
-1. Plugin Definition: Plugins are defined as Python classes that inherit from a base `Plugin` class.
-
-2. Plugin Registration: Available plugins are registered in an `ALL_PLUGINS` dictionary.
-
-3. Plugin Specification: Plugins are associate with `Agent.sandbox_plugins: list[PluginRequirement]`. Users can specify which plugins to load when initializing the runtime.
-
-4. Initialization: Plugins are initialized asynchronously when the runtime client starts.
-
-5. Usage: The runtime client can use initialized plugins to extend its capabilities (e.g., the JupyterPlugin for running IPython cells).
+1. Plugin Definition: Plugins are defined as Python classes that inherit from a base `Plugin` class
+2. Plugin Registration: Available plugins are registered in an `ALL_PLUGINS` dictionary
+3. Plugin Specification: Plugins are associated with `Agent.sandbox_plugins: list[PluginRequirement]`. Users can specify which plugins to load when initializing the runtime
+4. Initialization: Plugins are initialized asynchronously when the runtime client starts
+5. Usage: The runtime client can use initialized plugins to extend its capabilities (e.g., the JupyterPlugin for running IPython cells)

docs/modules/usage/feedback.md

@@ -4,11 +4,11 @@ sidebar_position: 5
 
 # ✅ Providing Feedback
 
-When using OpenHands, you will undoubtably encounter cases where things work well, and others where they don't. We encourage you to provide feedback when you use OpenHands to help give feedback to the development team, and perhaps more importantly, create an open corpus of coding agent training examples -- Share-OpenHands!
+When using OpenHands, you will encounter cases where things work well, and others where they don't. We encourage you to provide feedback when you use OpenHands to help give feedback to the development team, and perhaps more importantly, create an open corpus of coding agent training examples -- Share-OpenHands!
 
 ## 📝 How to Provide Feedback
 
-Providing feedback is easy! When you are using OpenHands, you can press the thumbs-up or thumbs-down button at any point during your interaction with. You will be prompted to provide your email address (e.g. so we can contact you if we want to ask any follow-up questions), and you can choose whether you want to provide feedback publicly or privately.
+Providing feedback is easy! When you are using OpenHands, you can press the thumbs-up or thumbs-down button at any point during your interaction. You will be prompted to provide your email address (e.g. so we can contact you if we want to ask any follow-up questions), and you can choose whether you want to provide feedback publicly or privately.
 
 <iframe width="560" height="315" src="https://www.youtube.com/embed/5rFx-StMVV0?si=svo7xzp6LhGK_GXr" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
 
@@ -31,7 +31,7 @@ The public data will be released when we hit fixed milestones, such as 1,000 pub
 At this time, we will follow the following release process:
 
 1. All people who contributed public feedback will receive an email describing the data release and being given an opportunity to opt out.
-2. The person or people in charge of the data release will perform quality control of the data, removing low-quality feedback, removing email submitter email addresses, and attempting to remove any sensitive information such as API keys.
+2. The person or people in charge of the data release will perform quality control of the data, removing low-quality feedback, removing email submitter email addresses, and attempting to remove any sensitive information.
 3. The data will be released publicly under the MIT license through commonly used sites such as github or Hugging Face.
 
 ### What if I want my data deleted?

docs/modules/usage/getting-started.mdx

@@ -0,0 +1,66 @@
+---
+sidebar_position: 2
+---
+
+# Getting Started
+
+## System Requirements
+
+* Docker version 26.0.0+ or Docker Desktop 4.31.0+
+* You must be using Linux or Mac OS
+  * If you are on Windows, you must use [WSL](https://learn.microsoft.com/en-us/windows/wsl/install)
+
+## Installation
+
+The easiest way to run OpenHands is in Docker. You can change `WORKSPACE_BASE` below to point OpenHands to
+existing code that you'd like to modify.
+
+```bash
+export WORKSPACE_BASE=$(pwd)/workspace
+
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.9-nikolaik \
+    -e SANDBOX_USER_ID=$(id -u) \
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -p 3000:3000 \
+    --add-host host.docker.internal:host-gateway \
+    --name openhands-app-$(date +%Y%m%d%H%M%S) \
+    ghcr.io/all-hands-ai/openhands:0.9
+```
+
+You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
+or as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode).
+
+## Setup
+
+After running the command above, you'll find OpenHands running at [http://localhost:3000](http://localhost:3000).
+
+The agent will have access to the `./workspace` folder to do its work. You can copy existing code here, or change `WORKSPACE_BASE` in the
+command to point to an existing folder.
+
+Upon launching OpenHands, you'll see a settings modal. You **must** select an `LLM Provider` and `LLM Model` and enter a corresponding `API Key`.
+These can be changed at any time by selecting the `Settings` button (gear icon) in the UI.
+
+If the required `LLM Model` does not exist in the list, you can toggle `Advanced Options` and manually enter it with the correct prefix
+in the `Custom Model` text box.
+The `Advanced Options` also allow you to specify a `Base URL` if required.
+
+<div style={{ display: 'flex', justifyContent: 'center', gap: '20px' }}>
+  <img src="/img/settings-screenshot.png" alt="settings-modal" width="340" />
+  <img src="/img/settings-advanced.png" alt="settings-modal" width="335" />
+</div>
+
+## Versions
+
+The command above pulls the `0.9` tag, which represents the most recent stable release of OpenHands. You have other options as well:
+- For a specific release, use `ghcr.io/all-hands-ai/openhands:$VERSION`, replacing $VERSION with the version number.
+- We use semver, and release major, minor, and patch tags. So `0.9` will automatically point to the latest `0.9.x` release, and `0` will point to the latest `0.x.x` release.
+- For the most up-to-date development version, you can use `ghcr.io/all-hands-ai/openhands:main`. This version is unstable and is recommended for testing or development purposes only.
+
+You can choose the tag that best suits your needs based on stability requirements and desired features.
+
+For the development workflow, see [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md).
+
+Are you having trouble? Check out our [Troubleshooting Guide](https://docs.all-hands.dev/modules/usage/troubleshooting).

docs/modules/usage/how-to/cli-mode.md

@@ -0,0 +1,108 @@
+# CLI Mode
+
+OpenHands can be run in an interactive CLI mode, which allows users to start an interactive session via the command line.
+
+This mode is different from the [headless mode](headless-mode), which is non-interactive and better for scripting.
+
+## With Python
+
+To start an interactive OpenHands session via the command line, follow these steps:
+
+1. Ensure you have followed the [Development setup instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md)
+
+2. Run the following command:
+
+```bash
+poetry run python -m openhands.core.cli
+```
+
+This command will start an interactive session where you can input tasks and receive responses from OpenHands.
+
+You'll need to be sure to set your model, API key, and other settings via environment variables
+[or the `config.toml` file](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml).
+
+
+## With Docker
+
+To run OpenHands in CLI mode with Docker, follow these steps:
+
+1. Set `WORKSPACE_BASE` to the directory you want OpenHands to edit:
+
+```bash
+WORKSPACE_BASE=$(pwd)/workspace
+```
+
+2. Set `LLM_MODEL` to the model you want to use:
+
+```bash
+LLM_MODEL="anthropic/claude-3-5-sonnet-20240620"
+```
+
+3. Set `LLM_API_KEY` to your API key:
+
+```bash
+LLM_API_KEY="sk_test_12345"
+```
+
+4. Run the following Docker command:
+
+```bash
+docker run -it \
+    --pull=always \
+    -e SANDBOX_USER_ID=$(id -u) \
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
+    -e LLM_API_KEY=$LLM_API_KEY \
+    -e LLM_MODEL=$LLM_MODEL \
+    -v $WORKSPACE_BASE:/opt/workspace_base \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    --add-host host.docker.internal:host-gateway \
+    --name openhands-app-$(date +%Y%m%d%H%M%S) \
+    ghcr.io/all-hands-ai/openhands:0.9 \
+    python -m openhands.core.cli
+```
+
+This command will start an interactive session in Docker where you can input tasks and receive responses from OpenHands.
+
+## Examples of CLI Commands and Expected Outputs
+
+Here are some examples of CLI commands and their expected outputs:
+
+### Example 1: Simple Task
+
+```bash
+How can I help? >> Write a Python script that prints "Hello, World!"
+```
+
+Expected Output:
+
+```bash
+🤖 Sure! Here is a Python script that prints "Hello, World!":
+
+❯ print("Hello, World!")
+```
+
+### Example 2: Bash Command
+
+```bash
+How can I help? >> Create a directory named "test_dir"
+```
+
+Expected Output:
+
+```bash
+🤖 Creating a directory named "test_dir":
+
+❯ mkdir test_dir
+```
+
+### Example 3: Error Handling
+
+```bash
+How can I help? >> Delete a non-existent file
+```
+
+Expected Output:
+
+```bash
+🤖 An error occurred. Please try again.
+```

docs/modules/usage/how-to/custom-sandbox-guide.md

@@ -1,27 +1,19 @@
-# Create and Use a Custom Docker Sandbox
+# Custom Sandbox
 
-The default OpenHands sandbox comes with a [minimal ubuntu configuration](https://github.com/All-Hands-AI/OpenHands/blob/main/containers/sandbox/Dockerfile).
+The sandbox is where the agent does its work. Instead of running commands directly on your computer
+(which could be dangerous), the agent runs them inside of a Docker container.
 
-Your use case may need additional software installed by default.
+The default OpenHands sandbox (`python-nodejs:python3.11-nodejs22`
+from [nikolaik/python-nodejs](https://hub.docker.com/r/nikolaik/python-nodejs)) comes with some packages installed such
+as python and Node.js but your use case may need additional software installed by default.
 
 There are two ways you can do so:
 
-1. Use an existing image from docker hub. For instance, if you want to have `nodejs` installed, you can do so by using the `node:20` image
+1. Use an existing image from docker hub
 2. Creating your own custom docker image and using it
 
 If you want to take the first approach, you can skip the `Create Your Docker Image` section.
 
-For a more feature-rich environment, you might consider using pre-built images like **[nikolaik/python-nodejs](https://hub.docker.com/r/nikolaik/python-nodejs)**, which comes with both Python and Node.js pre-installed, along with many other useful tools and libraries, like:
-
-- Node.js: 22.x
-- npm: 10.x
-- yarn: stable
-- Python: latest
-- pip: latest
-- pipenv: latest
-- poetry: latest
-- uv: latest
-
 ## Setup
 
 Make sure you are able to run OpenHands using the [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) first.
@@ -80,7 +72,7 @@ Run OpenHands by running ```make run``` in the top level directory.
 
 Navigate to ```localhost:3001``` and check if your desired dependencies are available.
 
-In the case of the example above, running ```node -v``` in the terminal produces ```v20.15.0```
+In the case of the example above, running ```node -v``` in the terminal produces ```v20.15.0```.
 
 Congratulations!
 

docs/modules/usage/how-to/evaluation-harness.md

@@ -1,11 +1,10 @@
-# Contribute to OpenHands Evaluation Harness
+# Evaluation
 
 This guide provides an overview of how to integrate your own evaluation benchmark into the OpenHands framework.
 
-## Before everything begins: Setup Environment and LLM Configuration
-
-Please follow instruction [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
+## Setup Environment and LLM Configuration
 
+Please follow instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment.
 OpenHands in development mode uses `config.toml` to keep track of most configurations.
 
 Here's an example configuration file you can use to define and use multiple LLMs:
@@ -13,7 +12,7 @@ Here's an example configuration file you can use to define and use multiple LLMs
 ```toml
 [llm]
 # IMPORTANT: add your API key here, and set the model to the one you want to evaluate
-model = "gpt-4o-2024-05-13"
+model = "claude-3-5-sonnet-20240620"
 api_key = "sk-XXX"
 
 [llm.eval_gpt4_1106_preview_llm]
@@ -61,9 +60,9 @@ This command runs OpenHands with:
 
 The main entry point for OpenHands is in `openhands/core/main.py`. Here's a simplified flow of how it works:
 
-1. Parse command-line arguments and load the configuration.
-2. Create a runtime environment using `create_runtime()`.
-3. Initialize the specified agent.
+1. Parse command-line arguments and load the configuration
+2. Create a runtime environment using `create_runtime()`
+3. Initialize the specified agent
 4. Run the controller using `run_controller()`, which:
    - Attaches the runtime to the agent
    - Executes the agent's task
@@ -127,7 +126,7 @@ To create an evaluation workflow for your benchmark, follow these steps:
 
 3. Initialize the runtime and set up the evaluation environment:
    ```python
-   async def initialize_runtime(runtime: Runtime, instance: pd.Series):
+   def initialize_runtime(runtime: Runtime, instance: pd.Series):
        # Set up your evaluation environment here
        # For example, setting environment variables, preparing files, etc.
        pass
@@ -135,14 +134,14 @@ To create an evaluation workflow for your benchmark, follow these steps:
 
 4. Create a function to process each instance:
    ```python
-   async def process_instance(instance: pd.Series, metadata: EvalMetadata) -> EvalOutput:
+   def process_instance(instance: pd.Series, metadata: EvalMetadata) -> EvalOutput:
        config = get_config(instance, metadata)
-       runtime = await create_runtime(config, sid=instance.instance_id)
-       await initialize_runtime(runtime, instance)
+       runtime = create_runtime(config, sid=instance.instance_id)
+       initialize_runtime(runtime, instance)
 
        instruction = get_instruction(instance, metadata)
 
-       state = await run_controller(
+       state = run_controller(
            config=config,
            task_str=instruction,
            runtime=runtime,
@@ -234,9 +233,9 @@ Here's a more accurate visual representation:
 
 In this workflow:
 
-- Executable actions (like running commands or executing code) are handled directly by the Runtime.
-- Non-executable actions (typically when the agent wants to communicate or ask for clarification) are handled by the `user_response_fn`.
-- The agent then processes the feedback, whether it's an Observation from the Runtime or a simulated response from the `user_response_fn`.
+- Executable actions (like running commands or executing code) are handled directly by the Runtime
+- Non-executable actions (typically when the agent wants to communicate or ask for clarification) are handled by the `user_response_fn`
+- The agent then processes the feedback, whether it's an Observation from the Runtime or a simulated response from the `user_response_fn`
 
 This approach allows for automated handling of both concrete actions and simulated user interactions, making it suitable for evaluation scenarios where you want to test the agent's ability to complete tasks with minimal human intervention.
 
@@ -270,8 +269,8 @@ def codeact_user_response(state: State | None) -> str:
 
 This function does the following:
 
-1. Provides a standard message encouraging the agent to continue working.
-2. Checks how many times the agent has attempted to communicate with the user.
-3. If the agent has made multiple attempts, it provides an option to give up.
+1. Provides a standard message encouraging the agent to continue working
+2. Checks how many times the agent has attempted to communicate with the user
+3. If the agent has made multiple attempts, it provides an option to give up
 
 By using this function, you can ensure consistent behavior across multiple evaluation runs and prevent the agent from getting stuck waiting for human input.

docs/modules/usage/how-to/headless-mode.md

@@ -1,9 +1,12 @@
-# Running in Headless Mode
+# Headless Mode
 
-You can run OpenHands via a CLI, without starting the web application. This makes it easy
-to automate tasks with OpenHands.
+You can run OpenHands with a single command, without starting the web application.
+This makes it easy to write scripts and automate tasks with OpenHands.
+
+This is different from [CLI Mode](cli-mode), which is interactive, and better for active development.
 
 ## With Python
+
 To run OpenHands in headless mode with Python,
 [follow the Development setup instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
 and then run:
@@ -12,19 +15,32 @@ and then run:
 poetry run python -m openhands.core.main -t "write a bash script that prints hi"
 ```
 
+You'll need to be sure to set your model, API key, and other settings via environment variables
+[or the `config.toml` file](https://github.com/All-Hands-AI/OpenHands/blob/main/config.template.toml).
+
 ## With Docker
-To run OpenHands in headless mode with Docker, run:
+
+1. Set `WORKSPACE_BASE` to the directory you want OpenHands to edit:
 
 ```bash
-# Set WORKSPACE_BASE to the directory you want OpenHands to edit
 WORKSPACE_BASE=$(pwd)/workspace
+```
 
-# Set LLM_API_KEY to an API key, e.g. for OpenAI or Anthropic
-LLM_API_KEY="abcde"
+2. Set `LLM_MODEL` to the model you want to use:
 
-# Set LLM_MODEL to the model you want to use
-LLM_MODEL="gpt-4o"
+```bash
+LLM_MODEL="anthropic/claude-3-5-sonnet-20240620"
+```
 
+3. Set `LLM_API_KEY` to your API key:
+
+```bash
+LLM_API_KEY="sk_test_12345"
+```
+
+4. Run the following Docker command:
+
+```bash
 docker run -it \
     --pull=always \
     -e SANDBOX_USER_ID=$(id -u) \
@@ -35,7 +51,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/all-hands-ai/openhands:main \ # TODO: pin a version here
-    python -m openhands.core.main \
-    -t "Write a bash script that prints Hello World"
+    ghcr.io/all-hands-ai/openhands:0.9 \
+    python -m openhands.core.main -t "write a bash script that prints hi"
 ```

docs/modules/usage/how-to/how-to.md

@@ -1,5 +0,0 @@
----
-sidebar_position: 6
----
-
-# 🔎 How To Section

docs/modules/usage/how-to/openshift-example.md

@@ -1,11 +1,11 @@
-# Use OpenHands in OpenShift/K8S
+# Kubernetes
 
-There are different ways and scenarios that you can do, we're just mentioning one example here:
-1. Create a PV "as a cluster admin" to map workspace_base data and docker directory to the pod through the worker node.
-2. Create a PVC to be able to mount those PVs to the POD
-3. Create a POD which contains two containers; the OpenHands and Sandbox containers.
+There are different ways you might run OpenHands on Kubernetes or OpenShift. This guide goes through one possible way:
+1. Create a PV "as a cluster admin" to map workspace_base data and docker directory to the pod through the worker node
+2. Create a PVC to be able to mount those PVs to the pod
+3. Create a pod which contains two containers; the OpenHands and Sandbox containers
 
-## Steps to follow the above example.
+## Detailed Steps for the Example Above
 
 > Note: Make sure you are logged in to the cluster first with the proper account for each step. PV creation requires cluster administrator!
 
@@ -135,8 +135,8 @@ LAST SEEN   TYPE     REASON                 OBJECT
 10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
 ```
 
-3. Create the POD yaml file:
-Sample POD yaml file below:
+3. Create the pod yaml file:
+Sample pod yaml file below:
 
 - pod.yaml
 
@@ -262,35 +262,3 @@ Events:                   <none>
 6. Connect to OpenHands UI, configure the Agent, then test:
 
 ![image](https://github.com/user-attachments/assets/12f94804-a0c7-4744-b873-e003c9caf40e)
-
-
-## Challenges
-Some of the challenages that would be needed to improve:
-
-1. Install GIT into the container:
-   This can be resolved by building a custom image which includes GIT software and use that image during pod deplyment.
-
-Example below: "to be tested!"
-
-```dockerfile
-FROM ghcr.io/all-hands-ai/openhands:main
-
-# Install Git
-RUN apt-get update && apt-get install -y git
-
-# Ensure /opt/workspace_base is writable
-RUN mkdir -p /opt/workspace_base && chown -R 1000:1000 /opt/workspace_base
-
-# Verify Git installation
-RUN git --version
-```
-
-2. Mount a shared development directory "i.e. one hosted in EC2 instance" to the POD:
-   This can be also done by sharing the developement directory to the worker node through a sharing software (NFS), then creating a pv and pvc as described above to access that directory.
-
-3. Not all Agents working! Just tested CoderAgent with an openai API key and produced results.
-
-
-## Discuss
-
-For other issues or questions join the [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) or [Discord](https://discord.gg/ESHStjSjD4) and ask!

docs/modules/usage/intro.mdx

@@ -1,71 +0,0 @@
----
-sidebar_position: 1
----
-
-# 💻 OpenHands
-
-OpenHands is an **autonomous AI software engineer** capable of executing complex engineering tasks and collaborating actively with users on software development projects.
-This project is fully open-source, so you can use and modify it however you like.
-
-:::tip
-Explore the codebase of OpenHands on [GitHub](https://github.com/All-Hands-AI/OpenHands) or join one of our communities!
-
-<a href="https://github.com/All-Hands-AI/OpenHands/graphs/contributors">
-  <img
-    src="https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge"
-    alt="Contributors"
-  />
-</a>
-<a href="https://github.com/All-Hands-AI/OpenHands/network/members">
-  <img
-    src="https://img.shields.io/github/forks/All-Hands-AI/OpenHands?style=for-the-badge"
-    alt="Forks"
-  />
-</a>
-<a href="https://github.com/All-Hands-AI/OpenHands/stargazers">
-  <img
-    src="https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge"
-    alt="Stargazers"
-  />
-</a>
-<a href="https://github.com/All-Hands-AI/OpenHands/issues">
-  <img
-    src="https://img.shields.io/github/issues/All-Hands-AI/OpenHands?style=for-the-badge"
-    alt="Issues"
-  />
-</a>
-<br></br>
-<a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE">
-  <img
-    src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge"
-    alt="MIT License"
-  />
-</a>
-<br></br>
-<a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA">
-  <img
-    src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge"
-    alt="Join our Slack community"
-  />
-</a>
-<a href="https://discord.gg/ESHStjSjD4">
-  <img
-    src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge"
-    alt="Join our Discord community"
-  />
-</a>
-:::
-
-## 🛠️ Getting Started
-[Check out the getting started guide on Github](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-getting-started)
-
-[contributors-shield]: https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge
-[contributors-url]: https://github.com/All-Hands-AI/OpenHands/graphs/contributors
-[forks-shield]: https://img.shields.io/github/forks/All-Hands-AI/OpenHands?style=for-the-badge
-[forks-url]: https://github.com/All-Hands-AI/OpenHands/network/members
-[stars-shield]: https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge
-[stars-url]: https://github.com/All-Hands-AI/OpenHands/stargazers
-[issues-shield]: https://img.shields.io/github/issues/All-Hands-AI/OpenHands?style=for-the-badge
-[issues-url]: https://github.com/All-Hands-AI/OpenHands/issues
-[license-shield]: https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge
-[license-url]: https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE

docs/modules/usage/llms/azure-llms.md

@@ -1,55 +1,46 @@
-# Azure OpenAI LLM
+# Azure
 
-## Completion
+OpenHands uses LiteLLM for completion calls. You can find their documentation on Azure [here](https://docs.litellm.ai/docs/providers/azure).
 
-OpenHands uses LiteLLM for completion calls. You can find their documentation on Azure [here](https://docs.litellm.ai/docs/providers/azure)
+## Azure OpenAI Configuration
 
-### Azure openai configs
-
-When running the OpenHands Docker image, you'll need to set the following environment variables using `-e`:
+When running OpenHands, you'll need to set the following environment variable using `-e` in the
+[docker run command](/modules/usage/getting-started#installation):
 
 ```
-LLM_BASE_URL="<azure-api-base-url>"          # e.g. "https://openai-gpt-4-test-v-1.openai.azure.com/"
-LLM_API_KEY="<azure-api-key>"
-LLM_MODEL="azure/<your-gpt-deployment-name>"
-LLM_API_VERSION="<api-version>"          # e.g. "2024-02-15-preview"
+LLM_API_VERSION="<api-version>"              # e.g. "2023-05-15"
 ```
 
 Example:
 ```bash
-docker run -it \
---pull=always \
--e SANDBOX_USER_ID=$(id -u) \
--e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
--e LLM_BASE_URL="x.openai.azure.com" \
--e LLM_API_VERSION="2024-02-15-preview" \
--v $WORKSPACE_BASE:/opt/workspace_base \
--v /var/run/docker.sock:/var/run/docker.sock \
--p 3000:3000 \
---add-host host.docker.internal:host-gateway \
---name openhands-app-$(date +%Y%m%d%H%M%S) \
-ghcr.io/all-hands-ai/openhands:main
+docker run -it --pull=always \
+    -e LLM_API_VERSION="2023-05-15"
+    ...
 ```
 
-You can set the LLM_MODEL and LLM_API_KEY in the OpenHands UI itself.
+Then set the following in the OpenHands UI through the Settings:
 
 :::note
-You can find your ChatGPT deployment name on the deployments page in Azure. It could be the same with the chat model name (e.g. 'GPT4-1106-preview'), by default or initially set, but it doesn't have to be the same. Run openhands, and when you load it in the browser, go to Settings and set model as above: "azure/&lt;your-actual-gpt-deployment-name&gt;". If it's not in the list, enter your own text and save it.
+You will need your ChatGPT deployment name which can be found on the deployments page in Azure. This is referenced as
+&lt;deployment-name&gt; below.
 :::
 
+* Enable `Advanced Options`
+* `Custom Model` to azure/&lt;deployment-name&gt;
+* `Base URL` to your Azure API Base URL (Example: `https://example-endpoint.openai.azure.com`)
+* `API Key` to your Azure API key
+
 ## Embeddings
 
-OpenHands uses llama-index for embeddings. You can find their documentation on Azure [here](https://docs.llamaindex.ai/en/stable/api_reference/embeddings/azure_openai/)
+OpenHands uses llama-index for embeddings. You can find their documentation on Azure [here](https://docs.llamaindex.ai/en/stable/api_reference/embeddings/azure_openai/).
 
-### Azure openai configs
+### Azure OpenAI Configuration
 
-The model used for Azure OpenAI embeddings is "text-embedding-ada-002".
-You need the correct deployment name for this model in your Azure account.
-
-When running OpenHands in Docker, set the following environment variables using `-e`:
+When running OpenHands, set the following environment variables using `-e` in the
+[docker run command](/modules/usage/getting-started#installation):
 
 ```
 LLM_EMBEDDING_MODEL="azureopenai"
-LLM_EMBEDDING_DEPLOYMENT_NAME="<your-embedding-deployment-name>"        # e.g. "TextEmbedding...<etc>"
-LLM_API_VERSION="<api-version>"         # e.g. "2024-02-15-preview"
+LLM_EMBEDDING_DEPLOYMENT_NAME="<your-embedding-deployment-name>"   # e.g. "TextEmbedding...<etc>"
+LLM_API_VERSION="<api-version>"                                    # e.g. "2024-02-15-preview"
 ```

docs/modules/usage/llms/google-llms.md

@@ -1,28 +1,30 @@
-# Google Gemini/Vertex LLM
+# Google Gemini/Vertex
 
-## Completion
-
-OpenHands uses LiteLLM for completion calls. The following resources are relevant for using OpenHands with Google's LLMs
+OpenHands uses LiteLLM for completion calls. The following resources are relevant for using OpenHands with Google's LLMs:
 
 - [Gemini - Google AI Studio](https://docs.litellm.ai/docs/providers/gemini)
 - [VertexAI - Google Cloud Platform](https://docs.litellm.ai/docs/providers/vertex)
 
-### Gemini - Google AI Studio Configs
+## Gemini - Google AI Studio Configs
 
-To use Gemini through Google AI Studio when running the OpenHands Docker image, you'll need to set the following environment variables using `-e`:
+When running OpenHands, you'll need to set the following in the OpenHands UI through the Settings:
+* `LLM Provider` to `Gemini`
+* `LLM Model` to the model you will be using.
+If the model is not in the list, toggle `Advanced Options`, and enter it in `Custom Model` (i.e. gemini/<model-name>).
+* `API Key` to your Gemini API key
 
-```
-GEMINI_API_KEY="<your-google-api-key>"
-LLM_MODEL="gemini/gemini-1.5-pro"
-```
+## VertexAI - Google Cloud Platform Configs
 
-### Vertex AI - Google Cloud Platform Configs
-
-To use Vertex AI through Google Cloud Platform when running the OpenHands Docker image, you'll need to set the following environment variables using `-e`:
+To use Vertex AI through Google Cloud Platform when running OpenHands, you'll need to set the following environment
+variables using `-e` in the [docker run command](/modules/usage/getting-started#installation):
 
 ```
 GOOGLE_APPLICATION_CREDENTIALS="<json-dump-of-gcp-service-account-json>"
 VERTEXAI_PROJECT="<your-gcp-project-id>"
 VERTEXAI_LOCATION="<your-gcp-location>"
-LLM_MODEL="vertex_ai/<desired-llm-model>"
 ```
+
+Then set the following in the OpenHands UI through the Settings:
+* `LLM Provider` to `VertexAI`
+* `LLM Model` to the model you will be using.
+If the model is not in the list, toggle `Advanced Options`, and enter it in `Custom Model` (i.e. vertex_ai/&lt;model-name&gt;).

docs/modules/usage/llms/groq.md

@@ -0,0 +1,23 @@
+# Groq
+
+OpenHands uses LiteLLM to make calls to chat models on Groq. You can find their full documentation on using Groq as provider [here](https://docs.litellm.ai/docs/providers/groq).
+
+## Configuration
+
+When running OpenHands, you'll need to set the following in the OpenHands UI through the Settings:
+* `LLM Provider` to `Groq`
+* `LLM Model` to the model you will be using. [Visit **here** to see the list of
+models that Groq hosts](https://console.groq.com/docs/models). If the model is not in the list, toggle
+`Advanced Options`, and enter it in `Custom Model` (i.e. groq/<model-name>)
+* `API key` to your Groq API key. To find or create your Groq API Key, [see **here**](https://console.groq.com/keys)
+
+
+
+## Using Groq as an OpenAI-Compatible Endpoint
+
+The Groq endpoint for chat completion is [mostly OpenAI-compatible](https://console.groq.com/docs/openai). Therefore, you can access Groq models as you
+would access any OpenAI-compatible endpoint. You can set the following in the OpenHands UI through the Settings:
+* Enable `Advanced Options`
+* `Custom Model` to the prefix `openai/` + the model you will be using (Example: `openai/llama3-8b-8192`)
+* `Base URL` to `https://api.groq.com/openai/v1`
+* `API Key` to your Groq API key

docs/modules/usage/llms/llms.md

@@ -1,49 +1,67 @@
 ---
-sidebar_position: 2
+sidebar_position: 3
 ---
 
 # 🤖 LLM Backends
 
-OpenHands can work with any LLM backend.
-For a full list of the LM providers and models available, please consult the
-[litellm documentation](https://docs.litellm.ai/docs/providers).
+OpenHands can connect to any LLM supported by LiteLLM. However, it requires a powerful model to work.
+The following are verified by the community to work with OpenHands:
+
+* claude-3-5-sonnet
+* gemini-1.5-pro / gemini-1.5-flash
+* gpt-4 / gpt-4o
+* llama-3.1-405b / hermes-3-llama-3.1-405b
+* wizardlm-2-8x22b
 
 :::warning
-OpenHands will issue many prompts to the LLM you configure. Most of these LLMs cost money--be sure to set spending limits and monitor usage.
+OpenHands will issue many prompts to the LLM you configure. Most of these LLMs cost money, so be sure to set spending
+limits and monitor usage.
 :::
 
-The `LLM_MODEL` environment variable controls which model is used in programmatic interactions.
-But when using the OpenHands UI, you'll need to choose your model in the settings window.
+If you have successfully run OpenHands with specific LLMs not in the list, please add them to the verified list. We
+also encourage you to open a PR to share your setup process to help others using the same provider and LLM!
 
-The following environment variables might be necessary for some LLMs/providers:
+For a full list of the providers and models available, please consult the
+[litellm documentation](https://docs.litellm.ai/docs/providers).
 
-- `LLM_API_KEY`
-- `LLM_BASE_URL`
-- `LLM_EMBEDDING_MODEL`
-- `LLM_EMBEDDING_DEPLOYMENT_NAME`
-- `LLM_API_VERSION`
-- `LLM_DROP_PARAMS`
+:::note
+Most current local and open source models are not as powerful. When using such models, you may see long
+wait times between messages, poor responses, or errors about malformed JSON. OpenHands can only be as powerful as the
+models driving it. However, if you do find ones that work, please add them to the verified list above.
+:::
+
+## LLM Configuration
+
+The following can be set in the OpenHands UI through the Settings:
+* `LLM Provider`
+* `LLM Model`
+* `API Key`
+* `Base URL` (through `Advanced Settings`)
+
+There are some settings that may be necessary for some LLMs/providers that cannot be set through the UI. Instead, these
+can be set through environment variables passed to the [docker run command](/modules/usage/getting-started#installation)
+using `-e`:
+
+* `LLM_API_VERSION`
+* `LLM_EMBEDDING_MODEL`
+* `LLM_EMBEDDING_DEPLOYMENT_NAME`
+* `LLM_DROP_PARAMS`
+* `LLM_DISABLE_VISION`
+* `LLM_CACHING_PROMPT`
 
 We have a few guides for running OpenHands with specific model providers:
 
-- [OpenAI](llms/openai-llms)
-- [ollama](llms/local-llms)
-- [Azure](llms/azure-llms)
-- [Google](llms/google-llms)
+* [Azure](llms/azure-llms)
+* [Google](llms/google-llms)
+* [Groq](llms/groq)
+* [ollama](llms/local-llms)
+* [OpenAI](llms/openai-llms)
 
-If you're using another provider, we encourage you to open a PR to share your setup!
-
-## Note on Alternative Models
-
-The best models are GPT-4 and Claude 3. Current local and open source models are
-not nearly as powerful. When using an alternative model,
-you may see long wait times between messages,
-poor responses, or errors about malformed JSON. OpenHands
-can only be as powerful as the models driving it--fortunately folks on our team
-are actively working on building better open source models!
-
-## API retries and rate limits
+### API retries and rate limits
 
 Some LLMs have rate limits and may require retries. OpenHands will automatically retry requests if it receives a 429 error or API connection error.
-You can set `LLM_NUM_RETRIES`, `LLM_RETRY_MIN_WAIT`, `LLM_RETRY_MAX_WAIT` environment variables to control the number of retries and the time between retries.
-By default, `LLM_NUM_RETRIES` is 5 and `LLM_RETRY_MIN_WAIT`, `LLM_RETRY_MAX_WAIT` are 3 seconds and 60 seconds respectively.
+You can set the following environment variables to control the number of retries and the time between retries:
+
+* `LLM_NUM_RETRIES` (Default of 8)
+* `LLM_RETRY_MIN_WAIT` (Default of 15 seconds)
+* `LLM_RETRY_MAX_WAIT` (Default of 120 seconds)

docs/modules/usage/llms/local-llms.md

@@ -1,7 +1,11 @@
 # Local LLM with Ollama
 
+:::warning
+When using a Local LLM, OpenHands may have limited functionality.
+:::
+
 Ensure that you have the Ollama server up and running.
-For detailed startup instructions, refer to [here](https://github.com/ollama/ollama)
+For detailed startup instructions, refer to [here](https://github.com/ollama/ollama).
 
 This guide assumes you've started ollama with `ollama serve`. If you're running ollama differently (e.g. inside docker), the instructions might need to be modified. Please note that if you're running WSL the default ollama configuration blocks requests from docker containers. See [here](#configuring-ollama-service-wsl-en).
 
@@ -28,7 +32,7 @@ starcoder2:latest               f67ae0f64584    1.7 GB  19 hours ago
 
 ### Docker
 
-Use the instructions [here](../intro) to start OpenHands using Docker.
+Use the instructions [here](../getting-started) to start OpenHands using Docker.
 But when running `docker run`, you'll need to add a few more arguments:
 
 ```bash
@@ -196,9 +200,9 @@ base_url="http://localhost:1234/v1"
 custom_llm_provider="openai"
 ```
 
-Done! Now you can start Devin by: `make run` without Docker. You now should be able to connect to `http://localhost:3000/`
+Done! Now you can start OpenHands by: `make run` without Docker. You now should be able to connect to `http://localhost:3000/`
 
-# Note:
+# Note
 
 For WSL, run the following commands in cmd to set up the networking mode to mirrored:
 

docs/modules/usage/llms/openai-llms.md

@@ -1,75 +1,24 @@
 # OpenAI
 
-OpenHands uses [LiteLLM](https://www.litellm.ai/) to make calls to OpenAI's chat models. You can find their full documentation on OpenAI chat calls [here](https://docs.litellm.ai/docs/providers/openai).
+OpenHands uses LiteLLM to make calls to OpenAI's chat models. You can find their full documentation on OpenAI chat calls [here](https://docs.litellm.ai/docs/providers/openai).
 
 ## Configuration
 
-### Manual Configuration
-
-When running the OpenHands Docker image, you'll need to set the following environment variables:
-
-```sh
-LLM_MODEL="openai/<gpt-model-name>" # e.g. "openai/gpt-4o"
-LLM_API_KEY="<your-openai-project-api-key>"
-```
-
-To see a full list of OpenAI models that LiteLLM supports, please visit https://docs.litellm.ai/docs/providers/openai#openai-chat-completion-models.
-
-To find or create your OpenAI Project API Key, please visit https://platform.openai.com/api-keys.
-
-**Example**:
-
-```sh
-export WORKSPACE_BASE=$(pwd)/workspace
-
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e LLM_MODEL="openai/<gpt-model-name>" \
-    -e LLM_API_KEY="<your-openai-project-api-key>" \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    -p 3000:3000 \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/opendevin/opendevin:0.8
-```
-
-### UI Configuration
-
-You can also directly set the `LLM_MODEL` and `LLM_API_KEY` in the OpenHands client itself. Follow this guide to get up and running with the OpenHands client.
-
-From there, you can set your model and API key in the settings window.
+When running OpenHands, you'll need to set the following in the OpenHands UI through the Settings:
+* `LLM Provider` to `OpenAI`
+* `LLM Model` to the model you will be using.
+[Visit **here** to see a full list of OpenAI models that LiteLLM supports.](https://docs.litellm.ai/docs/providers/openai#openai-chat-completion-models)
+If the model is not in the list, toggle `Advanced Options`, and enter it in `Custom Model` (i.e. openai/&lt;model-name&gt;).
+* `API Key` to your OpenAI API key. To find or create your OpenAI Project API Key, [see **here**](https://platform.openai.com/api-keys).
 
 ## Using OpenAI-Compatible Endpoints
 
 Just as for OpenAI Chat completions, we use LiteLLM for OpenAI-compatible endpoints. You can find their full documentation on this topic [here](https://docs.litellm.ai/docs/providers/openai_compatible).
 
-When running the OpenHands Docker image, you'll need to set the following environment variables:
+## Using an OpenAI Proxy
 
-```sh
-LLM_BASE_URL="<api-base-url>" # e.g. "http://0.0.0.0:3000"
-LLM_MODEL="openai/<model-name>" # e.g. "openai/mistral"
-LLM_API_KEY="<your-api-key>"
-```
-
-**Example**:
-
-```sh
-export WORKSPACE_BASE=$(pwd)/workspace
-
-docker run -it \
-    --pull=always \
-    -e SANDBOX_USER_ID=$(id -u) \
-    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
-    -e LLM_BASE_URL="<api-base-url>" \
-    -e LLM_MODEL="openai/<model-name>" \
-    -e LLM_API_KEY="<your-api-key>" \
-    -v $WORKSPACE_BASE:/opt/workspace_base \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    -p 3000:3000 \
-    --add-host host.docker.internal:host-gateway \
-    --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    ghcr.io/opendevin/opendevin:0.8
-```
+If you're using an OpenAI proxy, you'll need to set the following in the OpenHands UI through the Settings:
+* Enable `Advanced Options`
+* `Custom Model` to openai/&lt;model-name&gt; (i.e.: `openai/gpt-4o` or openai/&lt;proxy-prefix&gt;/&lt;model-name&gt;)
+* `Base URL` to the URL of your OpenAI proxy
+* `API Key` to your OpenAI API key

docs/modules/usage/troubleshooting/troubleshooting.md

@@ -5,24 +5,19 @@ sidebar_position: 4
 # 🚧 Troubleshooting
 
 There are some error messages that frequently get reported by users.
-
-We'll try to make the install process easier and these error messages
-better in the future. But for now, you can look for your error message below and see if there are any workarounds.
-
-For each of these error messages **there is an existing issue**. Please do not
-open a new issue--just comment there.
-
-If you find more information or a workaround for one of these issues, please
-open a *PR* to add details to this file.
+We'll try to make the install process easier, but for now you can look for your error message below and see if there are any workarounds.
+If you find more information or a workaround for one of these issues, please open a *PR* to add details to this file.
 
 :::tip
-If you're running on Windows and having trouble, check out our [guide for Windows (WSL) users](troubleshooting/windows).
+OpenHands only supports Windows via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+Please be sure to run all commands inside your WSL terminal.
+Check out [Notes for WSL on Windows Users](troubleshooting/windows) for some troubleshooting guides.
 :::
 
 ## Common Issues
 
 * [Unable to connect to Docker](#unable-to-connect-to-docker)
-* [Unable to connect to SSH box](#unable-to-connect-to-ssh-box)
+* [Unable to connect to LLM](#unable-to-connect-to-llm)
 * [404 Resource not found](#404-resource-not-found)
 * [`make build` getting stuck on package installations](#make-build-getting-stuck-on-package-installations)
 * [Sessions are not restored](#sessions-are-not-restored)
@@ -52,31 +47,6 @@ OpenHands uses a Docker container to do its work safely, without potentially bre
 * If you are on a Mac, check the [permissions requirements](https://docs.docker.com/desktop/mac/permission-requirements/) and in particular consider enabling the `Allow the default Docker socket to be used` under `Settings > Advanced` in Docker Desktop.
 * In addition, upgrade your Docker to the latest version under `Check for Updates`
 
----
-### Unable to connect to SSH box
-
-[GitHub Issue](https://github.com/All-Hands-AI/OpenHands/issues/1156)
-
-**Symptoms**
-
-```python
-self.shell = DockerSSHBox(
-...
-pexpect.pxssh.ExceptionPxssh: Could not establish connection to host
-```
-
-**Details**
-
-By default, OpenHands connects to a running container using SSH. On some machines,
-especially Windows, this seems to fail.
-
-**Workarounds**
-
-* Restart your computer (sometimes it does work)
-* Be sure to have the latest versions of WSL and Docker
-* Check that your distribution in WSL is up to date as well
-* Try [this reinstallation guide](https://github.com/All-Hands-AI/OpenHands/issues/1156#issuecomment-2064549427)
-
 ---
 ### Unable to connect to LLM
 
@@ -141,7 +111,7 @@ the API endpoint you're trying to connect to. Most often this happens for Azure
 **Workarounds**
 
 * Check that you've set `LLM_BASE_URL` properly
-* Check that model is set properly, based on the [LiteLLM docs](https://docs.litellm.ai/docs/providers)
+* Check that the model is set properly, based on the [LiteLLM docs](https://docs.litellm.ai/docs/providers)
   * If you're running inside the UI, be sure to set the `model` in the settings modal
   * If you're running headless (via main.py) be sure to set `LLM_MODEL` in your env/config
 * Make sure you've followed any special instructions for your LLM provider

docs/modules/usage/troubleshooting/windows.md

@@ -1,4 +1,4 @@
-# Notes for Windows and WSL Users
+# Notes for WSL on Windows Users
 
 OpenHands only supports Windows via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 Please be sure to run all commands inside your WSL terminal.
@@ -8,12 +8,11 @@ Please be sure to run all commands inside your WSL terminal.
 ### Recommendation: Do not run as root user
 
 For security reasons, it is highly recommended to not run OpenHands as the root user, but a user with a non-zero UID.
-In addition, persistent sandboxes won't be supported when running as root and during start of OpenHands an appropriate message may appear.
 
 References:
 
 * [Why it is bad to login as root](https://askubuntu.com/questions/16178/why-is-it-bad-to-log-in-as-root)
-* [Set default user in WSL](https://www.tenforums.com/tutorials/128152-set-default-user-windows-subsystem-linux-distro-windows-10-a.html#option2)  
+* [Set default user in WSL](https://www.tenforums.com/tutorials/128152-set-default-user-windows-subsystem-linux-distro-windows-10-a.html#option2)
 Hint about the 2nd reference: for Ubuntu users, the command could actually be "ubuntupreview" instead of "ubuntu".
 
 ---
@@ -22,21 +21,6 @@ Hint about the 2nd reference: for Ubuntu users, the command could actually be "u
 If you are using Docker Desktop, make sure to start it before calling any docker command from inside WSL.
 Docker also needs to have the WSL integration option activated.
 
----
-### Failed to create openhands user
-
-If you encounter the following error during setup:
-
-```sh
-Exception: Failed to create openhands user in sandbox: 'useradd: UID 0 is not unique'
- ```
-
-You can resolve it by running:
-
-```sh
-export SANDBOX_USER_ID=1000
-```
-
 ---
 ### Poetry Installation
 
@@ -76,5 +60,5 @@ localhostForwarding=true
 
 * Save the `.wslconfig` file.
 * Restart WSL2 completely by exiting any running WSL2 instances and executing the command `wsl --shutdown` in your command prompt or terminal.
-* After restarting WSL, attempt to execute `make run` again.  
-The networking issue should be resolved.
+* After restarting WSL, attempt to execute `make run` again.
+The networking issue should be resolved.

docs/package-lock.json

@@ -14,7 +14,7 @@
         "@docusaurus/theme-mermaid": "^3.5.2",
         "@mdx-js/react": "^3.0.0",
         "clsx": "^2.0.0",
-        "prism-react-renderer": "^2.3.0",
+        "prism-react-renderer": "^2.4.0",
         "react": "^18.3.1",
         "react-dom": "^18.3.1",
         "react-icons": "^5.3.0",
@@ -24,7 +24,7 @@
         "@docusaurus/module-type-aliases": "^3.5.1",
         "@docusaurus/tsconfig": "^3.5.2",
         "@docusaurus/types": "^3.5.1",
-        "typescript": "~5.5.4"
+        "typescript": "~5.6.2"
       },
       "engines": {
         "node": ">=18.0"
@@ -12640,9 +12640,9 @@
       }
     },
     "node_modules/prism-react-renderer": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.3.1.tgz",
-      "integrity": "sha512-Rdf+HzBLR7KYjzpJ1rSoxT9ioO85nZngQEoFIhL07XhtJHlCU3SOz0GJ6+qvMyQe0Se+BV3qpe6Yd/NmQF5Juw==",
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.0.tgz",
+      "integrity": "sha512-327BsVCD/unU4CNLZTWVHyUHKnsqcvj2qbPlQ8MiBE2eq2rgctjigPA1Gp9HLF83kZ20zNN6jgizHJeEsyFYOw==",
       "dependencies": {
         "@types/prismjs": "^1.26.0",
         "clsx": "^2.0.0"
@@ -14853,9 +14853,9 @@
       }
     },
     "node_modules/typescript": {
-      "version": "5.5.4",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz",
-      "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==",
+      "version": "5.6.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
+      "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"

docs/package.json

@@ -21,7 +21,7 @@
     "@docusaurus/theme-mermaid": "^3.5.2",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.0.0",
-    "prism-react-renderer": "^2.3.0",
+    "prism-react-renderer": "^2.4.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-icons": "^5.3.0",
@@ -31,7 +31,7 @@
     "@docusaurus/module-type-aliases": "^3.5.1",
     "@docusaurus/tsconfig": "^3.5.2",
     "@docusaurus/types": "^3.5.1",
-    "typescript": "~5.5.4"
+    "typescript": "~5.6.2"
   },
   "browserslist": {
     "production": [

docs/sidebars.ts

@@ -1,8 +1,113 @@
 import type { SidebarsConfig } from "@docusaurus/plugin-content-docs";
 
 const sidebars: SidebarsConfig = {
-  docsSidebar: [{ type: "autogenerated", dirName: "usage" }],
   apiSidebar: [require("./modules/python/sidebar.json")],
+  docsSidebar: [
+    {
+      type: 'doc',
+      label: 'Getting Started',
+      id: 'usage/getting-started',
+    },
+    {
+      type: 'category',
+      label: 'LLMs',
+      items: [
+        {
+          type: 'doc',
+          label: 'Overview',
+          id: 'usage/llms/llms',
+        },
+        {
+          type: 'category',
+          label: 'Providers',
+          items: [
+            {
+              type: 'doc',
+              label: 'OpenAI',
+              id: 'usage/llms/openai-llms',
+            },
+            {
+              type: 'doc',
+              label: 'Azure',
+              id: 'usage/llms/azure-llms',
+            },
+            {
+              type: 'doc',
+              label: 'Google',
+              id: 'usage/llms/google-llms',
+            },
+            {
+              type: 'doc',
+              label: 'Groq',
+              id: 'usage/llms/groq',
+            },
+            {
+              type: 'doc',
+              label: 'Local/ollama',
+              id: 'usage/llms/local-llms',
+            }
+          ],
+        },
+      ],
+    },
+    {
+      type: 'doc',
+      label: 'Troubleshooting',
+      id: 'usage/troubleshooting/troubleshooting',
+    },
+    {
+      type: 'doc',
+      label: 'Feedback',
+      id: 'usage/feedback',
+    },
+    {
+      type: 'category',
+      label: 'How-to Guides',
+      items: [
+        {
+          type: 'doc',
+          id: 'usage/how-to/cli-mode',
+        },
+        {
+          type: 'doc',
+          id: 'usage/how-to/headless-mode',
+        },
+        {
+          type: 'doc',
+          id: 'usage/how-to/custom-sandbox-guide',
+        },
+        {
+          type: 'doc',
+          id: 'usage/how-to/evaluation-harness',
+        },
+        {
+          type: 'doc',
+          id: 'usage/how-to/openshift-example',
+        }
+      ]
+    },
+    {
+      type: 'category',
+      label: 'Architecture',
+      items: [
+        {
+          type: 'doc',
+          label: 'Backend',
+          id: 'usage/architecture/backend',
+        },
+        {
+          type: 'doc',
+          label: 'Runtime',
+          id: 'usage/architecture/runtime',
+        }
+      ],
+    },
+    {
+      type: 'doc',
+      label: 'About',
+      id: 'usage/about',
+    }
+  ],
 };
 
 export default sidebars;

docs/src/components/CustomFooter.tsx

@@ -7,17 +7,6 @@ function CustomFooter() {
   return (
     <footer className="custom-footer">
       <div className="footer-content">
-        <div className="footer-top">
-          <div className="footer-title">
-            <Translate id="footer.title">OpenHands</Translate>
-          </div>
-          <div className="footer-link">
-            <a href="/modules/usage/intro">
-              <Translate id="footer.docs">Docs</Translate>
-            </a>
-          </div>
-        </div>
-
         <div className="footer-icons">
           <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA" target="_blank" rel="noopener noreferrer">
             <FaSlack />
@@ -32,7 +21,7 @@ function CustomFooter() {
         <div className="footer-bottom">
           <p>
             <Translate id="footer.copyright" values={{ year: new Date().getFullYear() }}>
-              {'Copyright © {year} OpenHands'}
+              {'Copyright © {year} All Hands AI, Inc'}
             </Translate>
           </p>
         </div>

docs/src/components/HomepageHeader/HomepageHeader.tsx

@@ -17,23 +17,19 @@ export function HomepageHeader() {
 
         <p className="header-subtitle">{siteConfig.tagline}</p>
 
-        <div className="header-links">
-          <a href="https://github.com/All-Hands-AI/OpenHands">
-            <img src="https://img.shields.io/badge/Code-Github-purple?logo=github&logoColor=white&style=for-the-badge" alt="Code" />
-          </a>
-          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA">
-            <img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" />
-          </a>
-          <a href="https://discord.gg/ESHStjSjD4">
-            <img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community" />
-          </a>
-
-          <a href="https://arxiv.org/abs/2407.16741">
-            <img src="https://img.shields.io/badge/Paper-%20on%20Arxiv-red?logo=arxiv&style=for-the-badge" alt="Paper on Arxiv" />
-          </a>
-          <a href="https://huggingface.co/spaces/OpenHands/evaluation">
-            <img src="https://img.shields.io/badge/Evaluation-Benchmark%20on%20HF%20Space-green?logo=huggingface&style=for-the-badge" alt="Evaluation Benchmark" />
-          </a>
+        <div align="center" className="header-links">
+          <a href="https://github.com/All-Hands-AI/OpenHands/graphs/contributors"><img src="https://img.shields.io/github/contributors/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Contributors" /></a>
+          <a href="https://github.com/All-Hands-AI/OpenHands/stargazers"><img src="https://img.shields.io/github/stars/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="Stargazers" /></a>
+          <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" /></a>
+          <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License" /></a>
+          <br/>
+          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
+          <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community" /></a>
+          <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits" /></a>
+          <br/>
+          <a href="https://docs.all-hands.dev/modules/usage/getting-started"><img src="https://img.shields.io/badge/Documentation-000?logo=googledocs&logoColor=FFE165&style=for-the-badge" alt="Check out the documentation" /></a>
+          <a href="https://arxiv.org/abs/2407.16741"><img src="https://img.shields.io/badge/Paper%20on%20Arxiv-000?logoColor=FFE165&logo=arxiv&style=for-the-badge" alt="Paper on Arxiv" /></a>
+          <a href="https://huggingface.co/spaces/OpenHands/evaluation"><img src="https://img.shields.io/badge/Benchmark%20score-000?logoColor=FFE165&logo=huggingface&style=for-the-badge" alt="Evaluation Benchmark Score" /></a>
         </div>
 
         <Demo />

docs/src/css/custom.css

@@ -7,8 +7,10 @@
 /* You can override the default Infima variables here. */
 
 :root {
-  --ifm-color-primary: #4465db;
   --ifm-code-font-size: 95%;
+  --ifm-color-primary: #000;
+  --ifm-background-color: #F1EAE0;
+  --ifm-navbar-background-color: #F1EAE0;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
   --secondary: #171717;
   --secondary-dark: #0a0a0a;
@@ -17,21 +19,15 @@
 
 /* For readability concerns, you should choose a lighter palette in dark mode. */
 [data-theme="dark"] {
-  --ifm-color-primary: #4465db;
+  --ifm-color-primary: #FFF;
+  --ifm-background-color: #000;
+  --ifm-navbar-background-color: #000;
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
   --secondary: #737373;
   --secondary-dark: #171717;
-  --secondary-light: #d4d4d4;
+  --secondary-light: #ccc;
 }
 
-.footer--dark {
-  background-image: linear-gradient(
-    140deg,
-    var(--secondary) 20%,
-    var(--secondary-light) 100%
-  );
-}
-
-.a {
+article a, .a {
   text-decoration: underline;
-}
+}

docs/src/css/footer.css

@@ -2,13 +2,19 @@
 
 .custom-footer {
     background-color: dark;
-    color: white;
-    height: 200px;
+    color: #000;
+    height: 100px;
     /* background: linear-gradient(to bottom, #1a1a1a, #1a1a1a); */
-    background: linear-gradient(to bottom, #1f2937, #000000);
+    background-color: #F1EAE0;
 
   }
 
+[data-theme="dark"] .custom-footer {
+  background-color: #000;
+  color: #fff;
+}
+
+
   .footer-content {
     display: flex;
     flex-direction: column;
@@ -47,7 +53,6 @@
   }
 
   .footer-community {
-    text-transform: uppercase;
     font-weight: 300;
   }
 
@@ -65,7 +70,3 @@
   .footer-icons a:hover {
     color: white;
   }
-
-  .footer-bottom {
-    text-transform: uppercase;
-  }

docs/src/css/homepageHeader.css

@@ -2,8 +2,6 @@
 
 .homepage-header {
   height: 800px;
-  color: white;
-  background: linear-gradient(to top, #64748b, #000000);
 }
 
 .header-content {

docs/src/pages/index.tsx

@@ -20,8 +20,7 @@ export default function Home(): JSX.Element {
       title={`${siteConfig.title}`}
       description={translate({
         id: 'homepage.description',
-        message: 'An Open Platform for AI Software Developers as Generalist Agents',
-        description: 'The homepage description',
+        message: 'Code Less, Make More',
       })}
     >
     <HomepageHeader />

evaluation/EDA/run_infer.py

@@ -74,7 +74,7 @@ def get_config(
     return config
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -117,13 +117,17 @@ async def process_instance(
     instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    runtime = await create_runtime(config, sid=instance['text'].strip())
+    runtime = create_runtime(config, sid=instance['text'].strip())
 
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+        )
     )
     # ======= Attempt to evaluate the agent's edits =======
     # If you are working on simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
@@ -214,12 +218,10 @@ if __name__ == '__main__':
         eda_dataset.to_pandas(), output_file, args.eval_n_limit
     )
 
-    asyncio.run(
-        run_evaluation(
-            prepared_dataset,
-            metadata,
-            output_file,
-            args.eval_num_workers,
-            process_instance,
-        )
+    run_evaluation(
+        prepared_dataset,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
     )

evaluation/README.md

@@ -3,16 +3,18 @@
 This folder contains code and resources to run experiments and evaluations.
 
 ## Logistics
+
 To better organize the evaluation folder, we should follow the rules below:
-  - Each subfolder contains a specific benchmark or experiment. For example, `evaluation/swe_bench` should contain
+
+- Each subfolder contains a specific benchmark or experiment. For example, `evaluation/swe_bench` should contain
 all the preprocessing/evaluation/analysis scripts.
-  - Raw data and experimental records should not be stored within this repo.
-    - For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.
-  - Important data files of manageable size and analysis scripts (e.g., jupyter notebooks) can be directly uploaded to this repo.
+- Raw data and experimental records should not be stored within this repo.
+- For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.
+- Important data files of manageable size and analysis scripts (e.g., jupyter notebooks) can be directly uploaded to this repo.
 
 ## Supported Benchmarks
 
-To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/evaluation_harness).
+To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/how-to/evaluation-harness).
 
 ### Software Engineering
 
@@ -23,6 +25,7 @@ To learn more about how to integrate your benchmark into OpenHands, check out [t
 - ML-Bench: [`evaluation/ml_bench`](./ml_bench)
 - APIBench: [`evaluation/gorilla`](./gorilla/)
 - ToolQA: [`evaluation/toolqa`](./toolqa/)
+- AiderBench: [`evaluation/aider_bench`](./aider_bench/)
 
 ### Web Browsing
 
@@ -38,7 +41,6 @@ To learn more about how to integrate your benchmark into OpenHands, check out [t
 - Entity deduction Arena (EDA): [`evaluation/EDA`](./EDA)
 - ProofWriter: [`evaluation/logic_reasoning`](./logic_reasoning)
 
-
 ## Before everything begins: Setup Environment and LLM Configuration
 
 Please follow instruction [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
@@ -65,12 +67,10 @@ api_key = "XXX"
 temperature = 0.0
 ```
 
-
 ### Result Visualization
 
 Check [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization of existing experimental results.
 
-
 ### Upload your results
 
 You can start your own fork of [our huggingface evaluation outputs](https://huggingface.co/spaces/OpenHands/evaluation) and submit a PR of your evaluation results to our hosted huggingface repo via PR following the guide [here](https://huggingface.co/docs/hub/en/repositories-pull-requests-discussions#pull-requests-and-discussions).

evaluation/agent_bench/run_infer.py

@@ -56,7 +56,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -70,12 +70,12 @@ async def initialize_runtime(
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     init_cmd = instance.init
@@ -85,7 +85,7 @@ async def initialize_runtime(
         with tempfile.TemporaryDirectory() as tmpdir:
             host_script_path = os.path.join(tmpdir, script_name)
             create_sh_file(host_script_path, init_cmd)
-            await runtime.copy_to(
+            runtime.copy_to(
                 host_script_path,
                 '/workspace',
             )
@@ -93,14 +93,14 @@ async def initialize_runtime(
         logger.info(f'Running init script: {script_name}')
         action = CmdRunAction(command=f'chmod +x ./{script_name} && ./{script_name}')
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert obs.exit_code == 0
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -121,7 +121,7 @@ async def complete_runtime(
         with tempfile.TemporaryDirectory() as tmpdir:
             host_script_path = os.path.join(tmpdir, script_name)
             create_sh_file(host_script_path, get_agent_result_cmd)
-            await runtime.copy_to(
+            runtime.copy_to(
                 host_script_path,
                 '/workspace',
             )
@@ -132,7 +132,7 @@ async def complete_runtime(
             keep_prompt=False,
         )
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert obs.exit_code == 0
         agent_answer = obs.content
@@ -149,7 +149,7 @@ async def complete_runtime(
             with tempfile.TemporaryDirectory() as tmpdir:
                 host_script_path = os.path.join(tmpdir, script_name)
                 create_sh_file(host_script_path, get_ground_truth_cmd)
-                await runtime.copy_to(
+                runtime.copy_to(
                     host_script_path,
                     '/workspace',
                 )
@@ -160,7 +160,7 @@ async def complete_runtime(
                 keep_prompt=False,
             )
             logger.info(action, extra={'msg_type': 'ACTION'})
-            obs = await runtime.run_action(action)
+            obs = runtime.run_action(action)
             logger.info(obs, extra={'msg_type': 'OBSERVATION'})
             final_ans = obs.content
 
@@ -171,7 +171,7 @@ async def complete_runtime(
     }
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -209,16 +209,18 @@ async def process_instance(
     # create sandbox and run the agent
     # =============================================
 
-    runtime: Runtime = await create_runtime(config, sid=instance.instance_id)
+    runtime: Runtime = create_runtime(config, sid=instance.instance_id)
 
-    await initialize_runtime(runtime, instance=instance)
+    initialize_runtime(runtime, instance=instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+        )
     )
     if state is None:
         raise ValueError('State should not be None.')
@@ -227,7 +229,7 @@ async def process_instance(
     # result evaluation
     # =============================================
 
-    return_val = await complete_runtime(runtime, instance)
+    return_val = complete_runtime(runtime, instance)
     agent_answer = return_val['agent_answer']
     final_ans = return_val['final_ans']
 
@@ -313,8 +315,6 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(agent_bench_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/aider_bench/README.md

@@ -16,52 +16,86 @@ development environment and LLM.
 ## Start the evaluation
 
 ```bash
-./evaluation/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
 ```
 
--   `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
     your LLM settings, as defined in your `config.toml`.
--   `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version
-    you would like to evaluate. It could also be a release tag like `0.6.2`.
--   `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks,
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version
+    you would like to evaluate. It could also be a release tag like `0.9.0`.
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks,
     defaulting to `CodeActAgent`.
--   `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit`
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit`
     instances. By default, the script evaluates the entire Exercism test set
     (133 issues). Note: in order to use `eval_limit`, you must also set `agent`.
+- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`.
+- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the
+    given IDs (comma separated).
+
+There are also following optional environment variables you can set:
+
+```bash
+export USE_UNIT_TESTS=true # if you want to allow the Agent to verify correctness using unittests. Default to false.
+export SKIP_NUM=12 # skip the first 12 instances from the dataset
+```
 
 Following is the basic command to start the evaluation.
 
 You can update the arguments in the script
-`evaluation/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`,
-`--eval-num-workers` and so on.
+`evaluation/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
+`--eval-num-workers` and so on:
 
--   `--agent-cls`, the agent to use. For example, `CodeActAgent`.
--   `--llm-config`: the LLM configuration to use. For example,
-    `eval_gpt4_1106_preview`.
--   `--max-iterations`: the number of iterations to run the evaluation. For
-    example, `30`.
--   `--eval-num-workers`: the number of workers to use for evaluation. For
-    example, `5`.
--   `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
+- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
+- `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
+- `--max-iterations`: the max allowed number of iterations to run the evaluation. Default: `30`.
+- `--eval-num-workers`: the number of workers to use for evaluation. Default: `1`.
+- `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
+- `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`.
 
 ```bash
-./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
+./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
 ```
 
 ## Summarize Results
 
 ```bash
-poetry run python ./evaluation/agent_bench/scripts/summarise_results.py [path_to_output_jsonl_file]
+poetry run python ./evaluation/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file] [model_name]
+# with optional SKIP_NUM
+poetry run python SKIP_NUM=12 ./evaluation/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file] [model_name]
+```
+
+Full example:
+
+```bash
+poetry run python ./evaluation/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl claude-3-5-sonnet@20240620
 ```
 
 This will list the instances that passed and the instances that failed. For each
 instance, the corresponding set of test cases (which can vary for each instance)
 are run on the file edited by the agent. We consider an instance to be passed
 only if ALL test cases are passed. Sometimes even a single failed test case will
-cause the entire instance to be marked as filed.
+cause the entire instance to be marked as failed.
 
-You can inspect the test_results field in the output json file to know the exact
+You can inspect the `test_results` field in the `output.jsonl` file to find the exact
 outcome of the tests. If there are no syntax or indentation errors, you can
-expect to see something like "..F...EF..", where "." means the test case
-passed, "E" means there was an error while executing the test case and "F"
-means some assertion failed and returned output was not as expected.
+expect to see something like "`..F...EF..`", where "`.`" means the test case
+passed, "`E`" means there was an error while executing the test case and "`F`"
+means some assertion failed and some returned output was not as expected.
+
+## Visualization
+
+If the required Python libraries are installed (`matplotlib.pyplot` and `seaborn`),
+the `summarize_results.py` script will also generate two histograms to
+the output folder.
+
+### Cost Histogram
+
+The cost histogram shows the number of successful and failed instances per cost point.
+
+![Cost Histogram](./examples/cost_histogram.png)
+
+### Actions Histogram
+
+The actions histogram shows per number of actions the number of successful and failed instances.
+
+![Actions Histogram](./examples/actions_histogram.png)

evaluation/aider_bench/run_infer.py

@@ -32,6 +32,13 @@ from openhands.events.action import CmdRunAction
 from openhands.events.observation import CmdOutputObservation
 from openhands.runtime.runtime import Runtime
 
+# Configure visibility of unit tests to the Agent.
+USE_UNIT_TESTS = os.environ.get('USE_UNIT_TESTS', 'false').lower() == 'true'
+SKIP_NUM = os.environ.get('SKIP_NUM')
+SKIP_NUM = (
+    int(SKIP_NUM) if SKIP_NUM and SKIP_NUM.isdigit() and int(SKIP_NUM) >= 0 else None
+)
+
 
 def get_config(
     metadata: EvalMetadata,
@@ -55,7 +62,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,
 ):
@@ -63,32 +70,40 @@ async def initialize_runtime(
 
     This function is called before the runtime is used to run the agent.
     """
-    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    logger.info(f"\n{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}\n")
     obs: CmdOutputObservation
 
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     with tempfile.TemporaryDirectory() as tmpdir:
         file_path = os.path.join(tmpdir, f'{instance.instance_name}.py')
         with open(file_path, 'w') as f:
             f.write(instance.signature)
-        await runtime.copy_to(
+        runtime.copy_to(
             file_path,
             '/workspace',
         )
-    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+        if USE_UNIT_TESTS:
+            file_path = os.path.join(tmpdir, f'{instance.instance_name}_test.py')
+            with open(file_path, 'w') as f:
+                f.write(instance.test)
+            runtime.copy_to(
+                file_path,
+                '/workspace',
+            )
+    logger.info(f"\n{'-' * 50} END Runtime Initialization Fn {'-' * 50}\n")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,
 ) -> dict[str, Any]:
@@ -98,15 +113,16 @@ async def complete_runtime(
     If you need to do something in the sandbox to get the correctness metric after
     the agent has run, modify this function.
     """
-    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+    logger.info(f"\n{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}\n")
     obs: CmdOutputObservation
 
+    # Rewriting the test file to ignore any changes Agent may have made.
     script_name = f'{instance.instance_name}_test.py'
     with tempfile.TemporaryDirectory() as tmpdir:
         file_path = os.path.join(tmpdir, script_name)
         with open(file_path, 'w') as f:
             f.write(instance.test)
-        await runtime.copy_to(
+        runtime.copy_to(
             file_path,
             '/workspace',
         )
@@ -117,14 +133,16 @@ async def complete_runtime(
         keep_prompt=False,
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
 
     exit_code = 1
     if isinstance(obs, CmdOutputObservation):
         exit_code = obs.exit_code
 
-    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+    logger.info(f"\n{'-' * 50} END Runtime Completion Fn {'-' * 50}\n")
+
+    runtime.close()
 
     return {
         'test_output': obs.content,
@@ -132,7 +150,7 @@ async def complete_runtime(
     }
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -144,7 +162,9 @@ async def process_instance(
         log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
         reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir)
     else:
-        logger.info(f'Starting evaluation for instance {str(instance.instance_id)}.')
+        logger.info(
+            f'\nStarting evaluation for instance {str(instance.instance_id)}.\n'
+        )
 
     # =============================================
     # build instruction
@@ -156,6 +176,13 @@ async def process_instance(
     instruction += INSTRUCTIONS_ADDENDUM.format(
         signature_file=f'{instance.instance_name}.py',
     )
+    if USE_UNIT_TESTS:
+        print(f'\nInstruction to run test_file: {instance.instance_name}_test.py\n')
+        instruction += (
+            f'Use `python -m unittest {instance.instance_name}_test.py` to run the test_file '
+            'and verify the correctness of your solution. DO NOT EDIT the test file.\n\n'
+        )
+
     instruction += (
         'IMPORTANT: You should ONLY interact with the environment provided '
         'to you AND NEVER ASK FOR HUMAN HELP.\n'
@@ -167,16 +194,18 @@ async def process_instance(
     # create sandbox and run the agent
     # =============================================
 
-    runtime: Runtime = await create_runtime(config, sid=str(instance.instance_id))
+    runtime: Runtime = create_runtime(config, sid=str(instance.instance_id))
 
-    await initialize_runtime(runtime, instance=instance)
+    initialize_runtime(runtime, instance=instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class],
+        )
     )
     if state is None:
         raise ValueError('State should not be None.')
@@ -185,7 +214,7 @@ async def process_instance(
     # # result evaluation
     # # =============================================
 
-    return_val = await complete_runtime(runtime, instance)
+    return_val = complete_runtime(runtime, instance)
     exit_code = return_val['exit_code']
     test_output = return_val['test_output']
 
@@ -245,14 +274,25 @@ if __name__ == '__main__':
         args.eval_output_dir,
     )
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-    instances = prepare_dataset(aider_bench_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances,
-            metadata,
-            output_file,
-            args.eval_num_workers,
-            process_instance,
-        )
+    # Parse dataset IDs if provided
+    eval_ids = None
+    if args.eval_ids:
+        eval_ids = str(args.eval_ids).split(',')
+        logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n')
+
+    instances = prepare_dataset(
+        aider_bench_tests,
+        output_file,
+        args.eval_n_limit,
+        eval_ids=eval_ids,
+        skip_num=SKIP_NUM,
+    )
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
     )

evaluation/aider_bench/scripts/run_infer.sh

@@ -8,6 +8,7 @@ COMMIT_HASH=$2
 AGENT=$3
 EVAL_LIMIT=$4
 NUM_WORKERS=$5
+EVAL_IDS=$6
 
 if [ -z "$NUM_WORKERS" ]; then
   NUM_WORKERS=1
@@ -34,10 +35,21 @@ COMMAND="export PYTHONPATH=evaluation/aider_bench:\$PYTHONPATH && poetry run pyt
   --eval-num-workers $NUM_WORKERS \
   --eval-note $AGENT_VERSION"
 
+# Default to NOT use unit tests.
+if [ -z "$USE_UNIT_TESTS" ]; then
+  export USE_UNIT_TESTS=false
+fi
+echo "USE_UNIT_TESTS: $USE_UNIT_TESTS"
+
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"
   COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
 fi
 
+if [ -n "$EVAL_IDS" ]; then
+  echo "EVAL_IDS: $EVAL_IDS"
+  COMMAND="$COMMAND --eval-ids $EVAL_IDS"
+fi
+
 # Run the command
 eval $COMMAND

evaluation/aider_bench/scripts/summarize_results.py

@@ -1,6 +1,36 @@
 import json
+import os
 import sys
 
+import numpy as np
+import pandas as pd
+
+# Try to import visualization libraries
+visualization_available = False
+try:
+    import matplotlib.pyplot as plt
+    import seaborn as sns
+
+    visualization_available = True
+except ImportError:
+    print(
+        '\n*** WARNING: libraries matplotlib and/or seaborn are not installed.\n*** Visualization will not be available!\n'
+    )
+
+
+def show_usage():
+    print(
+        'Usage: poetry run python summarize_results.py <path_to_output_jsonl_file> <model_name>'
+    )
+    print(
+        'Example:\npoetry run python summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl claude-3-5-sonnet@20240620\n'
+    )
+
+
+def print_error(message: str):
+    print(f'\n***\n*** ERROR: {message}\n***\n')
+    show_usage()
+
 
 def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:
     passed = []
@@ -19,19 +49,87 @@ def extract_test_results(res_file_path: str) -> tuple[list[str], list[str]]:
     return passed, failed
 
 
+def visualize_results(json_file_path: str, model: str, output_dir: str):
+    # based on a Colab notebook by RajMaheshwari
+    with open(json_file_path, 'r') as f:
+        data = [json.loads(line) for line in f]
+
+    df = pd.DataFrame.from_records(data)
+
+    df1 = pd.DataFrame()
+    df1['cost'] = df['metrics'].apply(pd.Series)['accumulated_cost']
+    df1['result'] = (
+        df['test_result'].apply(pd.Series)['exit_code'].map({0: 'Pass', 1: 'Fail'})
+    )
+    df1['actions'] = pd.Series([len(a) - 1 for a in df['history']])
+
+    passed = np.sum(df1['result'] == 'Pass')
+    total = df.shape[0]
+    resolve_rate = round((passed / total) * 100, 2)
+
+    print('Number of passed tests:', f'{passed}/{total}')
+
+    if not visualization_available:
+        return resolve_rate
+
+    # Cost histogram
+    plt.figure(figsize=(10, 6))
+    bins = 10
+    mx = pd.Series.max(df1['cost'])
+    g = sns.histplot(df1, x='cost', bins=bins, hue='result', multiple='stack')
+    x_ticks = np.around(np.linspace(0, mx, bins + 1), 3)
+    g.set_xticks(x_ticks)
+    g.set_xlabel('Cost in $')
+    g.set_title(f'MODEL: {model}, RESOLVE_RATE: {resolve_rate}%', size=9)
+    plt.tight_layout()
+    plt.savefig(os.path.join(output_dir, 'cost_histogram.png'))
+    plt.close()
+
+    # Actions histogram
+    plt.figure(figsize=(10, 6))
+    bins = np.arange(0, 31, 2)
+    g = sns.histplot(df1, x='actions', bins=bins, hue='result', multiple='stack')
+    g.set_xticks(bins)
+    g.set_xlabel('# of actions')
+    g.set_title(f'MODEL: {model}, RESOLVE_RATE: {resolve_rate}%', size=9)
+    plt.tight_layout()
+    plt.savefig(os.path.join(output_dir, 'actions_histogram.png'))
+    plt.close()
+
+    return resolve_rate
+
+
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        print(
-            'Usage: poetry run python summarise_results.py <path_to_output_jsonl_file>'
-        )
+    if len(sys.argv) != 3:
+        print_error('Argument(s) missing!')
         sys.exit(1)
+
     json_file_path = sys.argv[1]
+    model_name = sys.argv[2]
+
+    if not os.path.exists(json_file_path):
+        print_error('Output file does not exist!')
+        sys.exit(1)
+    if not os.path.isfile(json_file_path):
+        print_error('Path-to-output-file is not a file!')
+        sys.exit(1)
+
+    output_dir = os.path.dirname(json_file_path)
+    if not os.access(output_dir, os.W_OK):
+        print_error('Output folder is not writable!')
+        sys.exit(1)
+
     passed_tests, failed_tests = extract_test_results(json_file_path)
-    succ_rate = len(passed_tests) / (len(passed_tests) + len(failed_tests))
+    resolve_rate = visualize_results(json_file_path, model_name, output_dir)
+
     print(
-        f'\nPassed {len(passed_tests)} tests, failed {len(failed_tests)} tests, resolve rate = {succ_rate}'
+        f'\nPassed {len(passed_tests)} tests, failed {len(failed_tests)} tests, resolve rate = {resolve_rate:.2f}%'
     )
     print('PASSED TESTS:')
     print(passed_tests)
     print('FAILED TESTS:')
     print(failed_tests)
+    print(
+        '\nVisualization results were saved as cost_histogram.png and actions_histogram.png'
+    )
+    print('in folder: ', output_dir)

evaluation/biocoder/run_infer.py

@@ -74,7 +74,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: BiocoderData,  # this argument is not required
 ):
@@ -89,19 +89,19 @@ async def initialize_runtime(
 
     action = CmdRunAction(command='mkdir -p /workspace && mkdir -p /testing_files')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     with tempfile.TemporaryDirectory() as tmpdir:
         context_path = os.path.join(tmpdir, 'context.' + file_ext)
         with open(context_path, 'w') as f:
             f.write(instance.contextCode)
-        await runtime.copy_to(context_path, '/testing_files')
+        runtime.copy_to(context_path, '/testing_files')
 
         golden_path = os.path.join(tmpdir, 'golden.' + file_ext)
         with open(golden_path, 'w') as f:
             f.write(instance.goldenCode)
-        await runtime.copy_to(golden_path, '/testing_files')
+        runtime.copy_to(golden_path, '/testing_files')
 
         testcase_json = {
             'test_case_id': instance.test_case_id,
@@ -112,36 +112,36 @@ async def initialize_runtime(
         with open(testcase_path, 'w') as f:
             f.write(json.dumps(testcase_json, indent=4))
 
-        await runtime.copy_to(testcase_path, '/testing_files')
+        runtime.copy_to(testcase_path, '/testing_files')
 
     # setup paths
     remove_code_script = os.path.join(
         os.path.dirname(__file__), 'scripts', 'setup', 'remove_code.py'
     )
-    await runtime.copy_to(remove_code_script, '/testing_files')
+    runtime.copy_to(remove_code_script, '/testing_files')
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     # download repository archive
     repository_url = f"https://biocoder.lilbillbiscuit.com/repos/{instance.repository.split('/')[1]}.zip"
     action = CmdRunAction(command='wget -O repo.zip ' + repository_url)
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0, f'Failed to download the repository: {obs.content}'
 
     # unzip the repository
     action = CmdRunAction(command='unzip -o -q repo.zip && rm repo.zip')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0, f'Failed to unzip the repository: {obs.content}'
 
     # chmod 777
     action = CmdRunAction(command='chmod -R 777 /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0, f'Failed to chmod the files: {obs.content}'
 
     # remove code for evaluation instance
@@ -155,13 +155,13 @@ async def initialize_runtime(
         command=f'python3 /testing_files/remove_code.py --target_filepath {target_filepath} --line_start {line_start} --line_end {line_end} --language {language}'
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0, f'Failed to remove the code: {obs.content}'
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -179,7 +179,7 @@ async def complete_runtime(
     copy_changed_code_script = os.path.join(
         os.path.dirname(__file__), 'scripts', 'setup', 'copy_changed_code.py'
     )
-    await runtime.copy_to(copy_changed_code_script, '/testing_files')
+    runtime.copy_to(copy_changed_code_script, '/testing_files')
 
     file_ext = FILE_EXT_MAP[instance.language.lower()]
     target_filepath = os.path.join(
@@ -191,13 +191,13 @@ async def complete_runtime(
         command=f'python3 /testing_files/copy_changed_code.py --target_filepath {target_filepath} --generated_code_filepath {generated_path} --line_start {instance.lineStart} --include_signature'
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     if obs.exit_code == 0:
         test_result['metadata']['1_copy_change_success'] = True
 
         action = CmdRunAction(command=f'cat {generated_path}', keep_prompt=False)
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         assert obs.exit_code == 0
 
         code = obs.content
@@ -208,14 +208,14 @@ async def complete_runtime(
 
     action = CmdRunAction(command='cd /testing_files')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(
         command='/home/openhands/mambaforge/bin/mamba run -n test python3 /testing/start_test_openhands.py'
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
@@ -223,7 +223,7 @@ async def complete_runtime(
         command='cat /testing_files/results_biocoder.json', keep_prompt=False
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     if obs.exit_code == 0:
         test_result['metadata']['2_run_test_success'] = True
         test_result['metadata']['2_run_test_result'] = str(obs.content)
@@ -237,7 +237,7 @@ async def complete_runtime(
     return test_result
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -277,22 +277,26 @@ async def process_instance(
     # use a session id for concurrent evaluation
     sid = instance.instance_id.replace('/', '__')
 
-    runtime = await create_runtime(config, sid=sid)
+    runtime = create_runtime(config, sid=sid)
 
-    await initialize_runtime(runtime, instance)
+    initialize_runtime(runtime, instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+        )
     )
 
     if state is None:
         raise ValueError('State should not be None.')
 
-    test_result = await complete_runtime(runtime, instance)
+    test_result = complete_runtime(runtime, instance)
     metrics = state.metrics.get() if state.metrics else None
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
@@ -340,8 +344,6 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(biocoder_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/bird/run_infer.py

@@ -242,7 +242,7 @@ def load_bird():
     return bird_dataset
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -261,14 +261,14 @@ async def initialize_runtime(
         instance.db_id,
         f'{instance.db_id}.sqlite',
     )
-    await runtime.copy_to(db_file, '/workspace')
+    runtime.copy_to(db_file, '/workspace')
 
     # Check the database is copied
     action = CmdRunAction(
         command='cd /workspace && ls -l',
         keep_prompt=False,
     )
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
     assert f'{instance.db_id}.sqlite' in obs.content
@@ -276,7 +276,7 @@ async def initialize_runtime(
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -300,7 +300,7 @@ async def complete_runtime(
         command=f'cat {path}',
         keep_prompt=False,
     )
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
 
     if obs.exit_code != 0:
@@ -350,7 +350,7 @@ async def complete_runtime(
     return test_result
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -402,19 +402,23 @@ async def process_instance(
     # NOTE: You can actually set slightly different instruction for different agents
     instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
 
-    runtime = await create_runtime(config, sid=instance_id)
-    await initialize_runtime(runtime, instance)
+    runtime = create_runtime(config, sid=instance_id)
+    initialize_runtime(runtime, instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[metadata.agent_class],
-        runtime=runtime,
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+            runtime=runtime,
+        )
     )
 
     # ======= Attempt to evaluate the agent's edits =======
-    test_result = await complete_runtime(runtime, instance)
+    test_result = complete_runtime(runtime, instance)
 
     # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
     # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
@@ -463,8 +467,6 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/browsing_delegation/run_infer.py

@@ -51,7 +51,7 @@ def get_config(
     return config
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -71,12 +71,14 @@ async def process_instance(
         f'NOTE: You should copy the "query" as is into the <execute_browse> tag. DO NOT change ANYTHING in the query.'
     )
 
-    runtime = await create_runtime(config, sid=instance.instance_id)
+    runtime = create_runtime(config, sid=instance.instance_id)
 
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+        )
     )
 
     if state is None:
@@ -158,12 +160,10 @@ if __name__ == '__main__':
 
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
-    asyncio.run(
-        run_evaluation(
-            instances,
-            metadata,
-            output_file,
-            args.eval_num_workers,
-            process_instance,
-        )
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
     )

evaluation/gaia/run_infer.py

@@ -63,7 +63,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -76,7 +76,7 @@ async def initialize_runtime(
 
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     if instance['file_name'] != '':
@@ -87,7 +87,7 @@ async def initialize_runtime(
         )
         assert os.path.exists(src_file)
         dest_file = os.path.join('/workspace', instance['file_name'])
-        await runtime.copy_to(src_file, dest_file)
+        runtime.copy_to(src_file, dest_file)
 
         # rename to file.extension_name
         extension_name = instance['file_name'].split('.')[-1]
@@ -95,18 +95,18 @@ async def initialize_runtime(
             command=f'mv /workspace/{instance["file_name"]} /workspace/file.{extension_name}'
         )
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -141,15 +141,19 @@ async def process_instance(
     instruction += AGENT_CLS_TO_INST_SUFFIX.get(metadata.agent_class, '')
     logger.info(f'Instruction:\n{instruction}', extra={'msg_type': 'OBSERVATION'})
 
-    runtime = await create_runtime(config, sid=instance['instance_id'])
-    await initialize_runtime(runtime, instance)
+    runtime = create_runtime(config, sid=instance['instance_id'])
+    initialize_runtime(runtime, instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+        )
     )
     # ======= Attempt to evaluate the agent's edits =======
     # If you are working on simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
@@ -257,12 +261,10 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     prepared_dataset = prepare_dataset(gaia_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            dataset=prepared_dataset,
-            metadata=metadata,
-            output_file=output_file,
-            num_workers=args.eval_num_workers,
-            process_instance_func=process_instance,
-        )
+    run_evaluation(
+        dataset=prepared_dataset,
+        metadata=metadata,
+        output_file=output_file,
+        num_workers=args.eval_num_workers,
+        process_instance_func=process_instance,
     )

evaluation/gorilla/run_infer.py

@@ -55,7 +55,7 @@ def get_config(
     return config
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -79,14 +79,16 @@ async def process_instance(
     # logger.info(f'Instruction:\n{instruction}', extra={'msg_type': 'OBSERVATION'})
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    runtime = await create_runtime(config, sid=instance_id)
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-            metadata.agent_class
-        ),
+    runtime = create_runtime(config, sid=instance_id)
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
     )
     # ======= Attempt to evaluate the agent's edits =======
     # If you are working on simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
@@ -179,14 +181,12 @@ if __name__ == '__main__':
     else:
         print('File already exists, skipping download.')
 
-    asyncio.run(
-        run_evaluation(
-            dataset=dataset,
-            metadata=metadata,
-            output_file=output_file,
-            num_workers=args.eval_num_workers,
-            process_instance_func=process_instance,
-        )
+    run_evaluation(
+        dataset=dataset,
+        metadata=metadata,
+        output_file=output_file,
+        num_workers=args.eval_num_workers,
+        process_instance_func=process_instance,
     )
 
     # Read the output file and calculate the accuracy

evaluation/gpqa/run_infer.py

@@ -169,7 +169,7 @@ def convert_instance_dict(instance):
     return out_instance_dict
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -214,15 +214,17 @@ Again do not quit without reporting the answer first.
 Ok now its time to start solving the question. Good luck!
 """
 
-    runtime = await create_runtime(config, sid=f'gptq_{str(instance.instance_id)}')
+    runtime = create_runtime(config, sid=f'gptq_{str(instance.instance_id)}')
 
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-            metadata.agent_class
-        ),
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
     )
     assert state is not None, 'State should not be None.'
 
@@ -355,12 +357,10 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     prepared_dataset = prepare_dataset(gpqa_dataset, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            dataset=prepared_dataset,
-            metadata=metadata,
-            output_file=output_file,
-            num_workers=args.eval_num_workers,
-            process_instance_func=process_instance,
-        )
+    run_evaluation(
+        dataset=prepared_dataset,
+        metadata=metadata,
+        output_file=output_file,
+        num_workers=args.eval_num_workers,
+        process_instance_func=process_instance,
     )

evaluation/humanevalfix/run_infer.py

@@ -102,7 +102,7 @@ def _get_instance_id(instance: pd.Series) -> str:
     return instance.task_id.replace('/', '__')
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -115,12 +115,12 @@ async def initialize_runtime(
 
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     problem_statement = (
@@ -131,20 +131,20 @@ async def initialize_runtime(
         host_script_path = os.path.join(tmpdir, filename)
         with open(host_script_path, 'w') as f:
             f.write(problem_statement)
-        await runtime.copy_to(
+        runtime.copy_to(
             host_script_path,
             '/workspace',
         )
 
     # check file exists
     action = CmdRunAction(command=f'ls /workspace/{_get_instance_id(instance)}.py')
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -170,7 +170,7 @@ async def complete_runtime(
     action = CmdRunAction(
         command=f'cat /workspace/{_get_instance_id(instance)}.py', keep_prompt=False
     )
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     function = obs.content.replace('\r\n', '\n')
@@ -194,7 +194,7 @@ async def complete_runtime(
     return test_result
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -232,21 +232,23 @@ async def process_instance(
     instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    runtime = await create_runtime(config, sid=sid)
-    await initialize_runtime(runtime, instance)
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-            metadata.agent_class
-        ),
+    runtime = create_runtime(config, sid=sid)
+    initialize_runtime(runtime, instance)
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
     )
 
     if state is None:
         raise ValueError('State should not be None.')
     metrics = state.metrics.get() if state.metrics else None
-    test_result = await complete_runtime(runtime, instance)
+    test_result = complete_runtime(runtime, instance)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
@@ -294,12 +296,10 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(hefix_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances,
-            metadata,
-            output_file,
-            args.eval_num_workers,
-            process_instance,
-        )
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
     )

evaluation/logic_reasoning/.cache_program/rules.krb

@@ -49,4 +49,4 @@ fact8
 		facts.Smart(Dave, True)
 		facts.Kind(Dave, True)
 	assert
-		facts.Quiet(Dave, True)
+		facts.Quiet(Dave, True)

evaluation/logic_reasoning/Dockerfile

@@ -2,4 +2,4 @@ FROM python:3.11-bookworm
 
 RUN pip install scitools-pyke
 
-# docker build -t xingyaoww/openhands_logic_reasoning .
+# docker build -t xingyaoww/od_logic_reasoning .

evaluation/logic_reasoning/run_infer.py

@@ -52,7 +52,7 @@ def get_config(
             base_container_image='xingyaoww/od-eval-logic-reasoning:v1.0',
             enable_auto_lint=True,
             use_host_network=False,
-            runtime_extra_deps='$OPENHANDS_INTERPRETER_PATH -m pip install scitools-pyke',
+            runtime_extra_deps='$OH_INTERPRETER_PATH -m pip install scitools-pyke',
         ),
         # do not mount workspace
         workspace_base=None,
@@ -128,7 +128,7 @@ def get_test_result(
 CUR_EVAL_DIR = os.path.dirname(__file__)
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -142,33 +142,31 @@ async def initialize_runtime(
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     # copy logic_inference.py to /workspace
-    await runtime.copy_to(
-        os.path.join(CUR_EVAL_DIR, 'logic_inference.py'), '/workspace'
-    )
+    runtime.copy_to(os.path.join(CUR_EVAL_DIR, 'logic_inference.py'), '/workspace')
     # check if the file exists
-    obs = await runtime.run_action(CmdRunAction(command='ls /workspace'))
+    obs = runtime.run_action(CmdRunAction(command='ls /workspace'))
     assert obs.exit_code == 0
     assert 'logic_inference.py' in obs.content
 
-    await runtime.add_env_vars({'DATASET_NAME': metadata.dataset})
+    runtime.add_env_vars({'DATASET_NAME': metadata.dataset})
 
     action = CmdRunAction(command='mkdir -p /workspace/.cache_program')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = IPythonRunCellAction(code='%pip install scitools-pyke')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    ipynb_obs = await runtime.run_action(action)
+    ipynb_obs = runtime.run_action(action)
     logger.info(ipynb_obs, extra={'msg_type': 'OBSERVATION'})
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
@@ -179,7 +177,7 @@ with open(os.path.join(CUR_EVAL_DIR, 'instruction.txt'), 'r') as f:
     INSTRUCTION_TEMPLATE = f.read()
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -206,8 +204,8 @@ async def process_instance(
     # use a session id for concurrent evaluation
     sid = instance['instance_id']
 
-    runtime = await create_runtime(config, sid=sid)
-    await initialize_runtime(runtime, instance)
+    runtime = create_runtime(config, sid=sid)
+    initialize_runtime(runtime, instance)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
     state: State | None = asyncio.run(
@@ -303,8 +301,6 @@ if __name__ == '__main__':
     )
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(dataset_df, output_file, args.eval_n_limit)
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/miniwob/run_infer.py

@@ -62,7 +62,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
 ) -> str:
     """Initialize the runtime for the agent.
@@ -75,12 +75,12 @@ async def initialize_runtime(
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_GOAL_ACTION)
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     goal = obs.content
 
@@ -88,7 +88,7 @@ async def initialize_runtime(
     return goal
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
 ) -> dict[str, Any]:
     """Complete the runtime for the agent.
@@ -102,7 +102,7 @@ async def complete_runtime(
 
     action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_REWARDS_ACTION)
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
 
     logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
@@ -111,7 +111,7 @@ async def complete_runtime(
     }
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -126,8 +126,8 @@ async def process_instance(
     else:
         logger.info(f'Starting evaluation for instance {env_id}.')
 
-    runtime = await create_runtime(config, sid=env_id)
-    task_str = await initialize_runtime(runtime)
+    runtime = create_runtime(config, sid=env_id)
+    task_str = initialize_runtime(runtime)
 
     state: State | None = asyncio.run(
         run_controller(
@@ -154,7 +154,7 @@ async def process_instance(
             instruction = event.content
             break
 
-    return_val = await complete_runtime(runtime)
+    return_val = complete_runtime(runtime)
     logger.info(f'Return value from complete_runtime: {return_val}')
     reward = max(return_val['rewards'])
 
@@ -208,8 +208,6 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/mint/run_infer.py

@@ -1,3 +1,4 @@
+import asyncio
 import functools
 import os
 from typing import Any
@@ -104,7 +105,7 @@ def get_config(
             base_container_image='xingyaoww/od-eval-mint:v1.0',
             enable_auto_lint=True,
             use_host_network=False,
-            runtime_extra_deps=f'$OPENHANDS_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
+            runtime_extra_deps=f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
         ),
         # do not mount workspace
         workspace_base=None,
@@ -114,7 +115,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(runtime: Runtime):
+def initialize_runtime(runtime: Runtime):
     """Initialize the runtime for the agent.
 
     This function is called before the runtime is used to run the agent.
@@ -125,18 +126,18 @@ async def initialize_runtime(runtime: Runtime):
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='cd /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def process_instance(
+def process_instance(
     instance: Any,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -173,14 +174,16 @@ async def process_instance(
         },
     )
 
-    runtime = await create_runtime(config, sid=instance.instance_id)
-    await initialize_runtime(runtime)
+    runtime = create_runtime(config, sid=instance.instance_id)
+    initialize_runtime(runtime)
 
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=fake_user_response_fn,
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=fake_user_response_fn,
+        )
     )
 
     if state is None:

evaluation/ml_bench/run_infer.py

@@ -13,6 +13,7 @@ TODOs:
 - Clean up the code and docker image used for evaluation.
 """
 
+import asyncio
 import os
 from typing import Any
 
@@ -92,7 +93,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -106,38 +107,38 @@ async def initialize_runtime(
     # Set instance id
     action = CmdRunAction(command='mkdir -p /workspace')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     # Set up the task environment
     action = CmdRunAction(command=f'conda activate {ID2CONDA[instance["github_id"]]}')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     repo_url = instance['github']
     repo_name = repo_url.split('/')[-1]
     action = CmdRunAction(command=f'git clone {repo_url} /workspace/{repo_name}')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     action = CmdRunAction(command=f'chmod -R 777 /workspace/{repo_name}')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     # Navigate to the task's code path
     task_path = os.path.join('/workspace', repo_name, instance['path'][2:])
     action = CmdRunAction(command=f'cd {task_path}')
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     assert obs.exit_code == 0
 
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -160,7 +161,7 @@ async def complete_runtime(
 
     action = CmdRunAction(command=f'cat {eval_script}', keep_prompt=False)
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     if obs.exit_code == 0:
         eval_script_content = obs.content
     else:
@@ -172,7 +173,7 @@ async def complete_runtime(
         timeout=600,
     )
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     if obs.exit_code == 0:
         eval_output = obs.content
     else:
@@ -200,9 +201,7 @@ async def complete_runtime(
     return outputs
 
 
-async def process_instance(
-    instance: Any, metadata: EvalMetadata, reset_logger: bool = True
-):
+def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool = True):
     config = get_config(metadata)
 
     # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
@@ -236,22 +235,24 @@ async def process_instance(
     )
     instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
 
-    runtime = await create_runtime(config, sid=sid)
-    await initialize_runtime(runtime, instance)
+    runtime = create_runtime(config, sid=sid)
+    initialize_runtime(runtime, instance)
 
     # Run the agent
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
-            metadata.agent_class
-        ),
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
     )
     assert state is not None
     metrics = state.metrics.get() if state.metrics else {}
 
-    test_result = await complete_runtime(runtime)
+    test_result = complete_runtime(runtime)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/regression/README.md

@@ -14,9 +14,9 @@ To run the tests for OpenHands project, you can use the provided test runner scr
 3. Navigate to the root directory of the project.
 4. Run the test suite using the test runner script with the required arguments:
    ```
-   python evaluation/regression/run_tests.py --OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxx --model=gpt-3.5-turbo
+   python evaluation/regression/run_tests.py --OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxx --model=gpt-4o
    ```
-   Replace `sk-xxxxxxxxxxxxxxxxxxxxxx` with your actual OpenAI API key. The default model is `gpt-3.5-turbo`, but you can specify a different model if needed.
+   Replace `sk-xxxxxxxxxxxxxxxxxxxxxx` with your actual OpenAI API key. The default model is `gpt-4o`, but you can specify a different model if needed.
 
 The test runner will discover and execute all the test cases in the `cases/` directory, and display the results of the test suite, including the status of each individual test case and the overall summary.
 

evaluation/regression/cases/node-cli-rewrite/start/string_cli.py

@@ -29,21 +29,27 @@ if __name__ == '__main__':
 
     if command == 'reverse':
         from commands.reverse import reverse_string
+
         print(reverse_string(input_string))
     elif command == 'uppercase':
         from commands.uppercase import to_uppercase
+
         print(to_uppercase(input_string))
     elif command == 'lowercase':
         from commands.lowercase import to_lowercase
+
         print(to_lowercase(input_string))
     elif command == 'spongebob':
         from commands.spongebob import spongebob_case
+
         print(spongebob_case(input_string))
     elif command == 'length':
         from commands.length import string_length
+
         print(string_length(input_string))
     elif command == 'scramble':
         from commands.scramble import scramble_string
+
         print(scramble_string(input_string))
     else:
         print('Invalid command!')

evaluation/regression/cases/python-cli-help/start/string_cli.py

@@ -10,21 +10,27 @@ if __name__ == '__main__':
 
     if command == 'reverse':
         from commands.reverse import reverse_string
+
         print(reverse_string(input_string))
     elif command == 'uppercase':
         from commands.uppercase import to_uppercase
+
         print(to_uppercase(input_string))
     elif command == 'lowercase':
         from commands.lowercase import to_lowercase
+
         print(to_lowercase(input_string))
     elif command == 'spongebob':
         from commands.spongebob import spongebob_case
+
         print(spongebob_case(input_string))
     elif command == 'length':
         from commands.length import string_length
+
         print(string_length(input_string))
     elif command == 'scramble':
         from commands.scramble import scramble_string
+
         print(scramble_string(input_string))
     else:
         print('Invalid command!')

evaluation/swe_bench/README.md

@@ -19,27 +19,16 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
 OpenHands now support using the [official evaluation docker](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240627_docker/README.md) for both **[inference](#run-inference-on-swe-bench-instances) and [evaluation](#evaluate-generated-patches)**.
 This is now the default behavior.
 
-### Download Docker Images
-
-**(Recommended for reproducibility)** If you have extra local space (e.g., 100GB), you can try pull the [instance-level docker images](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240627_docker/README.md#choosing-the-right-cache_level) we've prepared by running:
-
-```bash
-evaluation/swe_bench/scripts/docker/pull_all_eval_docker.sh instance
-```
-
-If you want to save disk space a bit (e.g., with ~50GB free disk space), while speeding up the image pre-build process, you can pull the environment-level docker images:
-
-```bash
-evaluation/swe_bench/scripts/docker/pull_all_eval_docker.sh env
-```
 
 ## Run Inference on SWE-Bench Instances
 
-Make sure your Docker daemon is running, and you have pulled the [instance-level docker image](#openhands-swe-bench-instance-level-docker-support).
+Make sure your Docker daemon is running, and you have ample disk space (at least 200-500GB, depends on the SWE-Bench set you are running on) for the [instance-level docker image](#openhands-swe-bench-instance-level-docker-support).
+
+When the `run_infer.sh` script is started, it will automatically pull the relavant SWE-Bench images. For example, for instance ID `django_django-11011`, it will try to pull our pre-build docker image `sweb.eval.x86_64.django_s_django-11011` from DockerHub. This image will be used create an OpenHands runtime image where the agent will operate on.
 
 ```bash
-./evaluation/swe_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers]
-# e.g., ./evaluation/swe_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 300
+./evaluation/swe_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
+# e.g., ./evaluation/swe_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 300 30 1 princeton-nlp/SWE-bench_Lite test
 ```
 
 where `model_config` is mandatory, and the rest are optional.
@@ -57,6 +46,8 @@ in order to use `eval_limit`, you must also set `agent`.
 default, it is set to 30.
 - `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
 default, it is set to 1.
+- `dataset`, a huggingface dataset name. e.g. `princeton-nlp/SWE-bench` or `princeton-nlp/SWE-bench_Lite`, specifies which dataset to evaluate on.
+- `dataset_split`, split for the huggingface dataset. e.g., `test`, `dev`. Default to `test`.
 
 There are also two optional environment variables you can set.
 ```
@@ -72,6 +63,23 @@ then your command would be:
 ./evaluation/swe_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 10
 ```
 
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+# ./evaluation/swe_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
+ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote EVAL_DOCKER_IMAGE_PREFIX="us-docker.pkg.dev/evaluation-428620/swe-bench-images" \
+./evaluation/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 16 "princeton-nlp/SWE-bench_Lite" test
+# This example runs evaluation on CodeActAgent for 300 instances on "princeton-nlp/SWE-bench_Lite"'s test set, with max 30 iteration per instances, with 16 number of workers running in parallel
+```
+
+To clean-up all existing runtime you've already started, run:
+
+```bash
+ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/swe_bench/scripts/cleanup_remote_runtime.sh
+```
+
 ### Specify a subset of tasks to run infer
 
 If you would like to specify a list of tasks you'd like to benchmark on, you could
@@ -89,6 +97,28 @@ After running the inference, you will obtain a `output.jsonl` (by default it wil
 
 ## Evaluate Generated Patches
 
+### Download Docker Images
+
+**(Recommended for reproducibility)** If you have extra local space (e.g., 200GB), you can try pull the [instance-level docker images](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240627_docker/README.md#choosing-the-right-cache_level) we've prepared by running:
+
+```bash
+evaluation/swe_bench/scripts/docker/pull_all_eval_docker.sh instance
+```
+
+If you want to save disk space a bit (e.g., with ~50GB free disk space), while speeding up the image pre-build process, you can pull the environment-level docker images:
+
+```bash
+evaluation/swe_bench/scripts/docker/pull_all_eval_docker.sh env
+```
+
+If you want to evaluate on the full SWE-Bench test set:
+
+```bash
+evaluation/swe_bench/scripts/docker/pull_all_eval_docker.sh instance full
+```
+
+### Run evaluation
+
 With `output.jsonl` file, you can run `eval_infer.sh` to evaluate generated patches, and produce a fine-grained report.
 
 **This evaluation is performed using the official dockerized evaluation announced [here](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240627_docker/README.md).**
@@ -127,6 +157,23 @@ The final results will be saved to `evaluation/evaluation_outputs/outputs/swe_be
 - `report.json`: a JSON file that contains keys like `"resolved_ids"` pointing to instance IDs that are resolved by the agent.
 - `logs/`: a directory of test logs
 
+### Run evaluation with `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+# ./evaluation/swe_bench/scripts/eval_infer_remote.sh [output.jsonl filepath] [num_workers]
+ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote EVAL_DOCKER_IMAGE_PREFIX="us-docker.pkg.dev/evaluation-428620/swe-bench-images" evaluation/swe_bench/scripts/eval_infer_remote.sh evaluation/outputs/swe_bench_lite/CodeActAgent/Llama-3.1-70B-Instruct-Turbo_maxiter_30_N_v1.9-no-hint/output.jsonl 16 "princeton-nlp/SWE-bench_Lite" "test"
+# This example evaluate patches generated by CodeActAgent on Llama-3.1-70B-Instruct-Turbo on "princeton-nlp/SWE-bench_Lite"'s test set, with 16 number of workers running in parallel
+```
+
+To clean-up all existing runtimes that you've already started, run:
+
+```bash
+ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/swe_bench/scripts/cleanup_remote_runtime.sh
+```
+
+
 ## Visualize Results
 
 First you need to clone `https://huggingface.co/spaces/OpenHands/evaluation` and add your own running results from openhands into the `outputs` of the cloned repo.
@@ -149,7 +196,7 @@ Then, in a separate Python environment with `streamlit` library, you can run the
 ```bash
 # Make sure you are inside the cloned `evaluation` repo
 conda activate streamlit # if you follow the optional conda env setup above
-streamlit run 0_📊_OpenHands_Benchmark.py --server.port 8501 --server.address 0.0.0.0
+streamlit app.py --server.port 8501 --server.address 0.0.0.0
 ```
 
 Then you can access the SWE-Bench trajectory visualizer at `localhost:8501`.

evaluation/swe_bench/eval_infer.py

@@ -0,0 +1,349 @@
+import os
+import tempfile
+import time
+
+import pandas as pd
+from pydantic import BaseModel
+from swebench.harness.grading import get_eval_report
+from swebench.harness.run_evaluation import (
+    APPLY_PATCH_FAIL,
+    APPLY_PATCH_PASS,
+)
+from swebench.harness.test_spec import SWEbenchInstance, TestSpec, make_test_spec
+from swebench.harness.utils import load_swebench_dataset
+
+from evaluation.swe_bench.run_infer import get_instance_docker_image
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_parser,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime
+from openhands.events.action import CmdRunAction
+from openhands.events.observation import CmdOutputObservation
+
+# TODO: migrate all swe-bench docker to ghcr.io/openhands
+DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/')
+logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
+
+
+def get_config(instance: pd.Series) -> AppConfig:
+    # We use a different instance image for the each instance of swe-bench eval
+    base_container_image = get_instance_docker_image(instance['instance_id'])
+    logger.info(
+        f'Using instance container image: {base_container_image}. '
+        f'Please make sure this image exists. '
+        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+    )
+    config = AppConfig(
+        run_as_openhands=False,
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
+        sandbox=SandboxConfig(
+            base_container_image=base_container_image,
+            use_host_network=False,
+            # large enough timeout, since some testcases take very long to run
+            timeout=1800,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    return config
+
+
+class SWEBenchEvalResult(BaseModel):
+    instance_id: str
+    apply_patch_output: str
+    test_output: str
+    resolved: bool
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata | None = None,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        global output_file
+        log_dir = output_file.replace('.jsonl', '.logs')
+        os.makedirs(log_dir, exist_ok=True)
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    config = get_config(instance)
+    instance_id = instance.instance_id
+    model_patch = instance['model_patch']
+    test_spec: TestSpec = instance['test_spec']
+    logger.info(f'Starting evaluation for instance {instance_id}.')
+
+    if 'test_result' not in instance.keys():
+        instance['test_result'] = {}
+    instance['test_result']['report'] = {
+        'empty_generation': False,
+        'resolved': False,
+        'failed_apply_patch': False,
+        'error_eval': False,
+    }
+
+    if model_patch == '':
+        instance['test_result']['report']['empty_generation'] = True
+        return EvalOutput(
+            instance_id=instance_id,
+            test_result=instance['test_result'],
+        )
+
+    runtime = create_runtime(config, sid=instance_id)
+
+    # Get patch and save it to /tmp/patch.diff
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Patch file
+        patch_file_path = os.path.join(temp_dir, 'patch.diff')
+        with open(patch_file_path, 'w') as f:
+            f.write(model_patch)
+        runtime.copy_to(patch_file_path, '/tmp')
+        # Eval script
+        eval_script_path = os.path.join(temp_dir, 'eval.sh')
+        with open(eval_script_path, 'w') as f:
+            f.write(test_spec.eval_script)
+        runtime.copy_to(eval_script_path, '/tmp')
+
+    # Set +x
+    action = CmdRunAction(command='chmod +x /tmp/eval.sh')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert obs.exit_code == 0
+
+    # Apply patch
+    exec_command = (
+        'cd /testbed && '
+        "(git apply -v /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || "
+        "(echo 'Failed to apply patch with git apply, trying with patch command...' && "
+        "(patch --batch --fuzz=5 -p1 -i /tmp/patch.diff && echo 'APPLY_PATCH_PASS' || "
+        "echo 'APPLY_PATCH_FAIL')))"
+    )
+    action = CmdRunAction(command=exec_command, keep_prompt=False)
+    action.timeout = 600
+    obs = runtime.run_action(action)
+    assert isinstance(obs, CmdOutputObservation)
+    apply_patch_output = obs.content
+    assert isinstance(apply_patch_output, str)
+    instance['test_result']['apply_patch_output'] = apply_patch_output
+
+    try:
+        if 'APPLY_PATCH_FAIL' in apply_patch_output:
+            logger.info(f'[{instance_id}] {APPLY_PATCH_FAIL}:\n{apply_patch_output}')
+            instance['test_result']['report']['failed_apply_patch'] = True
+
+            return EvalOutput(
+                instance_id=instance_id,
+                test_result=instance['test_result'],
+            )
+        elif 'APPLY_PATCH_PASS' in apply_patch_output:
+            logger.info(f'[{instance_id}] {APPLY_PATCH_PASS}:\n{apply_patch_output}')
+
+            # Run eval script in background and save output to log file
+            log_file = '/tmp/eval_output.log'
+            action = CmdRunAction(
+                command=f'/tmp/eval.sh > {log_file} 2>&1 & echo $!', keep_prompt=False
+            )
+            action.timeout = 60  # Short timeout just to get the process ID
+            obs = runtime.run_action(action)
+
+            if isinstance(obs, CmdOutputObservation) and obs.exit_code == 0:
+                pid = obs.content.split()[-1].strip()
+                logger.info(
+                    f'[{instance_id}] Evaluation process started with PID: {pid}'
+                )
+
+                # Poll for completion
+                start_time = time.time()
+                timeout = 900  # 15 minutes
+                while True:
+                    seconds_elapsed = time.time() - start_time
+                    if seconds_elapsed > timeout:
+                        logger.info(
+                            f'[{instance_id}] Evaluation timed out after {timeout} seconds'
+                        )
+                        break
+                    check_action = CmdRunAction(
+                        command=f'ps -p {pid} > /dev/null; echo $?', keep_prompt=False
+                    )
+                    check_action.timeout = 60
+                    check_obs = runtime.run_action(check_action)
+                    if (
+                        isinstance(check_obs, CmdOutputObservation)
+                        and check_obs.content.split()[-1].strip() == '1'
+                    ):
+                        logger.info(
+                            f'[{instance_id}] Evaluation process completed after {seconds_elapsed} seconds'
+                        )
+                        break
+                    logger.info(
+                        f'[{instance_id}] [{seconds_elapsed:.0f}s] Evaluation still running, waiting...'
+                    )
+                    time.sleep(30)  # Wait for 30 seconds before checking again
+
+                # Read the log file
+                cat_action = CmdRunAction(command=f'cat {log_file}', keep_prompt=False)
+                cat_action.timeout = 300
+                cat_obs = runtime.run_action(cat_action)
+
+                # Grade answer
+                if isinstance(cat_obs, CmdOutputObservation) and cat_obs.exit_code == 0:
+                    test_output = cat_obs.content
+                    assert isinstance(test_output, str)
+                    instance['test_result']['test_output'] = test_output
+
+                    # Get report from test output
+                    logger.info(f'[{instance_id}] Grading answer...')
+                    with tempfile.TemporaryDirectory() as temp_dir:
+                        # Create a directory structure that matches the expected format
+                        # NOTE: this is a hack to make the eval report format consistent
+                        # with the original SWE-Bench eval script
+                        log_dir = os.path.join(temp_dir, 'logs', instance_id)
+                        os.makedirs(log_dir, exist_ok=True)
+                        test_output_path = os.path.join(log_dir, 'test_output.txt')
+                        with open(test_output_path, 'w') as f:
+                            f.write(test_output)
+
+                        _report = get_eval_report(
+                            test_spec=test_spec,
+                            prediction={
+                                'model_patch': model_patch,
+                                'instance_id': instance_id,
+                            },
+                            log_path=test_output_path,
+                            include_tests_status=True,
+                        )
+                        report = _report[instance_id]
+                        logger.info(
+                            f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}"
+                        )
+                        instance['test_result']['report']['resolved'] = report[
+                            'resolved'
+                        ]
+            else:
+                logger.info(f'[{instance_id}] Error when starting eval:\n{obs.content}')
+                instance['test_result']['report']['error_eval'] = True
+
+            return EvalOutput(
+                instance_id=instance_id,
+                test_result=instance['test_result'],
+            )
+        else:
+            logger.info(
+                f'[{instance_id}] Unexpected output when applying patch:\n{apply_patch_output}'
+            )
+            raise RuntimeError(
+                instance_id,
+                f'Unexpected output when applying patch:\n{apply_patch_output}',
+                logger,
+            )
+    finally:
+        runtime.close()
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--input-file',
+        type=str,
+        help='Path to input predictions file',
+        required=True,
+    )
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='princeton-nlp/SWE-bench',
+        help='data set to evaluate on, either full-test or lite-test',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+    args, _ = parser.parse_known_args()
+
+    # Load SWE-Bench dataset
+    full_dataset: list[SWEbenchInstance] = load_swebench_dataset(
+        args.dataset, args.split
+    )
+    instance_id_to_instance = {
+        instance['instance_id']: instance for instance in full_dataset
+    }
+    logger.info(
+        f'Loaded dataset {args.dataset} with split {args.split} to run inference on.'
+    )
+
+    # Load predictions
+    assert args.input_file.endswith('.jsonl'), 'Input file must be a jsonl file.'
+    predictions = pd.read_json(args.input_file, lines=True)
+    assert (
+        'instance_id' in predictions.columns
+    ), 'Input file must contain instance_id column.'
+
+    if 'model_patch' not in predictions.columns and (
+        'test_result' in predictions.columns
+        and 'model_patch' in predictions['test_result'].iloc[0]
+    ):
+        raise ValueError(
+            'Input file must contain model_patch column OR test_result column with model_patch field.'
+        )
+    assert len(predictions['instance_id'].unique()) == len(
+        predictions
+    ), 'instance_id column must be unique.'
+
+    if 'model_patch' not in predictions.columns:
+        predictions['model_patch'] = predictions['test_result'].apply(
+            lambda x: x['git_patch']
+        )
+    assert {'instance_id', 'model_patch'}.issubset(
+        set(predictions.columns)
+    ), 'Input file must contain instance_id and model_patch columns.'
+
+    # Merge predictions with dataset
+    predictions['instance'] = predictions['instance_id'].apply(
+        lambda x: instance_id_to_instance[x]
+    )
+    predictions['test_spec'] = predictions['instance'].apply(make_test_spec)
+
+    # Prepare dataset
+    output_file = args.input_file.replace('.jsonl', '.swebench_eval.jsonl')
+    instances = prepare_dataset(predictions, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata=None,
+        output_file=output_file,
+        num_workers=args.eval_num_workers,
+        process_instance_func=process_instance,
+    )
+
+    # Load evaluated predictions & print number of resolved predictions
+    evaluated_predictions = pd.read_json(output_file, lines=True)
+    fields = ['resolved', 'failed_apply_patch', 'error_eval', 'empty_generation']
+
+    def count_report_field(row, field):
+        return row['test_result']['report'][field]
+
+    for field in fields:
+        count = evaluated_predictions.apply(
+            count_report_field, args=(field,), axis=1
+        ).sum()
+        logger.info(
+            f'# {field}: {count} / {len(evaluated_predictions)}. ({count / len(evaluated_predictions):.2%})'
+        )

evaluation/swe_bench/run_infer.py

@@ -24,13 +24,14 @@ from openhands.core.config import (
     AppConfig,
     SandboxConfig,
     get_llm_config_arg,
-    parse_arguments,
+    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
 from openhands.events.action import CmdRunAction
 from openhands.events.observation import CmdOutputObservation, ErrorObservation
 from openhands.runtime.runtime import Runtime
+from openhands.runtime.utils.shutdown_listener import sleep_if_should_continue
 
 USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
 USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
@@ -86,6 +87,19 @@ def get_instruction(instance: pd.Series, metadata: EvalMetadata):
     return instruction
 
 
+# TODO: migrate all swe-bench docker to ghcr.io/openhands
+DOCKER_IMAGE_PREFIX = os.environ.get('EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/')
+logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
+
+
+def get_instance_docker_image(instance_id: str) -> str:
+    image_name = 'sweb.eval.x86_64.' + instance_id
+    image_name = image_name.replace(
+        '__', '_s_'
+    )  # to comply with docker image naming convention
+    return DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + image_name
+
+
 def get_config(
     instance: pd.Series,
     metadata: EvalMetadata,
@@ -93,22 +107,29 @@ def get_config(
     SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.2.1'
     if USE_INSTANCE_IMAGE:
         # We use a different instance image for the each instance of swe-bench eval
-        base_container_image = 'sweb.eval.x86_64.' + instance['instance_id']
+        base_container_image = get_instance_docker_image(instance['instance_id'])
+        logger.info(
+            f'Using instance container image: {base_container_image}. '
+            f'Please make sure this image exists. '
+            f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+        )
     else:
         base_container_image = SWE_BENCH_CONTAINER_IMAGE
+        logger.info(f'Using swe-bench container image: {base_container_image}')
 
     config = AppConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
-        runtime='eventstream',
         max_budget_per_task=4,
         max_iterations=metadata.max_iterations,
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
         sandbox=SandboxConfig(
             base_container_image=base_container_image,
             enable_auto_lint=True,
             use_host_network=False,
             # large enough timeout, since some testcases take very long to run
             timeout=300,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
         ),
         # do not mount workspace
         workspace_base=None,
@@ -118,7 +139,7 @@ def get_config(
     return config
 
 
-async def initialize_runtime(
+def initialize_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required
 ):
@@ -136,8 +157,16 @@ async def initialize_runtime(
     action = CmdRunAction(
         command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc"""
     )
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """)
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
@@ -147,8 +176,9 @@ async def initialize_runtime(
 
         # inject the instance info
         action = CmdRunAction(command='mkdir -p /swe_util/eval_data/instances')
+        action.timeout = 600
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert (
             obs.exit_code == 0
@@ -166,56 +196,63 @@ async def initialize_runtime(
                     json.dump([instance], f)
 
             # Copy the file to the desired location
-            await runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/')
+            runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/')
 
         # inject the instance swe entry
-        await runtime.copy_to(
+        runtime.copy_to(
             str(os.path.join(script_dir, 'scripts/setup/instance_swe_entry.sh')),
             '/swe_util/',
         )
         action = CmdRunAction(command='cat ~/.bashrc')
+        action.timeout = 600
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert obs.exit_code == 0
 
         action = CmdRunAction(command='source ~/.bashrc')
+        action.timeout = 600
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert obs.exit_code == 0
 
         action = CmdRunAction(command='source /swe_util/instance_swe_entry.sh')
+        action.timeout = 3600
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert obs.exit_code == 0
     else:
         action = CmdRunAction(command='source /swe_util/swe_entry.sh')
+        action.timeout = 1800
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         assert (
             obs.exit_code == 0
         ), f'Failed to source /swe_util/swe_entry.sh: {obs.content}'
 
     action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='git reset --hard')
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
     action = CmdRunAction(
         command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done'
     )
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
@@ -224,7 +261,7 @@ async def initialize_runtime(
     logger.info('-' * 30)
 
 
-async def complete_runtime(
+def complete_runtime(
     runtime: Runtime,
     instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
 ) -> dict[str, Any]:
@@ -241,20 +278,23 @@ async def complete_runtime(
     workspace_dir_name = _get_swebench_workspace_dir_name(instance)
 
     action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='git config --global core.pager ""')
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
     action = CmdRunAction(command='git add -A')
+    action.timeout = 600
     logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = await runtime.run_action(action)
+    obs = runtime.run_action(action)
     logger.info(obs, extra={'msg_type': 'OBSERVATION'})
     assert obs.exit_code == 0
 
@@ -267,7 +307,7 @@ async def complete_runtime(
         )
         action.timeout = 600 + 100 * n_retries
         logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = await runtime.run_action(action)
+        obs = runtime.run_action(action)
         logger.info(obs, extra={'msg_type': 'OBSERVATION'})
         n_retries += 1
         if isinstance(obs, CmdOutputObservation):
@@ -276,10 +316,10 @@ async def complete_runtime(
                 break
             else:
                 logger.info('Failed to get git diff, retrying...')
-                await asyncio.sleep(10)
+                sleep_if_should_continue(10)
         elif isinstance(obs, ErrorObservation):
             logger.error(f'Error occurred: {obs.content}. Retrying...')
-            await asyncio.sleep(10)
+            sleep_if_should_continue(10)
         else:
             raise ValueError(f'Unexpected observation type: {type(obs)}')
 
@@ -289,7 +329,7 @@ async def complete_runtime(
     return {'git_patch': git_patch}
 
 
-async def process_instance(
+def process_instance(
     instance: pd.Series,
     metadata: EvalMetadata,
     reset_logger: bool = True,
@@ -303,26 +343,32 @@ async def process_instance(
     else:
         logger.info(f'Starting evaluation for instance {instance.instance_id}.')
 
-    runtime = await create_runtime(config, sid=instance.instance_id)
-    await initialize_runtime(runtime, instance)
+    runtime = create_runtime(config, sid=instance.instance_id)
+    initialize_runtime(runtime, instance)
 
     instruction = get_instruction(instance, metadata)
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State | None = await run_controller(
-        config=config,
-        task_str=instruction,
-        runtime=runtime,
-        fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[metadata.agent_class],
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            task_str=instruction,
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                metadata.agent_class
+            ],
+        )
     )
 
     # ======= THIS IS SWE-Bench specific =======
     # Get git patch
-    return_val = await complete_runtime(runtime, instance)
+    return_val = complete_runtime(runtime, instance)
     git_patch = return_val['git_patch']
     logger.info(
         f'Got git diff for instance {instance.instance_id}:\n--------\n{git_patch}\n--------'
     )
+
+    runtime.close()
     # ==========================================
 
     # ======= Attempt to evaluate the agent's edits =======
@@ -374,12 +420,26 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
 
 
 if __name__ == '__main__':
-    args = parse_arguments()
+    parser = get_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='princeton-nlp/SWE-bench',
+        help='data set to evaluate on, either full-test or lite-test',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+    args, _ = parser.parse_known_args()
 
     # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
     # so we don't need to manage file uploading to OpenHands's repo
-    dataset = load_dataset('princeton-nlp/SWE-bench_Lite')
-    swe_bench_tests = filter_dataset(dataset['test'].to_pandas(), 'instance_id')
+    dataset = load_dataset(args.dataset, split=args.split)
+    logger.info(f'Loaded dataset {args.dataset} with split {args.split}')
+    swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
 
     llm_config = None
     if args.llm_config:
@@ -408,8 +468,12 @@ if __name__ == '__main__':
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
 
-    asyncio.run(
-        run_evaluation(
-            instances, metadata, output_file, args.eval_num_workers, process_instance
-        )
+    if len(instances) > 0 and not isinstance(
+        instances['PASS_TO_PASS'][instances['PASS_TO_PASS'].index[0]], str
+    ):
+        for col in ['PASS_TO_PASS', 'FAIL_TO_PASS']:
+            instances[col] = instances[col].apply(lambda x: str(x))
+
+    run_evaluation(
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/swe_bench/scripts/cleanup_remote_runtime.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+
+# API base URL
+BASE_URL="https://api.all-hands.dev/v0"
+
+# Get the list of runtimes
+response=$(curl --silent --location --request GET "${BASE_URL}/runtime/list" \
+  --header "X-API-Key: ${ALLHANDS_API_KEY}")
+
+n_runtimes=$(echo $response | jq -r '.total')
+echo "Found ${n_runtimes} runtimes. Stopping them..."
+
+runtime_ids=$(echo $response | jq -r '.runtimes | .[].runtime_id')
+# Loop through each runtime and stop it
+counter=1
+for runtime_id in $runtime_ids; do
+  echo "Stopping runtime ${counter}/${n_runtimes}: ${runtime_id}"
+  curl --silent --location --request POST "${BASE_URL}/runtime/stop" \
+    --header "X-API-Key: ${ALLHANDS_API_KEY}" \
+    --header "Content-Type: application/json" \
+    --data-raw "{\"runtime_id\": \"${runtime_id}\"}"
+  echo
+  ((counter++))
+done
+
+echo "All runtimes have been stopped."