style: Fix linting in test_listen.py

1. Delete send_pull_request endpoint in listen.py
2. Remove file writing dependency in resolve_issue endpoint
3. Rename process_single_issue to create_pull_request_from_resolver_output
4. Update all references and fix tests

.devcontainer/README.MD

@@ -0,0 +1 @@
+The files in this directory configure a development container for GitHub Codespaces.

.devcontainer/devcontainer.json

@@ -0,0 +1,15 @@
+{
+	"name": "OpenHands Codespaces",
+	"image": "mcr.microsoft.com/devcontainers/universal",
+	"customizations":{
+        "vscode":{
+            "extensions": [
+                "ms-python.python"
+            ]
+        }
+    },
+	"onCreateCommand": "sh ./.devcontainer/on_create.sh",
+	"postCreateCommand": "make build",
+	"postStartCommand": "USE_HOST_NETWORK=True nohup bash -c 'make run &'"
+
+}

.devcontainer/on_create.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+sudo apt update
+sudo apt install -y netcat
+sudo add-apt-repository -y ppa:deadsnakes/ppa
+sudo apt install -y python3.12
+curl -sSL https://install.python-poetry.org | python3.12 -

.github/dependabot.yml

@@ -16,9 +16,6 @@ updates:
       chromadb:
         patterns:
           - "chromadb"
-      browsergym:
-        patterns:
-          - "browsergym*"
       security-all:
         applies-to: "security-updates"
         patterns:

.github/workflows/eval-runner.yml

@@ -1,8 +1,10 @@
-name: Run SWE-Bench Evaluation
+name: Run Evaluation
 
 on:
   pull_request:
     types: [labeled]
+  schedule:
+    - cron: "0 1 * * *" # Run daily at 1 AM UTC
   workflow_dispatch:
     inputs:
       reason:
@@ -58,6 +60,24 @@ jobs:
           echo "api_key = \"$DEEPSEEK_API_KEY\"" >> config.toml
           echo "temperature = 0.0" >> config.toml
 
+      - name: Run integration test evaluation
+        env:
+          ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
+          RUNTIME: remote
+          SANDBOX_REMOTE_RUNTIME_API_URL: https://runtime.eval.all-hands.dev
+          EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
+
+        run: |
+          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES
+
+          # get evaluation report
+          REPORT_FILE=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek-chat_maxiter_10_N* -name "report.md" -type f | head -n 1)
+          echo "REPORT_FILE: $REPORT_FILE"
+          echo "INTEGRATION_TEST_REPORT<<EOF" >> $GITHUB_ENV
+          cat $REPORT_FILE >> $GITHUB_ENV
+          echo >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
       - name: Run SWE-Bench evaluation
         env:
           ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
@@ -66,12 +86,12 @@ jobs:
           EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
 
         run: |
-          poetry run ./evaluation/benchmarks/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
+          poetry run ./evaluation/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
           OUTPUT_FOLDER=$(find evaluation/evaluation_outputs/outputs/princeton-nlp__SWE-bench_Lite-test/CodeActAgent -name "deepseek-chat_maxiter_50_N_*-no-hint-run_1" -type d | head -n 1)
           echo "OUTPUT_FOLDER for SWE-bench evaluation: $OUTPUT_FOLDER"
-          poetry run ./evaluation/benchmarks/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
+          poetry run ./evaluation/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
 
-          poetry run ./evaluation/benchmarks/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
+          poetry run ./evaluation/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
           echo "SWEBENCH_REPORT<<EOF" >> $GITHUB_ENV
           cat summarize_outputs.log >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
@@ -125,6 +145,9 @@ jobs:
               **SWE-Bench Evaluation Report**
               ${{ env.SWEBENCH_REPORT }}
               ---
+              **Integration Tests Evaluation Report**
+              ${{ env.INTEGRATION_TEST_REPORT }}
+              ---
               You can download the full evaluation outputs [here](${{ env.ARTIFACT_URL }}).
 
       - name: Post to a Slack channel

.github/workflows/fe-unit-tests.yml

@@ -24,8 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [20, 22]
-      fail-fast: true
+        node-version: [20]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -36,9 +35,6 @@ jobs:
       - name: Install dependencies
         working-directory: ./frontend
         run: npm ci
-      - name: Run TypeScript compilation
-        working-directory: ./frontend
-        run: npm run make-i18n && tsc
       - name: Run tests and collect coverage
         working-directory: ./frontend
         run: npm run test:coverage

.github/workflows/ghcr-build.yml

@@ -291,7 +291,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
@@ -368,7 +368,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:

.github/workflows/integration-runner.yml

@@ -1,158 +0,0 @@
-name: Run Integration Tests
-
-on:
-  pull_request:
-    types: [labeled]
-  workflow_dispatch:
-    inputs:
-      reason:
-        description: 'Reason for manual trigger'
-        required: true
-        default: ''
-  schedule:
-    - cron: '30 22 * * *'  # Runs at 10:30pm UTC every day
-
-env:
-  N_PROCESSES: 10 # Global configuration for number of parallel processes for evaluation
-
-jobs:
-  run-integration-tests:
-    if: github.event.label.name == 'integration-test' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: "read"
-      id-token: "write"
-      pull-requests: "write"
-      issues: "write"
-    strategy:
-      matrix:
-        python-version: ["3.12"]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Install poetry via pipx
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-
-      - name: Comment on PR if 'integration-test' label is present
-        if: github.event_name == 'pull_request' && github.event.label.name == 'integration-test'
-        uses: KeisukeYamashita/create-comment@v1
-        with:
-          unique: false
-          comment: |
-            Hi! I started running the integration tests on your PR. You will receive a comment with the results shortly.
-
-      - name: Install Python dependencies using Poetry
-        run: poetry install --without evaluation,llama-index
-
-      - name: Configure config.toml for testing with Haiku
-        env:
-          LLM_MODEL: "litellm_proxy/claude-3-5-haiku-20241022"
-          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
-        run: |
-          echo "[llm.eval]" > config.toml
-          echo "model = \"$LLM_MODEL\"" >> config.toml
-          echo "api_key = \"$LLM_API_KEY\"" >> config.toml
-          echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
-          echo "temperature = 0.0" >> config.toml
-
-      - name: Build environment
-        run: make build
-
-      - name: Run integration test evaluation for Haiku
-        env:
-          SANDBOX_FORCE_REBUILD_RUNTIME: True
-        run: |
-          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'haiku_run'
-
-          # get integration tests report
-          REPORT_FILE_HAIKU=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/*haiku*_maxiter_10_N* -name "report.md" -type f | head -n 1)
-          echo "REPORT_FILE: $REPORT_FILE_HAIKU"
-          echo "INTEGRATION_TEST_REPORT_HAIKU<<EOF" >> $GITHUB_ENV
-          cat $REPORT_FILE_HAIKU >> $GITHUB_ENV
-          echo >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-
-      - name: Wait a little bit
-        run: sleep 10
-
-      - name: Configure config.toml for testing with DeepSeek
-        env:
-          LLM_MODEL: "litellm_proxy/deepseek-chat"
-          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
-        run: |
-          echo "[llm.eval]" > config.toml
-          echo "model = \"$LLM_MODEL\"" >> config.toml
-          echo "api_key = \"$LLM_API_KEY\"" >> config.toml
-          echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
-          echo "temperature = 0.0" >> config.toml
-
-      - name: Run integration test evaluation for DeepSeek
-        env:
-          SANDBOX_FORCE_REBUILD_RUNTIME: True
-        run: |
-          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'deepseek_run'
-
-          # get integration tests report
-          REPORT_FILE_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek*_maxiter_10_N* -name "report.md" -type f | head -n 1)
-          echo "REPORT_FILE: $REPORT_FILE_DEEPSEEK"
-          echo "INTEGRATION_TEST_REPORT_DEEPSEEK<<EOF" >> $GITHUB_ENV
-          cat $REPORT_FILE_DEEPSEEK >> $GITHUB_ENV
-          echo >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-
-      - name: Create archive of evaluation outputs
-        run: |
-          TIMESTAMP=$(date +'%y-%m-%d-%H-%M')
-          cd evaluation/evaluation_outputs/outputs  # Change to the outputs directory
-          tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/*  # Only include the actual result directories
-
-      - name: Upload evaluation results as artifact
-        uses: actions/upload-artifact@v4
-        id: upload_results_artifact
-        with:
-          name: integration-test-outputs-${{ github.run_id }}-${{ github.run_attempt }}
-          path: integration_tests_*.tar.gz
-
-      - name: Get artifact URLs
-        run: |
-          echo "ARTIFACT_URL=${{ steps.upload_results_artifact.outputs.artifact-url }}" >> $GITHUB_ENV
-
-      - name: Set timestamp and trigger reason
-        run: |
-          echo "TIMESTAMP=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            echo "TRIGGER_REASON=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "TRIGGER_REASON=manual-${{ github.event.inputs.reason }}" >> $GITHUB_ENV
-          else
-            echo "TRIGGER_REASON=nightly-scheduled" >> $GITHUB_ENV
-          fi
-
-      - name: Comment with results and artifact link
-        id: create_comment
-        uses: KeisukeYamashita/create-comment@v1
-        with:
-          # if triggered by PR, use PR number, otherwise use 5318 as fallback issue number for manual triggers
-          number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || 5318 }}
-          unique: false
-          comment: |
-              Trigger by: ${{ github.event_name == 'pull_request' && format('Pull Request (integration-test label on PR #{0})', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && format('Manual Trigger: {0}', github.event.inputs.reason)) || 'Nightly Scheduled Run' }}
-              Commit: ${{ github.sha }}
-              **Integration Tests Report (Haiku)**
-              Haiku LLM Test Results:
-              ${{ env.INTEGRATION_TEST_REPORT_HAIKU }}
-              ---
-              **Integration Tests Report (DeepSeek)**
-              DeepSeek LLM Test Results:
-              ${{ env.INTEGRATION_TEST_REPORT_DEEPSEEK }}
-              ---
-              Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }})

.github/workflows/lint-fix.yml

@@ -5,10 +5,9 @@ on:
     types: [labeled]
 
 jobs:
-  # Frontend lint fixes
-  lint-fix-frontend:
+  lint-fix:
     if: github.event.label.name == 'lint-fix'
-    name: Fix frontend linting issues
+    name: Fix linting issues
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -21,6 +20,7 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      # Frontend lint fixes
       - name: Install Node.js 20
         uses: actions/setup-node@v4
         with:
@@ -34,36 +34,7 @@ jobs:
           cd frontend
           npm run lint:fix
 
-      # Commit and push changes if any
-      - name: Check for changes
-        id: git-check
-        run: |
-          git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT
-      - name: Commit and push if there are changes
-        if: steps.git-check.outputs.changes == 'true'
-        run: |
-          git config --local user.email "openhands@all-hands.dev"
-          git config --local user.name "OpenHands Bot"
-          git add -A
-          git commit -m "🤖 Auto-fix frontend linting issues"
-          git push
-
-  # Python lint fixes
-  lint-fix-python:
-    if: github.event.label.name == 'lint-fix'
-    name: Fix Python linting issues
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-
+      # Python lint fixes
       - name: Set up python
         uses: actions/setup-python@v5
         with:
@@ -73,8 +44,7 @@ jobs:
         run: pip install pre-commit==3.7.0
       - name: Fix python lint issues
         run: |
-          # Run all pre-commit hooks and continue even if they modify files (exit code 1)
-          pre-commit run --config ./dev_config/python/.pre-commit-config.yaml --files openhands/**/* evaluation/**/* tests/**/* || true
+          pre-commit run --files openhands/**/* evaluation/**/* tests/**/* --config ./dev_config/python/.pre-commit-config.yaml
 
       # Commit and push changes if any
       - name: Check for changes
@@ -87,5 +57,5 @@ jobs:
           git config --local user.email "openhands@all-hands.dev"
           git config --local user.name "OpenHands Bot"
           git add -A
-          git commit -m "🤖 Auto-fix Python linting issues"
+          git commit -m "🤖 Auto-fix linting issues"
           git push

.github/workflows/lint.yml

@@ -30,11 +30,10 @@ jobs:
         run: |
           cd frontend
           npm install --frozen-lockfile
-      - name: Lint and TypeScript compilation
+      - name: Lint
         run: |
           cd frontend
           npm run lint
-          npm run make-i18n && tsc
 
   # Run lint on the python code
   lint-python:

.github/workflows/openhands-resolver.yml

@@ -11,31 +11,17 @@ on:
         required: false
         type: string
         default: "@openhands-agent"
-      target_branch:
-        required: false
-        type: string
-        default: "main"
-        description: "Target branch to pull and create PR against"
-      LLM_MODEL:
-        required: false
-        type: string
-        default: "anthropic/claude-3-5-sonnet-20241022"
-      base_container_image:
-        required: false
-        type: string
-        default: ""
-        description: "Custom sandbox env"
     secrets:
       LLM_MODEL:
-        required: false
+        required: true
       LLM_API_KEY:
         required: true
       LLM_BASE_URL:
         required: false
       PAT_TOKEN:
-        required: false
+        required: true
       PAT_USERNAME:
-        required: false
+        required: true
 
   issues:
     types: [labeled]
@@ -54,19 +40,21 @@ permissions:
   issues: write
 
 jobs:
+
   auto-fix:
     if: |
       github.event_name == 'workflow_call' ||
       github.event.label.name == 'fix-me' ||
       github.event.label.name == 'fix-me-experimental' ||
+
       (
         ((github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
-        contains(github.event.comment.body, inputs.macro || '@openhands-agent') &&
+        startsWith(github.event.comment.body, inputs.macro || '@openhands-agent') &&
         (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER')
         ) ||
 
         (github.event_name == 'pull_request_review' &&
-        contains(github.event.review.body, inputs.macro || '@openhands-agent') &&
+        startsWith(github.event.review.body, inputs.macro || '@openhands-agent') &&
         (github.event.review.author_association == 'OWNER' || github.event.review.author_association == 'COLLABORATOR' || github.event.review.author_association == 'MEMBER')
         )
       )
@@ -88,18 +76,7 @@ jobs:
           cat requirements.txt
 
       - name: Cache pip dependencies
-        if: |
-          !(
-            github.event.label.name == 'fix-me-experimental' ||
-            (
-              (github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
-              contains(github.event.comment.body, '@openhands-agent-exp')
-            ) ||
-            (
-              github.event_name == 'pull_request_review' &&
-              contains(github.event.review.body, '@openhands-agent-exp')
-            )
-          )
+        if: github.event.label.name != 'fix-me-experimental'
         uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/*
@@ -109,14 +86,13 @@ jobs:
 
       - name: Check required environment variables
         env:
-          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
           LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
           PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
           PAT_USERNAME: ${{ secrets.PAT_USERNAME }}
-          GITHUB_TOKEN: ${{ github.token }}
         run: |
-          required_vars=("LLM_API_KEY")
+          required_vars=("LLM_MODEL" "LLM_API_KEY" "PAT_TOKEN" "PAT_USERNAME")
           for var in "${required_vars[@]}"; do
             if [ -z "${!var}" ]; then
               echo "Error: Required environment variable $var is not set."
@@ -124,34 +100,17 @@ jobs:
             fi
           done
 
-          # Check optional variables and warn about fallbacks
-          if [ -z "$LLM_BASE_URL" ]; then
-            echo "Warning: LLM_BASE_URL is not set, will use default API endpoint"
-          fi
-
-          if [ -z "$PAT_TOKEN" ]; then
-            echo "Warning: PAT_TOKEN is not set, falling back to GITHUB_TOKEN"
-          fi
-
-          if [ -z "$PAT_USERNAME" ]; then
-            echo "Warning: PAT_USERNAME is not set, will use openhands-agent"
-          fi
-
       - name: Set environment variables
         run: |
-          # Handle pull request events first
-          if [ -n "${{ github.event.pull_request.number }}" ]; then
+          if [ -n "${{ github.event.review.body }}" ]; then
             echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
             echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          # Handle pull request review events
-          elif [ -n "${{ github.event.review.body }}" ]; then
-            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          # Handle issue comment events that reference a PR
           elif [ -n "${{ github.event.issue.pull_request }}" ]; then
             echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
             echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          # Handle regular issue events
+          elif [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
           else
             echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
             echo "ISSUE_TYPE=issue" >> $GITHUB_ENV
@@ -164,16 +123,12 @@ jobs:
           fi
 
           echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV
-          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.PAT_TOKEN || github.token }}" >> $GITHUB_ENV
-          echo "SANDBOX_ENV_BASE_CONTAINER_IMAGE=${{ inputs.base_container_image }}" >> $GITHUB_ENV
-
-          # Set branch variables
-          echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
 
       - name: Comment on issue with start message
         uses: actions/github-script@v7
         with:
-          github-token: ${{ secrets.PAT_TOKEN || github.token }}
+          github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
             const issueType = process.env.ISSUE_TYPE;
             github.rest.issues.createComment({
@@ -184,38 +139,19 @@ jobs:
             });
 
       - name: Install OpenHands
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const commentBody = `${{ github.event.comment.body || '' }}`.trim();
-            const reviewBody = `${{ github.event.review.body || '' }}`.trim();
-            const labelName = `${{ github.event.label.name || '' }}`.trim();
-            const eventName = `${{ github.event_name }}`.trim();
-
-            // Check conditions
-            const isExperimentalLabel = labelName === "fix-me-experimental";
-            const isIssueCommentExperimental =
-              (eventName === "issue_comment" || eventName === "pull_request_review_comment") &&
-              commentBody.includes("@openhands-agent-exp");
-            const isReviewCommentExperimental =
-              eventName === "pull_request_review" && reviewBody.includes("@openhands-agent-exp");
-
-            // Perform package installation
-            if (isExperimentalLabel || isIssueCommentExperimental || isReviewCommentExperimental) {
-              console.log("Installing experimental OpenHands...");
-              await exec.exec("python -m pip install --upgrade pip");
-              await exec.exec("pip install git+https://github.com/all-hands-ai/openhands.git");
-            } else {
-              console.log("Installing from requirements.txt...");
-              await exec.exec("python -m pip install --upgrade pip");
-              await exec.exec("pip install -r requirements.txt");
-            }
+        run: |
+          if [ "${{ github.event.label.name }}" == "fix-me-experimental" ]; then
+            python -m pip install --upgrade pip
+            pip install git+https://github.com/all-hands-ai/openhands.git
+          else
+            python -m pip install --upgrade -r requirements.txt
+          fi
 
       - name: Attempt to resolve issue
         env:
-          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }}
-          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }}
-          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
           LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
           PYTHONPATH: ""
@@ -247,9 +183,9 @@ jobs:
       - name: Create draft PR or push branch
         if: always() # Create PR or branch even if the previous steps fail
         env:
-          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }}
-          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }}
-          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
           LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
           PYTHONPATH: ""
@@ -257,8 +193,7 @@ jobs:
           if [ "${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}" == "true" ]; then
             cd /tmp && python -m openhands.resolver.send_pull_request \
               --issue-number ${{ env.ISSUE_NUMBER }} \
-              --pr-type draft \
-              --reviewer ${{ github.actor }} | tee pr_result.txt && \
+              --pr-type draft | tee pr_result.txt && \
               grep "draft created" pr_result.txt | sed 's/.*\///g' > pr_number.txt
           else
             cd /tmp && python -m openhands.resolver.send_pull_request \
@@ -268,58 +203,30 @@ jobs:
               grep "branch created" branch_result.txt | sed 's/.*\///g; s/.expand=1//g' > branch_name.txt
           fi
 
-      # Step leaves comment for when agent is invoked on PR
-      - name: Analyze Push Logs (Updated PR or No Changes) # Skip comment if PR update was successful OR leave comment if the agent made no code changes
-        uses: actions/github-script@v7
-        if: always()
-        env:
-          AGENT_RESPONDED: ${{ env.AGENT_RESPONDED || 'false' }}
-        with:
-          github-token: ${{ secrets.PAT_TOKEN || github.token }}
-          script: |
-            const fs = require('fs');
-            const issueNumber = ${{ env.ISSUE_NUMBER }};
-            let logContent = '';
-
-            try {
-              logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim();
-            } catch (error) {
-              console.error('Error reading pr_result.txt file:', error);
-            }
-
-            const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`;
-
-            // Check logs from send_pull_request.py (pushes code to GitHub)
-            if (logContent.includes("Updated pull request")) {
-              console.log("Updated pull request found. Skipping comment.");
-              process.env.AGENT_RESPONDED = 'true';
-            } else if (logContent.includes(noChangesMessage)) {
-              github.rest.issues.createComment({
-                issue_number: issueNumber,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.`
-              });
-              process.env.AGENT_RESPONDED = 'true';
-            }
-
-      # Step leaves comment for when agent is invoked on issue
-      - name: Comment on issue # Comment link to either PR or branch created by agent
+      - name: Comment on issue
         uses: actions/github-script@v7
         if: always() # Comment on issue even if the previous steps fail
-        env:
-          AGENT_RESPONDED: ${{ env.AGENT_RESPONDED || 'false' }}
         with:
-          github-token: ${{ secrets.PAT_TOKEN || github.token }}
+          github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
             const fs = require('fs');
-            const path = require('path');
             const issueNumber = ${{ env.ISSUE_NUMBER }};
             const success = ${{ steps.check_result.outputs.RESOLUTION_SUCCESS }};
 
             let prNumber = '';
             let branchName = '';
-            let resultExplanation = '';
+            let logContent = '';
+            const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`;
+
+            try {
+              if (success){
+                logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim();
+              } else {
+                logContent = fs.readFileSync('/tmp/branch_result.txt', 'utf8').trim();
+              }
+            } catch (error) {
+              console.error('Error reading results file:', error);
+            }
 
             try {
               if (success) {
@@ -331,63 +238,32 @@ jobs:
               console.error('Error reading file:', error);
             }
 
-
-            try {
-              if (!success){
-                // Read result_explanation from JSON file for failed resolution
-                const outputFilePath = path.resolve('/tmp/output/output.jsonl');
-                if (fs.existsSync(outputFilePath)) {
-                  const outputContent = fs.readFileSync(outputFilePath, 'utf8');
-                  const jsonLines = outputContent.split('\n').filter(line => line.trim() !== '');
-
-                  if (jsonLines.length > 0) {
-                    // First entry in JSON lines has the key 'result_explanation'
-                    const firstEntry = JSON.parse(jsonLines[0]);
-                    resultExplanation = firstEntry.result_explanation || '';
-                  }
-                }
-              }
-            } catch (error){
-              console.error('Error reading file:', error);
-            }
-
-            // Check "success" log from resolver output
-            if (success && prNumber) {
+            if (logContent.includes(noChangesMessage)) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.`
+              });
+            } else if (success && prNumber) {
               github.rest.issues.createComment({
                 issue_number: issueNumber,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 body: `A potential fix has been generated and a draft PR #${prNumber} has been created. Please review the changes.`
               });
-              process.env.AGENT_RESPONDED = 'true';
             } else if (!success && branchName) {
-              let commentBody = `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`;
-
-              if (resultExplanation) {
-                commentBody += `\n\nAdditional details about the failure:\n${resultExplanation}`;
-              }
-
               github.rest.issues.createComment({
                 issue_number: issueNumber,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                body: commentBody
+                body: `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`
+              });
+            } else {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.`
               });
-              process.env.AGENT_RESPONDED = 'true';
             }
-
-      # Leave error comment when both PR/Issue comment handling fail
-      - name: Fallback Error Comment
-        uses: actions/github-script@v7
-        if: ${{ env.AGENT_RESPONDED == 'false' }} # Only run if no conditions were met in previous steps
-        with:
-          github-token: ${{ secrets.PAT_TOKEN || github.token }}
-          script: |
-            const issueNumber = ${{ env.ISSUE_NUMBER }};
-
-            github.rest.issues.createComment({
-              issue_number: issueNumber,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.`
-            });

.github/workflows/py-unit-tests.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Tests
-        run: poetry run pytest --forked -n auto --cov=openhands --cov-report=xml -svv ./tests/unit --ignore=tests/unit/test_memory.py
+        run: poetry run pytest --forked --cov=openhands --cov-report=xml -svv ./tests/unit --ignore=tests/unit/test_memory.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:

.github/workflows/review-pr.yml

@@ -0,0 +1,81 @@
+# Workflow that uses OpenHands to review a pull request. PR must be labeled 'review-this'
+name: Use OpenHands to Review Pull Request
+
+on:
+  pull_request:
+    types: [synchronize, labeled]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  dogfood:
+    if: contains(github.event.pull_request.labels.*.name, 'review-this')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: install git, github cli
+      run: |
+        sudo apt-get install -y git gh
+        git config --global --add safe.directory $PWD
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.base.ref }} # check out the target branch
+    - name: Download Diff
+      run: |
+        curl -O "${{ github.event.pull_request.diff_url }}" -L
+    - name: Write Task File
+      run: |
+        echo "Your coworker wants to apply a pull request to this project." > task.txt
+        echo "Read and review ${{ github.event.pull_request.number }}.diff file. Create a review-${{ github.event.pull_request.number }}.txt and write your concise comments and suggestions there." >> task.txt
+        echo "Do not ask me for confirmation at any point." >> task.txt
+        echo "" >> task.txt
+        echo "Title" >> task.txt
+        echo "${{ github.event.pull_request.title }}" >> task.txt
+        echo "" >> task.txt
+        echo "Description" >> task.txt
+        echo "${{ github.event.pull_request.body }}" >> task.txt
+        echo "" >> task.txt
+        echo "Diff file is: ${{ github.event.pull_request.number }}.diff" >> task.txt
+    - name: Set up environment
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        export PATH="/github/home/.local/bin:$PATH"
+        poetry install --without evaluation,llama-index
+        poetry run playwright install --with-deps chromium
+    - name: Run OpenHands
+      env:
+        LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+        LLM_MODEL: ${{ vars.LLM_MODEL }}
+      run: |
+        # Append path to launch poetry
+        export PATH="/github/home/.local/bin:$PATH"
+        # Append path to correctly import package, note: must set pwd at first
+        export PYTHONPATH=$(pwd):$PYTHONPATH
+        export WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE
+        export WORKSPACE_BASE=$GITHUB_WORKSPACE
+        echo -e "/exit\n" | poetry run python openhands/core/main.py -i 50 -f task.txt
+        rm task.txt
+    - name: Check if review file is non-empty
+      id: check_file
+      run: |
+        ls -la
+        if [[ -s review-${{ github.event.pull_request.number }}.txt ]]; then
+          echo "non_empty=true" >> $GITHUB_OUTPUT
+        fi
+      shell: bash
+    - name: Create PR review if file is non-empty
+      env:
+        GH_TOKEN: ${{ github.token }}
+      if: steps.check_file.outputs.non_empty == 'true'
+      run: |
+        gh pr review ${{ github.event.pull_request.number }} --comment --body-file "review-${{ github.event.pull_request.number }}.txt"

.github/workflows/run-eval.yml

@@ -1,53 +0,0 @@
-# Run evaluation on a PR
-name: Run Eval
-
-# Runs when a PR is labeled with one of the "run-eval-" labels
-on:
-  pull_request:
-    types: [labeled]
-
-jobs:
-  trigger-job:
-    name: Trigger remote eval job
-    if: ${{ github.event.label.name == 'run-eval-xs' || github.event.label.name == 'run-eval-s' || github.event.label.name == 'run-eval-m' }}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout PR branch
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-
-      - name: Trigger remote job
-        run: |
-          REPO_URL="https://github.com/${{ github.repository }}"
-          PR_BRANCH="${{ github.head_ref }}"
-          echo "Repository URL: $REPO_URL"
-          echo "PR Branch: $PR_BRANCH"
-
-          if [[ "${{ github.event.label.name }}" == "run-eval-xs" ]]; then
-            EVAL_INSTANCES="1"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-s" ]]; then
-            EVAL_INSTANCES="5"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-m" ]]; then
-            EVAL_INSTANCES="30"
-          fi
-
-          curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
-            -H "Accept: application/vnd.github+json" \
-            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
-            https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
-
-          # Send Slack message
-          PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
-          slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
-          curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
-            https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
-
-      - name: Comment on PR
-        uses: KeisukeYamashita/create-comment@v1
-        with:
-          unique: false
-          comment: |
-            Running evaluation on the PR. Once eval is done, the results will be posted.

.gitignore

@@ -175,7 +175,6 @@ evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
 evaluation/scienceagentbench/benchmark
-evaluation/commit0_bench/repos
 
 # openhands resolver
 output/

.nvmrc

@@ -1 +0,0 @@
-22

COMMUNITY.md

@@ -1,43 +0,0 @@
-# 🙌 The OpenHands Community
-
-The OpenHands community is built around the belief that (1) AI and AI agents are going to fundamentally change the way
-we build software, and (2) if this is true, we should do everything we can to make sure that the benefits provided by
-such powerful technology are accessible to everyone.
-
-If this resonates with you, we'd love to have you join us in our quest!
-
-## 🤝 How to Join
-
-Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
-
-## 💪 Becoming a Contributor
-
-We welcome contributions from everyone! Whether you're a developer, a researcher, or simply enthusiastic about advancing
-the field of software engineering with AI, there are many ways to get involved:
-
-- **Code Contributions:** Help us develop new core functionality, improve our agents, improve the frontend and other
-interfaces, or anything else that would help make OpenHands better.
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in
-evaluating the models, or suggest improvements.
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
-
-For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
-
-## Code of Conduct
-
-We have a [Code of Conduct](./CODE_OF_CONDUCT.md) that we expect all contributors to adhere to.
-Long story short, we are aiming for an open, welcoming, diverse, inclusive, and healthy community.
-All contributors are expected to contribute to building this sort of community.
-
-## 🛠️ Becoming a Maintainer
-
-For contributors who have made significant and sustained contributions to the project, there is a possibility of joining
-the maintainer team. The process for this is as follows:
-
-1. Any contributor who has made sustained and high-quality contributions to the codebase can be nominated by any
-maintainer. If you feel that you may qualify you can reach out to any of the maintainers that have reviewed your PRs and ask if you can be nominated.
-2. Once a maintainer nominates a new maintainer, there will be a discussion period among the maintainers for at least 3 days.
-3. If no concerns are raised the nomination will be accepted by acclamation, and if concerns are raised there will be a discussion and possible vote.
-
-Note that just making many PRs does not immediately imply that you will become a maintainer. We will be looking
-at sustained high-quality contributions over a period of time, as well as good teamwork and adherence to our [Code of Conduct](./CODE_OF_CONDUCT.md).

CONTRIBUTING.md

@@ -21,14 +21,14 @@ There are many ways that you can contribute:
 
 1. **Download and use** OpenHands, and send [issues](https://github.com/All-Hands-AI/OpenHands/issues) when you encounter something that isn't working or a feature that you'd like to see.
 2. **Send feedback** after each session by [clicking the thumbs-up thumbs-down buttons](https://docs.all-hands.dev/modules/usage/feedback), so we can see where things are working and failing, and also build an open dataset for training code agents.
-3. **Improve the Codebase** by sending [PRs](#sending-pull-requests-to-openhands) (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
+3. **Improve the Codebase** by sending PRs (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
 
 ## What can I build?
 Here are a few ways you can help improve the codebase.
 
 #### UI/UX
 We're always looking to improve the look and feel of the application. If you've got a small fix
-for something that's bugging you, feel free to open up a PR that changes the [`./frontend`](./frontend) directory.
+for something that's bugging you, feel free to open up a PR that changes the `./frontend` directory.
 
 If you're looking to make a bigger change, add a new UI element, or significantly alter the style
 of the application, please open an issue first, or better, join the #frontend channel in our Slack
@@ -46,7 +46,7 @@ We use the [SWE-bench](https://www.swebench.com/) benchmark to test our agent. Y
 channel in Slack to learn more.
 
 #### Adding a new agent
-You may want to experiment with building new types of agents. You can add an agent to [`openhands/agenthub`](./openhands/agenthub)
+You may want to experiment with building new types of agents. You can add an agent to `openhands/agenthub`
 to help expand the capabilities of OpenHands.
 
 #### Adding a new runtime
@@ -54,11 +54,11 @@ The agent needs a place to run code and commands. When you run OpenHands on your
 to do this by default. But there are other ways of creating a sandbox for the agent.
 
 If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
-by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
+by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/runtime.py).
 
 #### Testing
-When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites.
-At the moment, we have two kinds of tests: [`unit`](./tests/unit) and [`integration`](./evaluation/integration_tests). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
+When you write code, it is also good to write tests. Please navigate to the `tests` folder to see existing test suites.
+At the moment, we have two kinds of tests: `unit` and `integration`. Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
 
 ## Sending Pull Requests to OpenHands
 
@@ -103,7 +103,7 @@ Further, if you see an issue you like, please leave a "thumbs-up" or a comment,
 
 ### Making Pull Requests
 
-We're generally happy to consider all [PRs](https://github.com/All-Hands-AI/OpenHands/pulls), with the evaluation process varying based on the type of change:
+We're generally happy to consider all PRs, with the evaluation process varying based on the type of change:
 
 #### For Small Improvements
 

Development.md

@@ -100,7 +100,7 @@ poetry run pytest ./tests/unit/test_*.py
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by
 setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
 
-Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.16-nikolaik`
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.14-nikolaik`
 
 ## Develop inside Docker container
 

README.md

@@ -12,7 +12,7 @@
   <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
   <br/>
-  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
+  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
   <br/>
@@ -29,11 +29,6 @@ call APIs, and yes—even copy code snippets from StackOverflow.
 
 Learn more at [docs.all-hands.dev](https://docs.all-hands.dev), or jump to the [Quick Start](#-quick-start).
 
-> [!IMPORTANT]
-> Using OpenHands for work? We'd love to chat! Fill out
-> [this short form](https://docs.google.com/forms/d/e/1FAIpQLSet3VbGaz8z32gW9Wm-Grl4jpt5WgMXPgJ4EDPVmCETCBpJtQ/viewform)
-> to join our Design Partner program, where you'll get early access to commercial features and the opportunity to provide input on our product roadmap.
-
 ![App screenshot](./docs/static/img/screenshot.png)
 
 ## ⚡ Quick Start
@@ -43,17 +38,16 @@ See the [Installation](https://docs.all-hands.dev/modules/usage/installation) gu
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
 
-docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik \
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
-    -v ~/.openhands:/home/openhands/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.16
+    docker.all-hands.dev/all-hands-ai/openhands:0.14
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
@@ -71,14 +65,6 @@ or run it on tagged issues with [a github action](https://github.com/All-Hands-A
 
 Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
-> [!CAUTION]
-> OpenHands is meant to be run by a single user on their local workstation.
-> It is not appropriate for multi-tenant deployments, where multiple users share the same instance--there is no built-in isolation or scalability.
->
-> If you're interested in running OpenHands in a multi-tenant environment, please
-> [get in touch with us](https://docs.google.com/forms/d/e/1FAIpQLSet3VbGaz8z32gW9Wm-Grl4jpt5WgMXPgJ4EDPVmCETCBpJtQ/viewform)
-> for advanced deployment options.
-
 If you want to modify the OpenHands source code, check out [Development.md](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md).
 
 Having issues? The [Troubleshooting Guide](https://docs.all-hands.dev/modules/usage/troubleshooting) can help.
@@ -91,21 +77,28 @@ To learn more about the project, and for tips on using OpenHands,
 There you'll find resources on how to use different LLM providers,
 troubleshooting resources, and advanced configuration options.
 
-## 🤝 How to Join the Community
+## 🤝 How to Contribute
 
-OpenHands is a community-driven project, and we welcome contributions from everyone. We do most of our communication
-through Slack, so this is the best place to start, but we also are happy to have you contact us on Discord or Github:
+OpenHands is a community-driven project, and we welcome contributions from everyone.
+Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of
+software engineering with AI, there are many ways to get involved:
 
-- [Join our Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw) - Here we talk about research, architecture, and future development.
-- [Join our Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
-- [Read or post Github Issues](https://github.com/All-Hands-AI/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
+- **Code Contributions:** Help us develop new agents, core functionality, the frontend and other interfaces, or sandboxing solutions.
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements.
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
 
-See more about the community in [COMMUNITY.md](./COMMUNITY.md) or find details on contributing in [CONTRIBUTING.md](./CONTRIBUTING.md).
+For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
+
+## 🤖 Join Our Community
+
+Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
+Let's make software engineering better together!
+
+- [Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
+- [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
 
 ## 📈 Progress
 
-See the monthly OpenHands roadmap [here](https://github.com/orgs/All-Hands-AI/projects/1) (updated at the maintainer's meeting at the end of each month).
-
 <p align="center">
   <a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
     <img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">

compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.16-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

config.template.toml

@@ -95,10 +95,10 @@ workspace_base = "./workspace"
 # AWS secret access key
 #aws_secret_access_key = ""
 
-# API key to use (For Headless / CLI only -  In Web this is overridden by Session Init)
+# API key to use
 api_key = "your-api-key"
 
-# API base URL (For Headless / CLI only -  In Web this is overridden by Session Init)
+# API base URL
 #base_url = ""
 
 # API version
@@ -131,7 +131,7 @@ embedding_model = "local"
 # Maximum number of output tokens
 #max_output_tokens = 0
 
-# Model to use. (For Headless / CLI only -  In Web this is overridden by Session Init)
+# Model to use
 model = "gpt-4o"
 
 # Number of retries to attempt when an operation fails with the LLM.
@@ -154,10 +154,6 @@ model = "gpt-4o"
 # Drop any unmapped (unsupported) params without causing an exception
 #drop_params = false
 
-# Modify params for litellm to do transformations like adding a default message, when a message is empty.
-# Note: this setting is global, unlike drop_params, it cannot be overridden in each call to litellm.
-#modify_params = true
-
 # Using the prompt caching feature if provided by the LLM and supported
 #caching_prompt = true
 
@@ -176,10 +172,6 @@ model = "gpt-4o"
 # If model is vision capable, this option allows to disable image processing (useful for cost reduction).
 #disable_vision = true
 
-# Custom tokenizer to use for token counting
-# https://docs.litellm.ai/docs/completion/token_usage
-#custom_tokenizer = ""
-
 [llm.gpt4o-mini]
 api_key = "your-api-key"
 model = "gpt-4o"
@@ -225,9 +217,6 @@ llm_config = 'gpt3'
 # Use host network
 #use_host_network = false
 
-# runtime extra build args
-#runtime_extra_build_args = ["--network=host", "--add-host=host.docker.internal:host-gateway"]
-
 # Enable auto linting after editing
 #enable_auto_lint = false
 
@@ -248,10 +237,10 @@ llm_config = 'gpt3'
 ##############################################################################
 [security]
 
-# Enable confirmation mode (For Headless / CLI only -  In Web this is overridden by Session Init)
+# Enable confirmation mode
 #confirmation_mode = false
 
-# The security analyzer to use (For Headless / CLI only -  In Web this is overridden by Session Init)
+# The security analyzer to use
 #security_analyzer = ""
 
 #################################### Eval ####################################

containers/app/Dockerfile

@@ -42,8 +42,6 @@ ENV USE_HOST_NETWORK=false
 ENV WORKSPACE_BASE=/opt/workspace_base
 ENV OPENHANDS_BUILD_VERSION=$OPENHANDS_BUILD_VERSION
 ENV SANDBOX_USER_ID=0
-ENV FILE_STORE=local
-ENV FILE_STORE_PATH=~/.openhands
 RUN mkdir -p $WORKSPACE_BASE
 
 RUN apt-get update -y \

containers/dev/README.md

@@ -1,8 +1,5 @@
 # Develop in Docker
 
-> [!WARNING]
-> This is not officially supported and may not work.
-
 Install [Docker](https://docs.docker.com/engine/install/) on your host machine and run:
 
 ```bash

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.16-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/about.md

@@ -27,7 +27,7 @@ Pour plus de détails, veuillez consulter [ce document](https://github.com/All-H
 
 Nous avons à la fois un espace de travail Slack pour la collaboration sur la construction d'OpenHands et un serveur Discord pour discuter de tout ce qui est lié, par exemple, à ce projet, LLM, agent, etc.
 
-- [Espace de travail Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw)
+- [Espace de travail Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
 - [Serveur Discord](https://discord.gg/ESHStjSjD4)
 
 Si vous souhaitez contribuer, n'hésitez pas à rejoindre notre communauté. Simplifions ensemble l'ingénierie logicielle !

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/custom_sandbox_guide.md

@@ -98,4 +98,4 @@ Si vous voyez un message d'erreur indiquant que le port est utilisé ou indispon
 
 ## Discuter
 
-Pour d'autres problèmes ou questions rejoignez le [Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw) ou le [Discord](https://discord.gg/ESHStjSjD4) et demandez!
+Pour d'autres problèmes ou questions rejoignez le [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) ou le [Discord](https://discord.gg/ESHStjSjD4) et demandez!

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/custom-sandbox-guide.md

@@ -80,4 +80,4 @@ Si vous voyez une erreur concernant un port déjà utilisé ou indisponible, ess
 
 ## Discuter
 
-Pour d'autres problèmes ou questions, rejoignez le [Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw) ou le [Discord](https://discord.gg/ESHStjSjD4) et demandez !
+Pour d'autres problèmes ou questions, rejoignez le [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) ou le [Discord](https://discord.gg/ESHStjSjD4) et demandez !

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -76,7 +76,7 @@ La fonction `run_controller()` est le cœur de l'exécution d'OpenHands. Elle g
 
 ## Le moyen le plus simple de commencer : Explorer les benchmarks existants
 
-Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/benchmarks/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) de notre dépôt.
+Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) de notre dépôt.
 
 Pour intégrer votre propre benchmark, nous vous suggérons de commencer par celui qui ressemble le plus à vos besoins. Cette approche peut considérablement rationaliser votre processus d'intégration, vous permettant de vous appuyer sur les structures existantes et de les adapter à vos exigences spécifiques.
 

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -14,4 +14,4 @@ Pour utiliser l'Action GitHub OpenHands dans le dépôt OpenHands, un mainteneur
 
 ## Installation de l'Action dans un nouveau dépôt
 
-Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
+Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/openshift-example.md

@@ -0,0 +1,338 @@
+
+
+# Kubernetes
+
+Il existe différentes façons d'exécuter OpenHands sur Kubernetes ou OpenShift. Ce guide présente une façon possible :
+1. Créer un PV "en tant qu'administrateur du cluster" pour mapper les données workspace_base et le répertoire docker au pod via le nœud worker
+2. Créer un PVC pour pouvoir monter ces PV sur le pod
+3. Créer un pod qui contient deux conteneurs : les conteneurs OpenHands et Sandbox
+
+## Étapes détaillées pour l'exemple ci-dessus
+
+> Remarque : Assurez-vous d'être connecté au cluster avec le compte approprié pour chaque étape. La création de PV nécessite un administrateur de cluster !
+
+> Assurez-vous d'avoir les autorisations de lecture/écriture sur le hostPath utilisé ci-dessous (c'est-à-dire /tmp/workspace)
+
+1. Créer le PV :
+Le fichier yaml d'exemple ci-dessous peut être utilisé par un administrateur de cluster pour créer le PV.
+- workspace-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: workspace-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /tmp/workspace
+```
+
+```bash
+# appliquer le fichier yaml
+$ oc create -f workspace-pv.yaml
+persistentvolume/workspace-pv created
+
+# vérifier :
+$ oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+- docker-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: docker-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /var/run/docker.sock
+```
+
+```bash
+# appliquer le fichier yaml
+$ oc create -f docker-pv.yaml
+persistentvolume/docker-pv created
+
+# vérifier :
+oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+docker-pv                                  2Gi        RWO            Retain           Available                                                  6m55s
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+2. Créer le PVC :
+Exemple de fichier yaml PVC ci-dessous :
+
+- workspace-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: workspace-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# créer le pvc
+$ oc create -f workspace-pvc.yaml
+persistentvolumeclaim/workspace-pvc created
+
+# vérifier
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+workspace-pvc   Pending                                      hcloud-volumes   4s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+8s          Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+- docker-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: docker-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# créer le pvc
+$ oc create -f docker-pvc.yaml
+persistentvolumeclaim/docker-pvc created
+
+# vérifier
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Pending                                      hcloud-volumes   4s
+workspace-pvc   Pending                                      hcloud-volumes   2m53s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+3. Créer le fichier yaml du pod :
+Exemple de fichier yaml de pod ci-dessous :
+
+- pod.yaml
+
+```yamlfile
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  containers:
+  - name: openhands-app-2024
+    image: ghcr.io/all-hands-ai/openhands:main
+    env:
+    - name: SANDBOX_USER_ID
+      value: "1000"
+    - name: WORKSPACE_MOUNT_PATH
+      value: "/opt/workspace_base"
+    volumeMounts:
+    - name: workspace-volume
+      mountPath: /opt/workspace_base
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+    ports:
+    - containerPort: 3000
+  - name: openhands-sandbox-2024
+    image: ghcr.io/all-hands-ai/sandbox:main
+    ports:
+    - containerPort: 51963
+    command: ["/usr/sbin/sshd", "-D", "-p 51963", "-o", "PermitRootLogin=yes"]
+  volumes:
+  - name: workspace-volume
+    persistentVolumeClaim:
+      claimName: workspace-pvc
+  - name: docker-sock
+    persistentVolumeClaim:
+      claimName: docker-pvc
+```
+
+
+```bash
+# créer le pod
+$ oc create -f pod.yaml
+W0716 11:22:07.776271  107626 warnings.go:70] would violate PodSecurity "restricted:v1.24": allowPrivilegeEscalation != false (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.capabilities.drop=["ALL"]), runAsNonRoot != true (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.runAsNonRoot=true), seccompProfile (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.seccompProfile.type to "RuntimeDefault" or "Localhost")
+pod/openhands-app-2024 created
+
+# L'avertissement ci-dessus peut être ignoré pour l'instant car nous ne modifierons pas les restrictions SCC.
+
+# vérifier
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   0/2     Pending   0          5s
+
+$ oc get pods
+NAME                 READY   STATUS              RESTARTS   AGE
+openhands-app-2024   0/2     ContainerCreating   0          15s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                   OBJECT                                MESSAGE
+38s         Normal   WaitForFirstConsumer     persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+23s         Normal   ExternalProvisioning     persistentvolumeclaim/docker-pvc      waiting for a volume to be created, either by external provisioner "csi.hetzner.cloud" or manually created by system administrator
+27s         Normal   Provisioning             persistentvolumeclaim/docker-pvc      External provisioner is provisioning volume for claim "openhands/docker-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/docker-pvc      Successfully provisioned volume pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252
+16s         Normal   Scheduled                pod/openhands-app-2024                Successfully assigned All-Hands-AI/OpenHands-app-2024 to worker1.hub.internal.blakane.com
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252"
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-31f15b25-faad-4665-a25f-201a530379af"
+6s          Normal   AddedInterface           pod/openhands-app-2024                Add eth0 [10.128.2.48/23] from openshift-sdn
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/openhands:main" already present on machine
+6s          Normal   Created                  pod/openhands-app-2024                Created container openhands-app-2024
+6s          Normal   Started                  pod/openhands-app-2024                Started container openhands-app-2024
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/sandbox:main" already present on machine
+5s          Normal   Created                  pod/openhands-app-2024                Created container openhands-sandbox-2024
+5s          Normal   Started                  pod/openhands-app-2024                Started container openhands-sandbox-2024
+83s         Normal   WaitForFirstConsumer     persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+27s         Normal   Provisioning             persistentvolumeclaim/workspace-pvc   External provisioner is provisioning volume for claim "openhands/workspace-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/workspace-pvc   Successfully provisioned volume pvc-31f15b25-faad-4665-a25f-201a530379af
+
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   2/2     Running   0          23s
+
+$ oc get pvc
+NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Bound    pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252   10Gi       RWO            hcloud-volumes   10m
+workspace-pvc   Bound    pvc-31f15b25-faad-4665-a25f-201a530379af   10Gi       RWO            hcloud-volumes   13m
+
+```
+
+4. Créer un service NodePort.
+Exemple de commande de création de service ci-dessous :
+
+```bash
+# créer le service de type NodePort
+$ oc create svc nodeport  openhands-app-2024  --tcp=3000:3000
+service/openhands-app-2024 created
+
+# vérifier
+
+$ oc get svc
+NAME                 TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
+openhands-app-2024   NodePort   172.30.225.42   <none>        3000:30495/TCP   4s
+
+$ oc describe svc openhands-app-2024
+Name:                     openhands-app-2024
+Namespace:                openhands
+Labels:                   app=openhands-app-2024
+Annotations:              <none>
+Selector:                 app=openhands-app-2024
+Type:                     NodePort
+IP Family Policy:         SingleStack
+IP Families:              IPv4
+IP:                       172.30.225.42
+IPs:                      172.30.225.42
+Port:                     3000-3000  3000/TCP
+TargetPort:               3000/TCP
+NodePort:                 3000-3000  30495/TCP
+Endpoints:                10.128.2.48:3000
+Session Affinity:         None
+External Traffic Policy:  Cluster
+Events:                   <none>
+```
+
+6. Se connecter à l'interface utilisateur d'OpenHands, configurer l'Agent, puis tester :
+
+![image](https://github.com/user-attachments/assets/12f94804-a0c7-4744-b873-e003c9caf40e)
+
+
+
+## Déploiement d'Openhands sur GCP GKE
+
+**Avertissement** : ce déploiement accorde à l'application OpenHands l'accès au socket docker de Kubernetes, ce qui crée un risque de sécurité. Utilisez à vos propres risques.
+1- Créer une politique pour l'accès privilégié
+2- Créer des informations d'identification gke (facultatif)
+3- Créer le déploiement openhands
+4- Commandes de vérification et d'accès à l'interface utilisateur
+5- Dépanner le pod pour vérifier le conteneur interne
+
+1. créer une politique pour l'accès privilégié
+```bash
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: privileged-role
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: privileged-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: privileged-role
+subjects:
+- kind: ServiceAccount
+  name: default  # Remplacez par le nom de votre compte de service
+  namespace: default
+```
+2. créer des informations d'identification gke (facultatif)
+```bash
+kubectl create secret generic google-cloud-key \
+  --from-file=key.json=/path/to/your/google-cloud-key.json
+  ```
+3. créer le déploiement openhands
+## comme cela est testé pour le nœud worker unique, si vous en avez plusieurs, spécifiez l'indicateur pour le worker unique
+
+```bash
+kind: Deployment
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  replicas: 1  # Vous pouvez augmenter ce nombre pour plusieurs réplicas
+  selector:
+    matchLabels:
+      app: openhands-app-2024
+  template:
+    metadata:
+      labels:
+        app: openhands-app-2024
+    spec:
+      containers:
+      -

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/intro.mdx

@@ -42,7 +42,7 @@ Explorez le code source d'OpenHands sur [GitHub](https://github.com/All-Hands-AI
   />
 </a>
 <br></br>
-<a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw">
+<a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA">
   <img
     src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge"
     alt="Join our Slack community"

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/troubleshooting/troubleshooting.md

@@ -9,6 +9,7 @@ Si vous trouvez plus d'informations ou une solution de contournement pour l'un d
 :::tip
 OpenHands ne prend en charge Windows que via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 Veuillez vous assurer d'exécuter toutes les commandes à l'intérieur de votre terminal WSL.
+Consultez les [Notes pour les utilisateurs de WSL sur Windows](troubleshooting/windows) pour des guides de dépannage.
 :::
 
 ## Problèmes courants

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/troubleshooting/windows.md

@@ -0,0 +1,66 @@
+
+
+# Notes pour les utilisateurs de WSL sur Windows
+
+OpenHands ne prend en charge Windows que via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+Veuillez vous assurer d'exécuter toutes les commandes dans votre terminal WSL.
+
+## Dépannage
+
+### Recommandation : Ne pas exécuter en tant qu'utilisateur root
+
+Pour des raisons de sécurité, il est fortement recommandé de ne pas exécuter OpenHands en tant qu'utilisateur root, mais en tant qu'utilisateur avec un UID non nul.
+
+Références :
+
+* [Pourquoi il est mauvais de se connecter en tant que root](https://askubuntu.com/questions/16178/why-is-it-bad-to-log-in-as-root)
+* [Définir l'utilisateur par défaut dans WSL](https://www.tenforums.com/tutorials/128152-set-default-user-windows-subsystem-linux-distro-windows-10-a.html#option2)
+Astuce concernant la 2ème référence : pour les utilisateurs d'Ubuntu, la commande pourrait en fait être "ubuntupreview" au lieu de "ubuntu".
+
+---
+### Erreur : 'docker' n'a pas pu être trouvé dans cette distribution WSL 2.
+
+Si vous utilisez Docker Desktop, assurez-vous de le démarrer avant d'appeler toute commande docker depuis WSL.
+Docker doit également avoir l'option d'intégration WSL activée.
+
+---
+### Installation de Poetry
+
+* Si vous rencontrez des problèmes pour exécuter Poetry même après l'avoir installé pendant le processus de build, vous devrez peut-être ajouter son chemin binaire à votre environnement :
+
+```sh
+export PATH="$HOME/.local/bin:$PATH"
+```
+
+* Si make build s'arrête sur une erreur comme celle-ci :
+
+```sh
+ModuleNotFoundError: no module named <module-name>
+```
+
+Cela pourrait être un problème avec le cache de Poetry.
+Essayez d'exécuter ces 2 commandes l'une après l'autre :
+
+```sh
+rm -r ~/.cache/pypoetry
+make build
+```
+
+---
+### L'objet NoneType n'a pas d'attribut 'request'
+
+Si vous rencontrez des problèmes liés au réseau, tels que `NoneType object has no attribute 'request'` lors de l'exécution de `make run`, vous devrez peut-être configurer les paramètres réseau de WSL2. Suivez ces étapes :
+
+* Ouvrez ou créez le fichier `.wslconfig` situé à `C:\Users\%username%\.wslconfig` sur votre machine hôte Windows.
+* Ajoutez la configuration suivante au fichier `.wslconfig` :
+
+```sh
+[wsl2]
+networkingMode=mirrored
+localhostForwarding=true
+```
+
+* Enregistrez le fichier `.wslconfig`.
+* Redémarrez complètement WSL2 en quittant toutes les instances WSL2 en cours d'exécution et en exécutant la commande `wsl --shutdown` dans votre invite de commande ou terminal.
+* Après avoir redémarré WSL, essayez d'exécuter à nouveau `make run`.
+Le problème de réseau devrait être résolu.

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/about.md

@@ -27,7 +27,7 @@ OpenHands 是一个社区驱动的项目，我们欢迎每个人的贡献。无
 
 我们有 Slack 工作区用于协作构建 OpenHands，也有 Discord 服务器用于讨论任何相关的内容，例如此项目、大语言模型、代理等。
 
-- [Slack 工作区](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw)
+- [Slack 工作区](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
 - [Discord 服务器](https://discord.gg/ESHStjSjD4)
 
 如果你想做出贡献，欢迎加入我们的社区。让我们一起简化软件工程！

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/custom_sandbox_guide.md

@@ -99,4 +99,4 @@ sandbox_user_id="1001"
 
 ## 讨论
 
-对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw) 或 [Discord](https://discord.gg/ESHStjSjD4) 提问！
+对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) 或 [Discord](https://discord.gg/ESHStjSjD4) 提问！

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/custom-sandbox-guide.md

@@ -78,4 +78,4 @@ sandbox_user_id="1001"
 
 ## 讨论
 
-对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw) 或 [Discord](https://discord.gg/ESHStjSjD4) 并提问！
+对于其他问题或疑问，请加入 [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) 或 [Discord](https://discord.gg/ESHStjSjD4) 并提问！

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -73,7 +73,7 @@ OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工
 
 ## 入门最简单的方法：探索现有基准
 
-我们鼓励您查看我们仓库的 [`evaluation/benchmarks/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks)中提供的各种评估基准。
+我们鼓励您查看我们仓库的 [`evaluation/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation)中提供的各种评估基准。
 
 要集成您自己的基准，我们建议从最接近您需求的基准开始。这种方法可以显著简化您的集成过程，允许您在现有结构的基础上进行构建并使其适应您的特定要求。
 

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -12,4 +12,4 @@
 
 ## 在新仓库中安装 Action
 
-要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md) 进行操作。
+要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow) 进行操作。

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/openshift-example.md

@@ -0,0 +1,343 @@
+以下是翻译后的内容:
+
+# Kubernetes
+
+在 Kubernetes 或 OpenShift 上运行 OpenHands 有不同的方式。本指南介绍了一种可能的方式:
+1. 作为集群管理员,创建一个 PV 将 workspace_base 数据和 docker 目录映射到 worker 节点上的 pod
+2. 创建一个 PVC 以便将这些 PV 挂载到 pod
+3. 创建一个包含两个容器的 pod:OpenHands 和 Sandbox 容器
+
+## 上述示例的详细步骤
+
+> 注意:确保首先使用适当的帐户登录到集群以执行每个步骤。创建 PV 需要集群管理员权限!
+
+> 确保你对下面使用的 hostPath(即 /tmp/workspace)有读写权限
+
+1. 创建 PV:
+集群管理员可以使用下面的示例 yaml 文件创建 PV。
+- workspace-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: workspace-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /tmp/workspace
+```
+
+```bash
+# 应用 yaml 文件
+$ oc create -f workspace-pv.yaml
+persistentvolume/workspace-pv created
+
+# 查看:
+$ oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+- docker-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: docker-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /var/run/docker.sock
+```
+
+```bash
+# 应用 yaml 文件
+$ oc create -f docker-pv.yaml
+persistentvolume/docker-pv created
+
+# 查看:
+oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+docker-pv                                  2Gi        RWO            Retain           Available                                                  6m55s
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+2. 创建 PVC:
+下面是示例 PVC yaml 文件:
+
+- workspace-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: workspace-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# 创建 pvc
+$ oc create -f workspace-pvc.yaml
+persistentvolumeclaim/workspace-pvc created
+
+# 查看
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+workspace-pvc   Pending                                      hcloud-volumes   4s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+8s          Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+- docker-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: docker-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# 创建 pvc
+$ oc create -f docker-pvc.yaml
+persistentvolumeclaim/docker-pvc created
+
+# 查看
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Pending                                      hcloud-volumes   4s
+workspace-pvc   Pending                                      hcloud-volumes   2m53s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+3. 创建 pod yaml 文件:
+下面是示例 pod yaml 文件:
+
+- pod.yaml
+
+```yamlfile
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  containers:
+  - name: openhands-app-2024
+    image: ghcr.io/all-hands-ai/openhands:main
+    env:
+    - name: SANDBOX_USER_ID
+      value: "1000"
+    - name: WORKSPACE_MOUNT_PATH
+      value: "/opt/workspace_base"
+    volumeMounts:
+    - name: workspace-volume
+      mountPath: /opt/workspace_base
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+    ports:
+    - containerPort: 3000
+  - name: openhands-sandbox-2024
+    image: ghcr.io/all-hands-ai/sandbox:main
+    ports:
+    - containerPort: 51963
+    command: ["/usr/sbin/sshd", "-D", "-p 51963", "-o", "PermitRootLogin=yes"]
+  volumes:
+  - name: workspace-volume
+    persistentVolumeClaim:
+      claimName: workspace-pvc
+  - name: docker-sock
+    persistentVolumeClaim:
+      claimName: docker-pvc
+```
+
+
+```bash
+# 创建 pod
+$ oc create -f pod.yaml
+W0716 11:22:07.776271  107626 warnings.go:70] would violate PodSecurity "restricted:v1.24": allowPrivilegeEscalation != false (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.capabilities.drop=["ALL"]), runAsNonRoot != true (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.runAsNonRoot=true), seccompProfile (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.seccompProfile.type to "RuntimeDefault" or "Localhost")
+pod/openhands-app-2024 created
+
+# 上面的警告可以暂时忽略,因为我们不会修改 SCC 限制。
+
+# 查看
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   0/2     Pending   0          5s
+
+$ oc get pods
+NAME                 READY   STATUS              RESTARTS   AGE
+openhands-app-2024   0/2     ContainerCreating   0          15s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                   OBJECT                                MESSAGE
+38s         Normal   WaitForFirstConsumer     persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+23s         Normal   ExternalProvisioning     persistentvolumeclaim/docker-pvc      waiting for a volume to be created, either by external provisioner "csi.hetzner.cloud" or manually created by system administrator
+27s         Normal   Provisioning             persistentvolumeclaim/docker-pvc      External provisioner is provisioning volume for claim "openhands/docker-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/docker-pvc      Successfully provisioned volume pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252
+16s         Normal   Scheduled                pod/openhands-app-2024                Successfully assigned All-Hands-AI/OpenHands-app-2024 to worker1.hub.internal.blakane.com
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252"
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-31f15b25-faad-4665-a25f-201a530379af"
+6s          Normal   AddedInterface           pod/openhands-app-2024                Add eth0 [10.128.2.48/23] from openshift-sdn
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/openhands:main" already present on machine
+6s          Normal   Created                  pod/openhands-app-2024                Created container openhands-app-2024
+6s          Normal   Started                  pod/openhands-app-2024                Started container openhands-app-2024
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "ghcr.io/all-hands-ai/sandbox:main" already present on machine
+5s          Normal   Created                  pod/openhands-app-2024                Created container openhands-sandbox-2024
+5s          Normal   Started                  pod/openhands-app-2024                Started container openhands-sandbox-2024
+83s         Normal   WaitForFirstConsumer     persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+27s         Normal   Provisioning             persistentvolumeclaim/workspace-pvc   External provisioner is provisioning volume for claim "openhands/workspace-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/workspace-pvc   Successfully provisioned volume pvc-31f15b25-faad-4665-a25f-201a530379af
+
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   2/2     Running   0          23s
+
+$ oc get pvc
+NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Bound    pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252   10Gi       RWO            hcloud-volumes   10m
+workspace-pvc   Bound    pvc-31f15b25-faad-4665-a25f-201a530379af   10Gi       RWO            hcloud-volumes   13m
+
+```
+
+4. 创建一个 NodePort 服务。
+下面是示例服务创建命令:
+
+```bash
+# 创建 NodePort 类型的服务
+$ oc create svc nodeport  openhands-app-2024  --tcp=3000:3000
+service/openhands-app-2024 created
+
+# 查看
+
+$ oc get svc
+NAME                 TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
+openhands-app-2024   NodePort   172.30.225.42   <none>        3000:30495/TCP   4s
+
+$ oc describe svc openhands-app-2024
+Name:                     openhands-app-2024
+Namespace:                openhands
+Labels:                   app=openhands-app-2024
+Annotations:              <none>
+Selector:                 app=openhands-app-2024
+Type:                     NodePort
+IP Family Policy:         SingleStack
+IP Families:              IPv4
+IP:                       172.30.225.42
+IPs:                      172.30.225.42
+Port:                     3000-3000  3000/TCP
+TargetPort:               3000/TCP
+NodePort:                 3000-3000  30495/TCP
+Endpoints:                10.128.2.48:3000
+Session Affinity:         None
+External Traffic Policy:  Cluster
+Events:                   <none>
+```
+
+6. 连接到 OpenHands UI,配置 Agent,然后测试:
+
+![image](https://github.com/user-attachments/assets/12f94804-a0c7-4744-b873-e003c9caf40e)
+
+
+
+## GCP GKE OpenHands 部署
+
+**警告**:此部署授予 OpenHands 应用程序访问 Kubernetes docker socket 的权限,这会带来安全风险。请自行决定是否使用。
+1- 创建特权访问策略
+2- 创建 gke 凭证(可选)
+3- 创建 openhands 部署
+4- 验证和 UI 访问命令
+5- 排查 pod 以验证内部容器
+
+1. 创建特权访问策略
+```bash
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: privileged-role
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: privileged-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: privileged-role
+subjects:
+- kind: ServiceAccount
+  name: default  # 更改为你的服务帐户名称
+  namespace: default
+```
+2. 创建 gke 凭证(可选)
+```bash
+kubectl create secret generic google-cloud-key \
+  --from-file=key.json=/path/to/your/google-cloud-key.json
+  ```
+3. 创建 openhands 部署
+## 由于这是针对单个工作节点进行测试的,如果你有多个节点,请指定单个工作节点的标志
+
+```bash
+kind: Deployment
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  replicas: 1  # 你可以增加这个数字以获得多个副本
+  selector:
+    matchLabels:
+      app: openhands-app-2024
+  template:
+    metadata:
+      labels:
+        app: openhands-app-2024
+    spec:
+      containers:
+      - name: openhands-app-2024
+        image: ghcr.io/all-hands-ai/openhands:main
+        env:
+        - name: SANDBOX_USER_ID
+          value: "1000"
+        - name: SANDBOX_API

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/intro.mdx

@@ -42,7 +42,7 @@ OpenHands 是一个**自主 AI 软件工程师**，能够执行复杂的工程
   />
 </a>
 <br></br>
-<a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw">
+<a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA">
   <img
     src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge"
     alt="Join our Slack community"

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/troubleshooting/troubleshooting.md

@@ -7,6 +7,7 @@
 :::tip
 OpenHands 仅通过 [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) 支持 Windows。
 请确保在您的 WSL 终端内运行所有命令。
+查看 [Windows 用户的 WSL 注意事项](troubleshooting/windows) 以获取一些故障排除指南。
 :::
 
 ## 常见问题

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/troubleshooting/windows.md

@@ -0,0 +1,66 @@
+以下是翻译后的内容:
+
+# 针对 Windows 上 WSL 用户的注意事项
+
+OpenHands 仅通过 [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) 支持 Windows。
+请确保在您的 WSL 终端内运行所有命令。
+
+## 故障排除
+
+### 建议: 不要以 root 用户身份运行
+
+出于安全原因,强烈建议不要以 root 用户身份运行 OpenHands,而是以具有非零 UID 的用户身份运行。
+
+参考:
+
+* [为什么以 root 身份登录不好](https://askubuntu.com/questions/16178/why-is-it-bad-to-log-in-as-root)
+* [在 WSL 中设置默认用户](https://www.tenforums.com/tutorials/128152-set-default-user-windows-subsystem-linux-distro-windows-10-a.html#option2)
+关于第二个参考的提示:对于 Ubuntu 用户,命令实际上可能是 "ubuntupreview" 而不是 "ubuntu"。
+
+---
+### 错误: 在此 WSL 2 发行版中找不到 'docker'。
+
+如果您正在使用 Docker Desktop,请确保在从 WSL 内部调用任何 docker 命令之前启动它。
+Docker 还需要激活 WSL 集成选项。
+
+---
+### Poetry 安装
+
+* 如果您在构建过程中安装 Poetry 后仍然面临运行 Poetry 的问题,您可能需要将其二进制路径添加到环境中:
+
+```sh
+export PATH="$HOME/.local/bin:$PATH"
+```
+
+* 如果 make build 在如下错误上停止:
+
+```sh
+ModuleNotFoundError: no module named <module-name>
+```
+
+这可能是 Poetry 缓存的问题。
+尝试依次运行这两个命令:
+
+```sh
+rm -r ~/.cache/pypoetry
+make build
+```
+
+---
+### NoneType 对象没有属性 'request'
+
+如果您在执行 `make run` 时遇到与网络相关的问题,例如 `NoneType 对象没有属性 'request'`,您可能需要配置 WSL2 网络设置。请按照以下步骤操作:
+
+* 在 Windows 主机上打开或创建位于 `C:\Users\%username%\.wslconfig` 的 `.wslconfig` 文件。
+* 将以下配置添加到 `.wslconfig` 文件中:
+
+```sh
+[wsl2]
+networkingMode=mirrored
+localhostForwarding=true
+```
+
+* 保存 `.wslconfig` 文件。
+* 通过退出任何正在运行的 WSL2 实例并在命令提示符或终端中执行 `wsl --shutdown` 命令来完全重启 WSL2。
+* 重新启动 WSL 后,再次尝试执行 `make run`。
+网络问题应该得到解决。

docs/modules/usage/about.md

@@ -21,6 +21,9 @@ OpenHands is built using a combination of powerful frameworks and libraries, pro
 
 Please note that the selection of these technologies is in progress, and additional technologies may be added or existing ones may be removed as the project evolves. We strive to adopt the most suitable and efficient tools to enhance the capabilities of OpenHands.
 
-## License
+## Licensing, Contributing, Community Servers
 
 Distributed under MIT [License](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE).
+
+For guides on how to contribute to OpenHands, joining our Discord and Slack servers
+[check out the OpenHands README.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-contribute)

docs/modules/usage/configuration-options.md

@@ -1,465 +0,0 @@
-# Configuration Options
-
-This guide details all configuration options available for OpenHands, helping you customize its behavior and integrate it with other services.
-
-:::note
-If you are running in [GUI Mode](https://docs.all-hands.dev/modules/usage/how-to/gui-mode), the settings available in the Settings UI will always
-take precedence.
-:::
-
----
-
-# Table of Contents
-
-1. [Core Configuration](#core-configuration)
-   - [API Keys](#api-keys)
-   - [Workspace](#workspace)
-   - [Debugging and Logging](#debugging-and-logging)
-   - [Session Management](#session-management)
-   - [Trajectories](#trajectories)
-   - [File Store](#file-store)
-   - [Task Management](#task-management)
-   - [Sandbox Configuration](#sandbox-configuration)
-   - [Miscellaneous](#miscellaneous)
-2. [LLM Configuration](#llm-configuration)
-   - [AWS Credentials](#aws-credentials)
-   - [API Configuration](#api-configuration)
-   - [Custom LLM Provider](#custom-llm-provider)
-   - [Embeddings](#embeddings)
-   - [Message Handling](#message-handling)
-   - [Model Selection](#model-selection)
-   - [Retrying](#retrying)
-   - [Advanced Options](#advanced-options)
-3. [Agent Configuration](#agent-configuration)
-   - [Microagent Configuration](#microagent-configuration)
-   - [Memory Configuration](#memory-configuration)
-   - [LLM Configuration](#llm-configuration-2)
-   - [ActionSpace Configuration](#actionspace-configuration)
-   - [Microagent Usage](#microagent-usage)
-4. [Sandbox Configuration](#sandbox-configuration-2)
-   - [Execution](#execution)
-   - [Container Image](#container-image)
-   - [Networking](#networking)
-   - [Linting and Plugins](#linting-and-plugins)
-   - [Dependencies and Environment](#dependencies-and-environment)
-   - [Evaluation](#evaluation)
-5. [Security Configuration](#security-configuration)
-   - [Confirmation Mode](#confirmation-mode)
-   - [Security Analyzer](#security-analyzer)
-
----
-
-## Core Configuration
-
-The core configuration options are defined in the `[core]` section of the `config.toml` file.
-
-**API Keys**
-- `e2b_api_key`
-  - Type: `str`
-  - Default: `""`
-  - Description: API key for E2B
-
-- `modal_api_token_id`
-  - Type: `str`
-  - Default: `""`
-  - Description: API token ID for Modal
-
-- `modal_api_token_secret`
-  - Type: `str`
-  - Default: `""`
-  - Description: API token secret for Modal
-
-**Workspace**
-- `workspace_base`
-  - Type: `str`
-  - Default: `"./workspace"`
-  - Description: Base path for the workspace
-
-- `cache_dir`
-  - Type: `str`
-  - Default: `"/tmp/cache"`
-  - Description: Cache directory path
-
-**Debugging and Logging**
-- `debug`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Enable debugging
-
-- `disable_color`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Disable color in terminal output
-
-**Trajectories**
-- `trajectories_path`
-  - Type: `str`
-  - Default: `"./trajectories"`
-  - Description: Path to store trajectories (can be a folder or a file). If it's a folder, the trajectories will be saved in a file named with the session id name and .json extension, in that folder.
-
-**File Store**
-- `file_store_path`
-  - Type: `str`
-  - Default: `"/tmp/file_store"`
-  - Description: File store path
-
-- `file_store`
-  - Type: `str`
-  - Default: `"memory"`
-  - Description: File store type
-
-- `file_uploads_allowed_extensions`
-  - Type: `list of str`
-  - Default: `[".*"]`
-  - Description: List of allowed file extensions for uploads
-
-- `file_uploads_max_file_size_mb`
-  - Type: `int`
-  - Default: `0`
-  - Description: Maximum file size for uploads, in megabytes
-
-- `file_uploads_restrict_file_types`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Restrict file types for file uploads
-
-- `file_uploads_allowed_extensions`
-  - Type: `list of str`
-  - Default: `[".*"]`
-  - Description: List of allowed file extensions for uploads
-
-**Task Management**
-- `max_budget_per_task`
-  - Type: `float`
-  - Default: `0.0`
-  - Description: Maximum budget per task (0.0 means no limit)
-
-- `max_iterations`
-  - Type: `int`
-  - Default: `100`
-  - Description: Maximum number of iterations
-
-**Sandbox Configuration**
-- `workspace_mount_path_in_sandbox`
-  - Type: `str`
-  - Default: `"/workspace"`
-  - Description: Path to mount the workspace in the sandbox
-
-- `workspace_mount_path`
-  - Type: `str`
-  - Default: `""`
-  - Description: Path to mount the workspace
-
-- `workspace_mount_rewrite`
-  - Type: `str`
-  - Default: `""`
-  - Description: Path to rewrite the workspace mount path to. You can usually ignore this, it refers to special cases of running inside another container.
-
-**Miscellaneous**
-- `run_as_openhands`
-  - Type: `bool`
-  - Default: `true`
-  - Description: Run as OpenHands
-
-- `runtime`
-  - Type: `str`
-  - Default: `"eventstream"`
-  - Description: Runtime environment
-
-- `default_agent`
-  - Type: `str`
-  - Default: `"CodeActAgent"`
-  - Description: Name of the default agent
-
-- `jwt_secret`
-  - Type: `str`
-  - Default: `uuid.uuid4().hex`
-  - Description: JWT secret for authentication. Please set it to your own value.
-
-## LLM Configuration
-
-The LLM (Large Language Model) configuration options are defined in the `[llm]` section of the `config.toml` file.
-
-To use these with the docker command, pass in `-e LLM_<option>`. Example: `-e LLM_NUM_RETRIES`.
-
-**AWS Credentials**
-- `aws_access_key_id`
-  - Type: `str`
-  - Default: `""`
-  - Description: AWS access key ID
-
-- `aws_region_name`
-  - Type: `str`
-  - Default: `""`
-  - Description: AWS region name
-
-- `aws_secret_access_key`
-  - Type: `str`
-  - Default: `""`
-  - Description: AWS secret access key
-
-**API Configuration**
-- `api_key`
-  - Type: `str`
-  - Default: `None`
-  - Description: API key to use
-
-- `base_url`
-  - Type: `str`
-  - Default: `""`
-  - Description: API base URL
-
-- `api_version`
-  - Type: `str`
-  - Default: `""`
-  - Description: API version
-
-- `input_cost_per_token`
-  - Type: `float`
-  - Default: `0.0`
-  - Description: Cost per input token
-
-- `output_cost_per_token`
-  - Type: `float`
-  - Default: `0.0`
-  - Description: Cost per output token
-
-**Custom LLM Provider**
-- `custom_llm_provider`
-  - Type: `str`
-  - Default: `""`
-  - Description: Custom LLM provider
-
-**Embeddings**
-- `embedding_base_url`
-  - Type: `str`
-  - Default: `""`
-  - Description: Embedding API base URL
-
-- `embedding_deployment_name`
-  - Type: `str`
-  - Default: `""`
-  - Description: Embedding deployment name
-
-- `embedding_model`
-  - Type: `str`
-  - Default: `"local"`
-  - Description: Embedding model to use
-
-**Message Handling**
-- `max_message_chars`
-  - Type: `int`
-  - Default: `30000`
-  - Description: The approximate maximum number of characters in the content of an event included in the prompt to the LLM. Larger observations are truncated.
-
-- `max_input_tokens`
-  - Type: `int`
-  - Default: `0`
-  - Description: Maximum number of input tokens
-
-- `max_output_tokens`
-  - Type: `int`
-  - Default: `0`
-  - Description: Maximum number of output tokens
-
-**Model Selection**
-- `model`
-  - Type: `str`
-  - Default: `"claude-3-5-sonnet-20241022"`
-  - Description: Model to use
-
-**Retrying**
-- `num_retries`
-  - Type: `int`
-  - Default: `8`
-  - Description: Number of retries to attempt
-
-- `retry_max_wait`
-  - Type: `int`
-  - Default: `120`
-  - Description: Maximum wait time (in seconds) between retry attempts
-
-- `retry_min_wait`
-  - Type: `int`
-  - Default: `15`
-  - Description: Minimum wait time (in seconds) between retry attempts
-
-- `retry_multiplier`
-  - Type: `float`
-  - Default: `2.0`
-  - Description: Multiplier for exponential backoff calculation
-
-**Advanced Options**
-- `drop_params`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Drop any unmapped (unsupported) params without causing an exception
-
-- `caching_prompt`
-  - Type: `bool`
-  - Default: `true`
-  - Description: Using the prompt caching feature if provided by the LLM and supported
-
-- `ollama_base_url`
-  - Type: `str`
-  - Default: `""`
-  - Description: Base URL for the OLLAMA API
-
-- `temperature`
-  - Type: `float`
-  - Default: `0.0`
-  - Description: Temperature for the API
-
-- `timeout`
-  - Type: `int`
-  - Default: `0`
-  - Description: Timeout for the API
-
-- `top_p`
-  - Type: `float`
-  - Default: `1.0`
-  - Description: Top p for the API
-
-- `disable_vision`
-  - Type: `bool`
-  - Default: `None`
-  - Description: If model is vision capable, this option allows to disable image processing (useful for cost reduction)
-
-## Agent Configuration
-
-The agent configuration options are defined in the `[agent]` and `[agent.<agent_name>]` sections of the `config.toml` file.
-
-**Microagent Configuration**
-- `micro_agent_name`
-  - Type: `str`
-  - Default: `""`
-  - Description: Name of the micro agent to use for this agent
-
-**Memory Configuration**
-- `memory_enabled`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Whether long-term memory (embeddings) is enabled
-
-- `memory_max_threads`
-  - Type: `int`
-  - Default: `3`
-  - Description: The maximum number of threads indexing at the same time for embeddings
-
-**LLM Configuration**
-- `llm_config`
-  - Type: `str`
-  - Default: `'your-llm-config-group'`
-  - Description: The name of the LLM config to use
-
-**ActionSpace Configuration**
-- `function_calling`
-  - Type: `bool`
-  - Default: `true`
-  - Description: Whether function calling is enabled
-
-- `codeact_enable_browsing`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Whether browsing delegate is enabled in the action space (only works with function calling)
-
-- `codeact_enable_llm_editor`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Whether LLM editor is enabled in the action space (only works with function calling)
-
-- `codeact_enable_jupyter`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Whether Jupyter is enabled in the action space
-
-**Microagent Usage**
-- `use_microagents`
-  - Type: `bool`
-  - Default: `true`
-  - Description: Whether to use microagents at all
-
-- `disabled_microagents`
-  - Type: `list of str`
-  - Default: `None`
-  - Description: A list of microagents to disable
-
-## Sandbox Configuration
-
-The sandbox configuration options are defined in the `[sandbox]` section of the `config.toml` file.
-
-To use these with the docker command, pass in `-e SANDBOX_<option>`. Example: `-e SANDBOX_TIMEOUT`.
-
-**Execution**
-- `timeout`
-  - Type: `int`
-  - Default: `120`
-  - Description: Sandbox timeout in seconds
-
-- `user_id`
-  - Type: `int`
-  - Default: `1000`
-  - Description: Sandbox user ID
-
-**Container Image**
-- `base_container_image`
-  - Type: `str`
-  - Default: `"nikolaik/python-nodejs:python3.12-nodejs22"`
-  - Description: Container image to use for the sandbox
-
-**Networking**
-- `use_host_network`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Use host network
-
-**Linting and Plugins**
-- `enable_auto_lint`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Enable auto linting after editing
-
-- `initialize_plugins`
-  - Type: `bool`
-  - Default: `true`
-  - Description: Whether to initialize plugins
-
-**Dependencies and Environment**
-- `runtime_extra_deps`
-  - Type: `str`
-  - Default: `""`
-  - Description: Extra dependencies to install in the runtime image
-
-- `runtime_startup_env_vars`
-  - Type: `dict`
-  - Default: `{}`
-  - Description: Environment variables to set at the launch of the runtime
-
-**Evaluation**
-- `browsergym_eval_env`
-  - Type: `str`
-  - Default: `""`
-  - Description: BrowserGym environment to use for evaluation
-
-## Security Configuration
-
-The security configuration options are defined in the `[security]` section of the `config.toml` file.
-
-To use these with the docker command, pass in `-e SECURITY_<option>`. Example: `-e SECURITY_CONFIRMATION_MODE`.
-
-**Confirmation Mode**
-- `confirmation_mode`
-  - Type: `bool`
-  - Default: `false`
-  - Description: Enable confirmation mode
-
-**Security Analyzer**
-- `security_analyzer`
-  - Type: `str`
-  - Default: `""`
-  - Description: The security analyzer to use
-
----
-
-> **Note**: Adjust configurations carefully, especially for memory, security, and network-related settings to ensure optimal performance and security.
-Please note that the configuration options may be subject to change in future versions of OpenHands. It's recommended to refer to the official documentation for the most up-to-date information.

docs/modules/usage/how-to/cli-mode.md

@@ -50,7 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -59,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.16 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
     python -m openhands.core.cli
 ```
 

docs/modules/usage/how-to/evaluation-harness.md

@@ -73,7 +73,7 @@ The `run_controller()` function is the core of OpenHands's execution. It manages
 
 ## Easiest way to get started: Exploring Existing Benchmarks
 
-We encourage you to review the various evaluation benchmarks available in the [`evaluation/benchmarks/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) of our repository.
+We encourage you to review the various evaluation benchmarks available in the [`evaluation/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) of our repository.
 
 To integrate your own benchmark, we suggest starting with the one that most closely resembles your needs. This approach can significantly streamline your integration process, allowing you to build upon existing structures and adapt them to your specific requirements.
 

docs/modules/usage/how-to/github-action.md

@@ -4,106 +4,13 @@ This guide explains how to use the OpenHands GitHub Action, both within the Open
 
 ## Using the Action in the OpenHands Repository
 
-To use the OpenHands GitHub Action in a repository, you can:
+To use the OpenHands GitHub Action in the OpenHands repository, an OpenHands maintainer can:
 
 1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue or leave a comment on the issue starting with `@openhands-agent`.
-
-The action will automatically trigger and attempt to resolve the issue.
+2. Add the `fix-me` label to the issue.
+3. The action will automatically trigger and attempt to resolve the issue.
 
 ## Installing the Action in a New Repository
 
 To install the OpenHands GitHub Action in your own repository, follow
 the [README for the OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
-
-## Usage Tips
-
-### Iterative resolution
-
-1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue, or leave a comment starting with `@openhands-agent`
-3. Review the attempt to resolve the issue by checking the pull request
-4. Follow up with feedback through general comments, review comments, or inline thread comments
-5. Add the `fix-me` label to the pull request, or address a specific comment by starting with `@openhands-agent`
-
-### Label versus Macro
-
-- Label (`fix-me`): Requests OpenHands to address the **entire** issue or pull request.
-- Macro (`@openhands-agent`): Requests OpenHands to consider only the issue/pull request description and **the specific comment**.
-
-## Advanced Settings
-
-### Add custom repository settings
-
-You can provide custom directions for OpenHands by following the [README for the resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md#providing-custom-instructions).
-
-### Custom configurations
-
-Github resolver will automatically check for valid [repository secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions?tool=webui#creating-secrets-for-a-repository) or [repository variables](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#creating-configuration-variables-for-a-repository) to customize its behavior.
-The customization options you can set are:
-
-| **Attribute name**               | **Type** | **Purpose**                                                                                                 | **Example**                                          |
-|----------------------------------| -------- |-------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
-| `LLM_MODEL`                      | Variable | Set the LLM to use with OpenHands                                                                           | `LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"`   |
-| `OPENHANDS_MAX_ITER`             | Variable | Set max limit for agent iterations                                                                          | `OPENHANDS_MAX_ITER=10`                              |
-| `OPENHANDS_MACRO`                | Variable | Customize default macro for invoking the resolver                                                           | `OPENHANDS_MACRO=@resolveit`                         |
-| `OPENHANDS_BASE_CONTAINER_IMAGE` | Variable | Custom Sandbox ([learn more](https://docs.all-hands.dev/modules/usage/how-to/custom-sandbox-guide))         | `OPENHANDS_BASE_CONTAINER_IMAGE="custom_image"`      |
-
-## Writing Effective .openhands_instructions Files
-
-The `.openhands_instructions` file is a file that you can put in the root directory of your repository to guide OpenHands
-in understanding and working with your repository effectively. Here are key tips for writing high-quality instructions:
-
-### Core Principles
-
-1. **Concise but Informative**: Provide a clear, focused overview of the repository that emphasizes the most common
-     actions OpenHands will need to perform.
-
-2. **Repository Structure**: Explain the key directories and their purposes, especially highlighting where different
-     types of code (e.g., frontend, backend) are located.
-
-3. **Development Workflows**: Document the essential commands for:
-
-   - Building and setting up the project
-   - Running tests
-   - Linting and code quality checks
-   - Any environment-specific requirements
-
-4. **Testing Guidelines**: Specify:
-   - Where tests are located
-   - How to run specific test suites
-   - Any testing conventions or requirements
-
-### Example Structure
-
-```markdown
-# Repository Overview
-
-[Brief description of the project]
-
-## General Setup
-
-- Main build command
-- Development environment setup
-- Pre-commit checks
-
-## Backend
-
-- Location and structure
-- Testing instructions
-- Environment requirements
-
-## Frontend
-
-- Setup prerequisites
-- Build and test commands
-- Environment variables
-
-## Additional Guidelines
-
-- Code style requirements
-- Special considerations
-- Common workflows
-```
-
-For a real-world example, refer to the [OpenHands repository's .openhands_instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/.openhands_instructions).

docs/modules/usage/how-to/gui-mode.md

@@ -23,75 +23,10 @@ OpenHands provides a user-friendly Graphical User Interface (GUI) mode for inter
 
 OpenHands automatically exports a `GITHUB_TOKEN` to the shell environment if it is available. This can happen in two ways:
 
-1. **Locally (OSS)**: The user directly inputs their GitHub token
-2. **Online (SaaS)**: The token is obtained through GitHub OAuth authentication
+1. Locally (OSS): The user directly inputs their GitHub token.
+2. Online (SaaS): The token is obtained through GitHub OAuth authentication.
 
-#### Setting Up a Local GitHub Token
-
-1. **Generate a Personal Access Token (PAT)**:
-   - Go to GitHub Settings > Developer Settings > Personal Access Tokens > Tokens (classic)
-   - Click "Generate new token (classic)"
-   - Required scopes:
-     - `repo` (Full control of private repositories)
-     - `workflow` (Update GitHub Action workflows)
-     - `read:org` (Read organization data)
-
-2. **Enter Token in OpenHands**:
-   - Click the Settings button (gear icon) in the top right
-   - Navigate to the "GitHub" section
-   - Paste your token in the "GitHub Token" field
-   - Click "Save" to apply the changes
-
-#### Organizational Token Policies
-
-If you're working with organizational repositories, additional setup may be required:
-
-1. **Check Organization Requirements**:
-   - Organization admins may enforce specific token policies
-   - Some organizations require tokens to be created with SSO enabled
-   - Review your organization's [token policy settings](https://docs.github.com/en/organizations/managing-programmatic-access-to-your-organization/setting-a-personal-access-token-policy-for-your-organization)
-
-2. **Verify Organization Access**:
-   - Go to your token settings on GitHub
-   - Look for the organization under "Organization access"
-   - If required, click "Enable SSO" next to your organization
-   - Complete the SSO authorization process
-
-#### OAuth Authentication (Online Mode)
-
-When using OpenHands in online mode, the GitHub OAuth flow:
-
-1. Requests the following permissions:
-   - Repository access (read/write)
-   - Workflow management
-   - Organization read access
-
-2. Authentication steps:
-   - Click "Sign in with GitHub" when prompted
-   - Review the requested permissions
-   - Authorize OpenHands to access your GitHub account
-   - If using an organization, authorize organization access if prompted
-
-#### Troubleshooting
-
-Common issues and solutions:
-
-1. **Token Not Recognized**:
-   - Ensure the token is properly saved in settings
-   - Check that the token hasn't expired
-   - Verify the token has the required scopes
-   - Try regenerating the token
-
-2. **Organization Access Denied**:
-   - Check if SSO is required but not enabled
-   - Verify organization membership
-   - Contact organization admin if token policies are blocking access
-
-3. **Verifying Token Works**:
-   - The app will show a green checkmark if the token is valid
-   - Try accessing a repository to confirm permissions
-   - Check the browser console for any error messages
-   - Use the "Test Connection" button in settings if available
+When you reach the `/app` route, the app checks if a token is present. If it finds one, it sets it in the environment for the agent to use.
 
 ### Advanced Settings
 

docs/modules/usage/how-to/headless-mode.md

@@ -44,7 +44,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -54,6 +54,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.16 \
-    python -m openhands.core.main -t "write a bash script that prints hi" --no-auto-continue
+    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
+    python -m openhands.core.main -t "write a bash script that prints hi"
 ```

docs/modules/usage/how-to/openshift-example.md

@@ -0,0 +1,429 @@
+# Kubernetes
+
+There are different ways you might run OpenHands on Kubernetes or OpenShift. This guide goes through one possible way:
+1. Create a PV "as a cluster admin" to map workspace_base data and docker directory to the pod through the worker node
+2. Create a PVC to be able to mount those PVs to the pod
+3. Create a pod which contains two containers; the OpenHands and Sandbox containers
+
+## Detailed Steps for the Example Above
+
+> Note: Make sure you are logged in to the cluster first with the proper account for each step. PV creation requires cluster administrator!
+
+> Make sure you have read/write permissions on the hostPath used below (i.e. /tmp/workspace)
+
+1. Create the PV:
+Sample yaml file below can be used by a cluster admin to create the PV.
+- workspace-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: workspace-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /tmp/workspace
+```
+
+```bash
+# apply yaml file
+$ oc create -f workspace-pv.yaml
+persistentvolume/workspace-pv created
+
+# review:
+$ oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+- docker-pv.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: docker-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /var/run/docker.sock
+```
+
+```bash
+# apply yaml file
+$ oc create -f docker-pv.yaml
+persistentvolume/docker-pv created
+
+# review:
+oc get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                STORAGECLASS     REASON   AGE
+docker-pv                                  2Gi        RWO            Retain           Available                                                  6m55s
+workspace-pv                               2Gi        RWO            Retain           Available                                                  7m23s
+```
+
+2. Create the PVC:
+Sample PVC yaml file below:
+
+- workspace-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: workspace-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# create the pvc
+$ oc create -f workspace-pvc.yaml
+persistentvolumeclaim/workspace-pvc created
+
+# review
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+workspace-pvc   Pending                                      hcloud-volumes   4s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+8s          Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+- docker-pvc.yaml
+
+```yamlfile
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: docker-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+```
+
+```bash
+# create pvc
+$ oc create -f docker-pvc.yaml
+persistentvolumeclaim/docker-pvc created
+
+# review
+$ oc get pvc
+NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Pending                                      hcloud-volumes   4s
+workspace-pvc   Pending                                      hcloud-volumes   2m53s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                 OBJECT                                MESSAGE
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+10s         Normal   WaitForFirstConsumer   persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+```
+
+3. Create the pod yaml file:
+Sample pod yaml file below:
+
+- pod.yaml
+
+```yamlfile
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  containers:
+  - name: openhands-app-2024
+    image: docker.all-hands.dev/all-hands-ai/openhands:main
+    env:
+    - name: SANDBOX_USER_ID
+      value: "1000"
+    - name: WORKSPACE_MOUNT_PATH
+      value: "/opt/workspace_base"
+    volumeMounts:
+    - name: workspace-volume
+      mountPath: /opt/workspace_base
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+    ports:
+    - containerPort: 3000
+  - name: openhands-sandbox-2024
+    image: docker.all-hands.dev/all-hands-ai/runtime:main
+    ports:
+    - containerPort: 51963
+    command: ["/usr/sbin/sshd", "-D", "-p 51963", "-o", "PermitRootLogin=yes"]
+  volumes:
+  - name: workspace-volume
+    persistentVolumeClaim:
+      claimName: workspace-pvc
+  - name: docker-sock
+    persistentVolumeClaim:
+      claimName: docker-pvc
+```
+
+
+```bash
+# create the pod
+$ oc create -f pod.yaml
+W0716 11:22:07.776271  107626 warnings.go:70] would violate PodSecurity "restricted:v1.24": allowPrivilegeEscalation != false (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.capabilities.drop=["ALL"]), runAsNonRoot != true (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.runAsNonRoot=true), seccompProfile (pod or containers "openhands-app-2024", "openhands-sandbox-2024" must set securityContext.seccompProfile.type to "RuntimeDefault" or "Localhost")
+pod/openhands-app-2024 created
+
+# Above warning can be ignored for now as we will not modify SCC restrictions.
+
+# review
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   0/2     Pending   0          5s
+
+$ oc get pods
+NAME                 READY   STATUS              RESTARTS   AGE
+openhands-app-2024   0/2     ContainerCreating   0          15s
+
+$ oc get events
+LAST SEEN   TYPE     REASON                   OBJECT                                MESSAGE
+38s         Normal   WaitForFirstConsumer     persistentvolumeclaim/docker-pvc      waiting for first consumer to be created before binding
+23s         Normal   ExternalProvisioning     persistentvolumeclaim/docker-pvc      waiting for a volume to be created, either by external provisioner "csi.hetzner.cloud" or manually created by system administrator
+27s         Normal   Provisioning             persistentvolumeclaim/docker-pvc      External provisioner is provisioning volume for claim "openhands/docker-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/docker-pvc      Successfully provisioned volume pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252
+16s         Normal   Scheduled                pod/openhands-app-2024                Successfully assigned All-Hands-AI/OpenHands-app-2024 to worker1.hub.internal.blakane.com
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252"
+9s          Normal   SuccessfulAttachVolume   pod/openhands-app-2024                AttachVolume.Attach succeeded for volume "pvc-31f15b25-faad-4665-a25f-201a530379af"
+6s          Normal   AddedInterface           pod/openhands-app-2024                Add eth0 [10.128.2.48/23] from openshift-sdn
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "docker.all-hands.dev/all-hands-ai/openhands:main" already present on machine
+6s          Normal   Created                  pod/openhands-app-2024                Created container openhands-app-2024
+6s          Normal   Started                  pod/openhands-app-2024                Started container openhands-app-2024
+6s          Normal   Pulled                   pod/openhands-app-2024                Container image "docker.all-hands.dev/all-hands-ai/sandbox:main" already present on machine
+5s          Normal   Created                  pod/openhands-app-2024                Created container openhands-sandbox-2024
+5s          Normal   Started                  pod/openhands-app-2024                Started container openhands-sandbox-2024
+83s         Normal   WaitForFirstConsumer     persistentvolumeclaim/workspace-pvc   waiting for first consumer to be created before binding
+27s         Normal   Provisioning             persistentvolumeclaim/workspace-pvc   External provisioner is provisioning volume for claim "openhands/workspace-pvc"
+17s         Normal   ProvisioningSucceeded    persistentvolumeclaim/workspace-pvc   Successfully provisioned volume pvc-31f15b25-faad-4665-a25f-201a530379af
+
+$ oc get pods
+NAME                 READY   STATUS    RESTARTS   AGE
+openhands-app-2024   2/2     Running   0          23s
+
+$ oc get pvc
+NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     AGE
+docker-pvc      Bound    pvc-2b1d223a-1c8f-4990-8e3d-68061a9ae252   10Gi       RWO            hcloud-volumes   10m
+workspace-pvc   Bound    pvc-31f15b25-faad-4665-a25f-201a530379af   10Gi       RWO            hcloud-volumes   13m
+
+```
+
+4. Create a NodePort service.
+Sample service creation command below:
+
+```bash
+# create the service of type NodePort
+$ oc create svc nodeport  openhands-app-2024  --tcp=3000:3000
+service/openhands-app-2024 created
+
+# review
+
+$ oc get svc
+NAME                 TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
+openhands-app-2024   NodePort   172.30.225.42   <none>        3000:30495/TCP   4s
+
+$ oc describe svc openhands-app-2024
+Name:                     openhands-app-2024
+Namespace:                openhands
+Labels:                   app=openhands-app-2024
+Annotations:              <none>
+Selector:                 app=openhands-app-2024
+Type:                     NodePort
+IP Family Policy:         SingleStack
+IP Families:              IPv4
+IP:                       172.30.225.42
+IPs:                      172.30.225.42
+Port:                     3000-3000  3000/TCP
+TargetPort:               3000/TCP
+NodePort:                 3000-3000  30495/TCP
+Endpoints:                10.128.2.48:3000
+Session Affinity:         None
+External Traffic Policy:  Cluster
+Events:                   <none>
+```
+
+6. Connect to OpenHands UI, configure the Agent, then test:
+
+![image](https://github.com/user-attachments/assets/12f94804-a0c7-4744-b873-e003c9caf40e)
+
+
+
+## GCP GKE Openhands deployment
+
+**Warning**: this deployment grants the OpenHands application access to the Kubernetes docker socket, which creates security risk. Use at your own discretion.
+1- Create policy for privillege access
+2- Create gke credentials(optional)
+3- Create openhands deployment
+4- Verification and ui access commands
+5- Tshoot pod to verify the internal container
+
+1. create policy for privillege access
+```bash
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: privileged-role
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: privileged-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: privileged-role
+subjects:
+- kind: ServiceAccount
+  name: default  # Change to your service account name
+  namespace: default
+```
+2. create gke credentials(optional)
+```bash
+kubectl create secret generic google-cloud-key \
+  --from-file=key.json=/path/to/your/google-cloud-key.json
+  ```
+3. create openhands deployment
+## as this is tested for the single worker node if you have multiple specify the flag for the single worker
+
+```bash
+kind: Deployment
+metadata:
+  name: openhands-app-2024
+  labels:
+    app: openhands-app-2024
+spec:
+  replicas: 1  # You can increase this number for multiple replicas
+  selector:
+    matchLabels:
+      app: openhands-app-2024
+  template:
+    metadata:
+      labels:
+        app: openhands-app-2024
+    spec:
+      containers:
+      - name: openhands-app-2024
+        image: docker.all-hands.dev/all-hands-ai/openhands:main
+        env:
+        - name: SANDBOX_USER_ID
+          value: "1000"
+        - name: SANDBOX_API_HOSTNAME
+          value: '10.164.0.4'
+        - name: WORKSPACE_MOUNT_PATH
+          value: "/tmp/workspace_base"
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: "/tmp/workspace_base/google-cloud-key.json"
+        volumeMounts:
+        - name: workspace-volume
+          mountPath: /tmp/workspace_base
+        - name: docker-sock
+          mountPath: /var/run/docker.sock
+        - name: google-credentials
+          mountPath: "/tmp/workspace_base/google-cloud-key.json"
+        securityContext:
+          privileged: true  # Add this to allow privileged access
+        ports:
+        - containerPort: 3000
+      - name: openhands-sandbox-2024
+        image: docker.all-hands.dev/all-hands-ai/runtime:main
+    #    securityContext:
+    #      privileged: true  # Add this to allow privileged access
+        ports:
+        - containerPort: 51963
+        command: ["/usr/sbin/sshd", "-D", "-p 51963", "-o", "PermitRootLogin=yes"]
+      volumes:
+      #- name: workspace-volume
+      #  persistentVolumeClaim:
+      #    claimName: workspace-pvc
+      - name: workspace-volume
+        emptyDir: {}
+      - name: docker-sock
+        hostPath:
+          path: /var/run/docker.sock       # Use host's Docker socket
+          type: Socket
+      - name: google-credentials
+        secret:
+          secretName: google-cloud-key
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openhands-app-2024-svc
+spec:
+  selector:
+    app: openhands-app-2024
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 3000
+  - name: ssh
+    protocol: TCP
+    port: 51963
+    targetPort: 51963
+  type: LoadBalancer
+  ```
+
+5. Tshoot pod to verify the internal container
+### if you want to know more regarding the internal container runtime use below mention pod deployment use kubectl exec -it to enter into container and you can check the contaienr run time using normal docker commands like "docker ps -a"
+
+```bash
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: docker-in-docker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: docker-in-docker
+  template:
+    metadata:
+      labels:
+        app: docker-in-docker
+    spec:
+      containers:
+      - name: dind
+        image: docker:20.10-dind
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: docker-sock
+          mountPath: /var/run/docker.sock
+      volumes:
+      - name: docker-sock
+        hostPath:
+          path: /var/run/docker.sock
+          type: Socket
+```

docs/modules/usage/how-to/persist-session-data.md

@@ -1,16 +0,0 @@
-# Persisting Session Data
-
-Using the standard installation, the session data is stored in memory. Currently, if OpenHands' service is restarted,
-previous sessions become invalid (a new secret is generated) and thus not recoverable.
-
-## How to Persist Session Data
-
-### Development Workflow
-In the `config.toml` file, specify the following:
-```
-[core]
-...
-file_store="local"
-file_store_path="/absolute/path/to/openhands/cache/directory"
-jwt_secret="secretpass"
-```

docs/modules/usage/installation.mdx

@@ -11,16 +11,16 @@
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.16
+    docker.all-hands.dev/all-hands-ai/openhands:0.14
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/modules/usage/micro-agents.md

@@ -1,213 +0,0 @@
-# Micro-Agents
-
-OpenHands uses specialized micro-agents to handle specific tasks and contexts efficiently. These micro-agents are small, focused components that provide specialized behavior and knowledge for particular scenarios.
-
-## Overview
-
-Micro-agents are defined in markdown files under the `openhands/agenthub/codeact_agent/micro/` directory. Each micro-agent is configured with:
-
-- A unique name
-- The agent type (typically CodeActAgent)
-- Trigger keywords that activate the agent
-- Specific instructions and capabilities
-
-## Available Micro-Agents
-
-### GitHub Agent
-**File**: `github.md`
-**Triggers**: `github`, `git`
-
-The GitHub agent specializes in GitHub API interactions and repository management. It:
-- Has access to a `GITHUB_TOKEN` for API authentication
-- Follows strict guidelines for repository interactions
-- Handles branch management and pull requests
-- Uses the GitHub API instead of web browser interactions
-
-Key features:
-- Branch protection (prevents direct pushes to main/master)
-- Automated PR creation
-- Git configuration management
-- API-first approach for GitHub operations
-
-### NPM Agent
-**File**: `npm.md`
-**Triggers**: `npm`
-
-Specializes in handling npm package management with specific focus on:
-- Non-interactive shell operations
-- Automated confirmation handling using Unix 'yes' command
-- Package installation automation
-
-### Custom Micro-Agents
-
-You can create your own micro-agents by adding new markdown files to the micro-agents directory. Each file should follow this structure:
-
-```markdown
----
-name: agent_name
-agent: CodeActAgent
-triggers:
-- trigger_word1
-- trigger_word2
----
-
-Instructions and capabilities for the micro-agent...
-```
-
-## Best Practices
-
-When working with micro-agents:
-
-1. **Use Appropriate Triggers**: Ensure your commands include the relevant trigger words to activate the correct micro-agent
-2. **Follow Agent Guidelines**: Each agent has specific instructions and limitations - respect these for optimal results
-3. **API-First Approach**: When available, use API endpoints rather than web interfaces
-4. **Automation Friendly**: Design commands that work well in non-interactive environments
-
-## Integration
-
-Micro-agents are automatically integrated into OpenHands' workflow. They:
-- Monitor incoming commands for their trigger words
-- Activate when relevant triggers are detected
-- Apply their specialized knowledge and capabilities
-- Follow their specific guidelines and restrictions
-
-## Example Usage
-
-```bash
-# GitHub agent example
-git checkout -b feature-branch
-git commit -m "Add new feature"
-git push origin feature-branch
-
-# NPM agent example
-yes | npm install package-name
-```
-
-For more information about specific agents, refer to their individual documentation files in the micro-agents directory.
-
-## Contributing a Micro-Agent
-
-To contribute a new micro-agent to OpenHands, follow these guidelines:
-
-### 1. Planning Your Micro-Agent
-
-Before creating a micro-agent, consider:
-- What specific problem or use case will it address?
-- What unique capabilities or knowledge should it have?
-- What trigger words make sense for activating it?
-- What constraints or guidelines should it follow?
-
-### 2. File Structure
-
-Create a new markdown file in `openhands/agenthub/codeact_agent/micro/` with a descriptive name (e.g., `docker.md` for a Docker-focused agent).
-
-### 3. Required Components
-
-Your micro-agent file must include:
-
-1. **Front Matter**: YAML metadata at the start of the file:
-```markdown
----
-name: your_agent_name
-agent: CodeActAgent
-triggers:
-- trigger_word1
-- trigger_word2
----
-```
-
-2. **Instructions**: Clear, specific guidelines for the agent's behavior:
-```markdown
-You are responsible for [specific task/domain].
-
-Key responsibilities:
-1. [Responsibility 1]
-2. [Responsibility 2]
-
-Guidelines:
-- [Guideline 1]
-- [Guideline 2]
-
-Examples of usage:
-[Example 1]
-[Example 2]
-```
-
-### 4. Best Practices for Micro-Agent Development
-
-1. **Clear Scope**: Keep the agent focused on a specific domain or task
-2. **Explicit Instructions**: Provide clear, unambiguous guidelines
-3. **Useful Examples**: Include practical examples of common use cases
-4. **Safety First**: Include necessary warnings and constraints
-5. **Integration Awareness**: Consider how the agent interacts with other components
-
-### 5. Testing Your Micro-Agent
-
-Before submitting:
-1. Test the agent with various prompts
-2. Verify trigger words activate the agent correctly
-3. Ensure instructions are clear and comprehensive
-4. Check for potential conflicts with existing agents
-
-### 6. Example Implementation
-
-Here's a template for a new micro-agent:
-
-```markdown
----
-name: docker
-agent: CodeActAgent
-triggers:
-- docker
-- container
----
-
-You are responsible for Docker container management and Dockerfile creation.
-
-Key responsibilities:
-1. Create and modify Dockerfiles
-2. Manage container lifecycle
-3. Handle Docker Compose configurations
-
-Guidelines:
-- Always use official base images when possible
-- Include necessary security considerations
-- Follow Docker best practices for layer optimization
-
-Examples:
-1. Creating a Dockerfile:
-   ```dockerfile
-   FROM node:18-alpine
-   WORKDIR /app
-   COPY package*.json ./
-   RUN npm install
-   COPY . .
-   CMD ["npm", "start"]
-   ```
-
-2. Docker Compose usage:
-   ```yaml
-   version: '3'
-   services:
-     web:
-       build: .
-       ports:
-         - "3000:3000"
-   ```
-
-Remember to:
-- Validate Dockerfile syntax
-- Check for security vulnerabilities
-- Optimize for build time and image size
-```
-
-### 7. Submission Process
-
-1. Create your micro-agent file in the correct directory
-2. Test thoroughly
-3. Submit a pull request with:
-   - The new micro-agent file
-   - Updated documentation if needed
-   - Description of the agent's purpose and capabilities
-
-Remember that micro-agents are a powerful way to extend OpenHands' capabilities in specific domains. Well-designed agents can significantly improve the system's ability to handle specialized tasks.

docs/modules/usage/prompting-best-practices.md

@@ -2,11 +2,6 @@
 
 When working with OpenHands AI software developer, it's crucial to provide clear and effective prompts. This guide outlines best practices for creating prompts that will yield the most accurate and useful responses.
 
-## Table of Contents
-
-- [Characteristics of Good Prompts](#characteristics-of-good-prompts)
-- [Customizing Prompts for your Project](#customizing-prompts-for-your-project)
-
 ## Characteristics of Good Prompts
 
 Good prompts are:
@@ -44,63 +39,3 @@ Good prompts are:
 Remember, the more precise and informative your prompt is, the better the AI can assist you in developing or modifying the OpenHands software.
 
 See [Getting Started with OpenHands](./getting-started) for more examples of helpful prompts.
-
-## Customizing Prompts for your Project
-
-OpenHands can be customized to work more effectively with specific repositories by providing repository-specific context and guidelines. This section explains how to optimize OpenHands for your project.
-
-### Repository Configuration
-
-You can customize OpenHands' behavior for your repository by creating a `.openhands_instructions` file in your repository's root directory. This file should contain:
-
-1. **Repository Overview**: A brief description of your project's purpose and architecture
-2. **Directory Structure**: Key directories and their purposes
-3. **Development Guidelines**: Project-specific coding standards and practices
-4. **Testing Requirements**: How to run tests and what types of tests are required
-5. **Setup Instructions**: Steps needed to build and run the project
-
-Example `.openhands_instructions` file:
-```
-Repository: MyProject
-Description: A web application for task management
-
-Directory Structure:
-- src/: Main application code
-- tests/: Test files
-- docs/: Documentation
-
-Setup:
-- Run `npm install` to install dependencies
-- Use `npm run dev` for development
-- Run `npm test` for testing
-
-Guidelines:
-- Follow ESLint configuration
-- Write tests for all new features
-- Use TypeScript for new code
-```
-
-### Customizing Prompts
-
-When working with a customized repository:
-
-1. **Reference Project Standards**: Mention specific coding standards or patterns used in your project
-2. **Include Context**: Reference relevant documentation or existing implementations
-3. **Specify Testing Requirements**: Include project-specific testing requirements in your prompts
-
-Example customized prompt:
-```
-Add a new task completion feature to src/components/TaskList.tsx following our existing component patterns.
-Include unit tests in tests/components/ and update the documentation in docs/features/.
-The component should use our shared styling from src/styles/components.
-```
-
-### Best Practices for Repository Customization
-
-1. **Keep Instructions Updated**: Regularly update your `.openhands_instructions` file as your project evolves
-2. **Be Specific**: Include specific paths, patterns, and requirements unique to your project
-3. **Document Dependencies**: List all tools and dependencies required for development
-4. **Include Examples**: Provide examples of good code patterns from your project
-5. **Specify Conventions**: Document naming conventions, file organization, and code style preferences
-
-By customizing OpenHands for your repository, you'll get more accurate and consistent results that align with your project's standards and requirements.

docs/modules/usage/runtimes.md

@@ -16,7 +16,7 @@ some flags being passed to `docker run` that make this possible:
 
 ```
 docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.16-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     # ...
 ```
@@ -28,22 +28,12 @@ You can also [build your own runtime image](how-to/custom-sandbox-guide).
 ### Connecting to Your filesystem
 One useful feature here is the ability to connect to your local filesystem.
 
-To mount your filesystem into the runtime, first set WORKSPACE_BASE:
+To mount your filesystem into the runtime, add the following options to
+the `docker run` command:
+
 ```bash
 export WORKSPACE_BASE=/path/to/your/code
 
-# Linux and Mac Example
-# export WORKSPACE_BASE=$HOME/OpenHands
-# Will set $WORKSPACE_BASE to /home/<username>/OpenHands
-#
-# WSL on Windows Example
-# export WORKSPACE_BASE=/mnt/c/dev/OpenHands
-# Will set $WORKSPACE_BASE to C:\dev\OpenHands
-```
-
-then add the following options to the `docker run` command:
-
-```bash
 docker run # ...
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \

docs/modules/usage/troubleshooting/troubleshooting.md

@@ -1,44 +1,180 @@
 # 🚧 Troubleshooting
 
+There are some error messages that frequently get reported by users.
+We'll try to make the install process easier, but for now you can look for your error message below and see if there are any workarounds.
+If you find more information or a workaround for one of these issues, please open a *PR* to add details to this file.
+
 :::tip
-OpenHands only supports Windows via WSL. Please be sure to run all commands inside your WSL terminal.
+OpenHands only supports Windows via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+Please be sure to run all commands inside your WSL terminal.
+Check out [Notes for WSL on Windows Users](troubleshooting/windows) for some troubleshooting guides.
 :::
 
-### Launch docker client failed
+## Common Issues
 
-**Description**
+* [Unable to connect to Docker](#unable-to-connect-to-docker)
+* [404 Resource not found](#404-resource-not-found)
+* [`make build` getting stuck on package installations](#make-build-getting-stuck-on-package-installations)
+* [Sessions are not restored](#sessions-are-not-restored)
+* [Connection to host.docker.internal timed out](#connection-to-host-docker-internal-timed-out)
 
-When running OpenHands, the following error is seen:
-```
-Launch docker client failed. Please make sure you have installed docker and started docker desktop/daemon.
+### Unable to connect to Docker
+
+[GitHub Issue](https://github.com/All-Hands-AI/OpenHands/issues/1226)
+
+**Symptoms**
+
+```bash
+Error creating controller. Please check Docker is running and visit `https://docs.all-hands.dev/modules/usage/troubleshooting` for more debugging information.
 ```
 
-**Resolution**
+```bash
+docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))
+```
+
+**Details**
+
+OpenHands uses a Docker container to do its work safely, without potentially breaking your machine.
+
+**Workarounds**
+
+* Run `docker ps` to ensure that docker is running
+* Make sure you don't need `sudo` to run docker [see here](https://www.baeldung.com/linux/docker-run-without-sudo)
+* If you are on a Mac, check the [permissions requirements](https://docs.docker.com/desktop/mac/permission-requirements/) and in particular consider enabling the `Allow the default Docker socket to be used` under `Settings > Advanced` in Docker Desktop.
+* In addition, upgrade your Docker to the latest version under `Check for Updates`
 
-Try these in order:
-* Confirm `docker` is running on your system. You should be able to run `docker ps` in the terminal successfully.
-* If using Docker Desktop, ensure `Settings > Advanced > Allow the default Docker socket to be used` is enabled.
-* Depending on your configuration you may need `Settings > Resources > Network > Enable host networking` enabled in Docker Desktop.
-* Reinstall Docker Desktop.
 ---
+### `404 Resource not found`
 
-# Development Workflow Specific
-### Error building runtime docker image
+**Symptoms**
 
-**Description**
-
-Attempts to start a new session fail, and errors with terms like the following appear in the logs:
-```
-debian-security bookworm-security
-InRelease At least one invalid signature was encountered.
+```python
+Traceback (most recent call last):
+  File "/app/.venv/lib/python3.12/site-packages/litellm/llms/openai.py", line 414, in completion
+    raise e
+  File "/app/.venv/lib/python3.12/site-packages/litellm/llms/openai.py", line 373, in completion
+    response = openai_client.chat.completions.create(**data, timeout=timeout)  # type: ignore
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/app/.venv/lib/python3.12/site-packages/openai/_utils/_utils.py", line 277, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/app/.venv/lib/python3.12/site-packages/openai/resources/chat/completions.py", line 579, in create
+    return self._post(
+           ^^^^^^^^^^^
+  File "/app/.venv/lib/python3.12/site-packages/openai/_base_client.py", line 1232, in post
+    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/app/.venv/lib/python3.12/site-packages/openai/_base_client.py", line 921, in request
+    return self._request(
+           ^^^^^^^^^^^^^^
+  File "/app/.venv/lib/python3.12/site-packages/openai/_base_client.py", line 1012, in _request
+    raise self._make_status_error_from_response(err.response) from None
+openai.NotFoundError: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}
 ```
 
-This seems to happen when the hash of an existing external library changes and your local docker instance has
-cached a previous version. To work around this, please try the following:
+**Details**
 
-* Stop any containers where the name has the prefix `openhands-runtime-` :
-  `docker ps --filter name=openhands-runtime- --filter status=running -aq | xargs docker stop`
-* Remove any containers where the name has the prefix `openhands-runtime-` :
-  `docker rmi $(docker images --filter name=openhands-runtime- -q --no-trunc)`
-* Stop and Remove any containers / images where the name has the prefix `openhands-runtime-`
-* Prune containers / images : `docker container prune -f && docker image prune -f`
+This happens when LiteLLM (our library for connecting to different LLM providers) can't find
+the API endpoint you're trying to connect to. Most often this happens for Azure or ollama users.
+
+**Workarounds**
+
+* Check that you've set `LLM_BASE_URL` properly
+* Check that the model is set properly, based on the [LiteLLM docs](https://docs.litellm.ai/docs/providers)
+  * If you're running inside the UI, be sure to set the `model` in the settings modal
+  * If you're running headless (via main.py) be sure to set `LLM_MODEL` in your env/config
+* Make sure you've followed any special instructions for your LLM provider
+  * [Azure](/modules/usage/llms/azure-llms)
+  * [Google](/modules/usage/llms/google-llms)
+* Make sure your API key is correct
+* See if you can connect to the LLM using `curl`
+* Try [connecting via LiteLLM directly](https://github.com/BerriAI/litellm) to test your setup
+
+---
+### `make build` getting stuck on package installations
+
+**Symptoms**
+
+Package installation stuck on `Pending...` without any error message:
+
+```bash
+Package operations: 286 installs, 0 updates, 0 removals
+
+  - Installing certifi (2024.2.2): Pending...
+  - Installing h11 (0.14.0): Pending...
+  - Installing idna (3.7): Pending...
+  - Installing sniffio (1.3.1): Pending...
+  - Installing typing-extensions (4.11.0): Pending...
+```
+
+**Details**
+
+In rare cases, `make build` can seemingly get stuck on package installations
+without any error message.
+
+**Workarounds**
+
+The package installer Poetry may miss a configuration setting for where credentials are to be looked up (keyring).
+
+First check with `env` if a value for `PYTHON_KEYRING_BACKEND` exists.
+If not, run the below command to set it to a known value and retry the build:
+
+```bash
+export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
+```
+
+---
+### Sessions are not restored
+
+**Symptoms**
+
+OpenHands usually asks whether to resume or start a new session when opening the UI.
+But clicking "Resume" still starts a fresh new chat.
+
+**Details**
+
+With a standard installation as of today session data is stored in memory.
+Currently, if OpenHands's service is restarted, previous sessions become
+invalid (a new secret is generated) and thus not recoverable.
+
+**Workarounds**
+
+* Change configuration to make sessions persistent by editing the `config.toml`
+file (in OpenHands's root folder) by specifying a `file_store` and an
+absolute `file_store_path`:
+
+```toml
+file_store="local"
+file_store_path="/absolute/path/to/openhands/cache/directory"
+```
+
+* Add a fixed jwt secret in your .bashrc, like below, so that previous session id's
+should stay accepted.
+
+```bash
+EXPORT JWT_SECRET=A_CONST_VALUE
+```
+
+---
+### Connection to host docker internal timed out
+
+**Symptoms**
+
+When you start the server using the docker command from the main [README](https://github.com/All-Hands-AI/OpenHands/README.md), you get a long timeout
+followed by the a stack trace containing messages like:
+
+* `Connection to host.docker.internal timed out. (connect timeout=310)`
+* `Max retries exceeded with url: /alive`
+
+**Details**
+
+If Docker Engine is installed rather than Docker Desktop, the main command will not work as expected.
+Docker Desktop includes easy DNS configuration for connecting processes running in different containers
+which OpenHands makes use of when the main server is running inside a docker container.
+(Further details: https://forums.docker.com/t/difference-between-docker-desktop-and-docker-engine/124612)
+
+**Workarounds**
+
+* [Install Docker Desktop](https://www.docker.com/products/docker-desktop/)
+* Run OpenHands in [Development Mode](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
+  So that the main server is not run inside a container, but still creates dockerized runtime sandboxes.

docs/modules/usage/troubleshooting/windows.md

@@ -0,0 +1,64 @@
+# Notes for WSL on Windows Users
+
+OpenHands only supports Windows via [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
+Please be sure to run all commands inside your WSL terminal.
+
+## Troubleshooting
+
+### Recommendation: Do not run as root user
+
+For security reasons, it is highly recommended to not run OpenHands as the root user, but a user with a non-zero UID.
+
+References:
+
+* [Why it is bad to login as root](https://askubuntu.com/questions/16178/why-is-it-bad-to-log-in-as-root)
+* [Set default user in WSL](https://www.tenforums.com/tutorials/128152-set-default-user-windows-subsystem-linux-distro-windows-10-a.html#option2)
+Hint about the 2nd reference: for Ubuntu users, the command could actually be "ubuntupreview" instead of "ubuntu".
+
+---
+### Error: 'docker' could not be found in this WSL 2 distro.
+
+If you are using Docker Desktop, make sure to start it before calling any docker command from inside WSL.
+Docker also needs to have the WSL integration option activated.
+
+---
+### Poetry Installation
+
+* If you face issues running Poetry even after installing it during the build process, you may need to add its binary path to your environment:
+
+```sh
+export PATH="$HOME/.local/bin:$PATH"
+```
+
+* If make build stops on an error like this:
+
+```sh
+ModuleNotFoundError: no module named <module-name>
+```
+
+This could be an issue with Poetry's cache.
+Try to run these 2 commands after another:
+
+```sh
+rm -r ~/.cache/pypoetry
+make build
+```
+
+---
+### NoneType object has no attribute 'request'
+
+If you are experiencing issues related to networking, such as `NoneType object has no attribute 'request'` when executing `make run`, you may need to configure your WSL2 networking settings. Follow these steps:
+
+* Open or create the `.wslconfig` file located at `C:\Users\%username%\.wslconfig` on your Windows host machine.
+* Add the following configuration to the `.wslconfig` file:
+
+```sh
+[wsl2]
+networkingMode=mirrored
+localhostForwarding=true
+```
+
+* Save the `.wslconfig` file.
+* Restart WSL2 completely by exiting any running WSL2 instances and executing the command `wsl --shutdown` in your command prompt or terminal.
+* After restarting WSL, attempt to execute `make run` again.
+The networking issue should be resolved.

docs/modules/usage/upgrade-guide.md

@@ -0,0 +1,71 @@
+# ⬆️ Upgrade Guide
+
+## 0.8.0 (2024-07-13)
+
+### Config breaking changes
+
+In this release we introduced a few breaking changes to backend configurations.
+If you have only been using OpenHands via frontend (web GUI), nothing needs
+to be taken care of.
+
+Here's a list of breaking changes in configs. They only apply to users who
+use OpenHands CLI via `main.py`. For more detail, see [#2756](https://github.com/All-Hands-AI/OpenHands/pull/2756).
+
+#### Removal of --model-name option from main.py
+
+Please note that `--model-name`, or `-m` option, no longer exists. You should set up the LLM
+configs in `config.toml` or via environmental variables.
+
+#### LLM config groups must be subgroups of 'llm'
+
+Prior to release 0.8, you can use arbitrary name for llm config in `config.toml`, e.g.
+
+```toml
+[gpt-4o]
+model="gpt-4o"
+api_key="<your_api_key>"
+```
+
+and then use `--llm-config` CLI argument to specify the desired LLM config group
+by name. This no longer works. Instead, the config group must be under `llm` group,
+e.g.:
+
+```toml
+[llm.gpt-4o]
+model="gpt-4o"
+api_key="<your_api_key>"
+```
+
+If you have a config group named `llm`, no need to change it, it will be used
+as the default LLM config group.
+
+#### 'agent' group no longer contains 'name' field
+
+Prior to release 0.8, you may or may not have a config group named `agent` that
+looks like this:
+
+```toml
+[agent]
+name="CodeActAgent"
+memory_max_threads=2
+```
+
+Note the `name` field is now removed. Instead, you should put `default_agent` field
+under `core` group, e.g.
+
+```toml
+[core]
+# other configs
+default_agent='CodeActAgent'
+
+[agent]
+llm_config='llm'
+memory_max_threads=2
+
+[agent.CodeActAgent]
+llm_config='gpt-4o'
+```
+
+Note that similar to `llm` subgroups, you can also define `agent` subgroups.
+Moreover, an agent can be associated with a specific LLM config group. For more
+detail, see the examples in `config.template.toml`.

docs/package.json

@@ -15,23 +15,23 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.6.3",
-    "@docusaurus/plugin-content-pages": "^3.6.3",
-    "@docusaurus/preset-classic": "^3.6.3",
-    "@docusaurus/theme-mermaid": "^3.6.3",
+    "@docusaurus/core": "^3.6.0",
+    "@docusaurus/plugin-content-pages": "^3.6.0",
+    "@docusaurus/preset-classic": "^3.6.0",
+    "@docusaurus/theme-mermaid": "^3.6.0",
     "@mdx-js/react": "^3.1.0",
     "clsx": "^2.0.0",
-    "prism-react-renderer": "^2.4.1",
+    "prism-react-renderer": "^2.4.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
-    "react-icons": "^5.4.0",
-    "react-use": "^17.6.0"
+    "react-icons": "^5.3.0",
+    "react-use": "^17.5.1"
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.1",
-    "@docusaurus/tsconfig": "^3.6.3",
+    "@docusaurus/tsconfig": "^3.6.0",
     "@docusaurus/types": "^3.5.1",
-    "typescript": "~5.7.2"
+    "typescript": "~5.6.3"
   },
   "browserslist": {
     "production": [

docs/sidebars.ts

@@ -14,20 +14,9 @@ const sidebars: SidebarsConfig = {
       id: 'usage/getting-started',
     },
     {
-      type: 'category',
-      label: 'Prompting',
-      items: [
-        {
-          type: 'doc',
-          label: 'Best Practices',
-          id: 'usage/prompting-best-practices',
-        },
-        {
-          type: 'doc',
-          label: 'Micro-Agents',
-          id: 'usage/micro-agents',
-        },
-      ],
+      type: 'doc',
+      label: 'Prompting Best Practices',
+      id: 'usage/prompting-best-practices',
     },
     {
       type: 'category',
@@ -111,21 +100,11 @@ const sidebars: SidebarsConfig = {
           label: 'Runtime Configuration',
           id: 'usage/runtimes',
         },
-        {
-          type: 'doc',
-          label: 'Configuration Options',
-          id: 'usage/configuration-options',
-        },
         {
           type: 'doc',
           label: 'Custom Sandbox',
           id: 'usage/how-to/custom-sandbox-guide',
         },
-        {
-          type: 'doc',
-          label: 'Persist Session Data',
-          id: 'usage/how-to/persist-session-data',
-        },
       ],
     },
     {
@@ -168,6 +147,11 @@ const sidebars: SidebarsConfig = {
           label: 'Evaluation',
           id: 'usage/how-to/evaluation-harness',
         },
+        {
+          type: 'doc',
+          label: 'Kubernetes Deployment',
+          id: 'usage/how-to/openshift-example',
+        },
       ],
     },
     {

docs/src/components/CustomFooter.tsx

@@ -8,7 +8,7 @@ function CustomFooter() {
     <footer className="custom-footer">
       <div className="footer-content">
         <div className="footer-icons">
-          <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw" target="_blank" rel="noopener noreferrer">
+          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA" target="_blank" rel="noopener noreferrer">
             <FaSlack />
           </a>
           <a href="https://discord.gg/ESHStjSjD4" target="_blank" rel="noopener noreferrer">

docs/src/components/HomepageHeader/HomepageHeader.tsx

@@ -23,7 +23,7 @@ export function HomepageHeader() {
           <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" /></a>
           <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License" /></a>
           <br/>
-          <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2wkh4pklz-w~h_DVDtEe9H5kyQlcNxVw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
+          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
           <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community" /></a>
           <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits" /></a>
           <br/>

evaluation/EDA/README.md

@@ -4,13 +4,15 @@ This folder contains evaluation harness for evaluating agents on the Entity-dedu
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+
 
 ## Start the evaluation
 
+
 ```bash
 export OPENAI_API_KEY="sk-XXX"; # This is required for evaluation (to simulate another party of conversation)
-./evaluation/benchmarks/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
+./evaluation/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
 ```
 
 where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@@ -31,12 +33,11 @@ to `CodeActAgent`.
 For example,
 
 ```bash
-./evaluation/benchmarks/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
+./evaluation/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
 ```
 
 ## Reference
-
-```bibtex
+```
 @inproceedings{zhang2023entity,
   title={Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games},
   author={Zhang, Yizhe and Lu, Jiarui and Jaitly, Navdeep},

evaluation/EDA/run_infer.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.benchmarks.EDA.game import Q20Game, Q20GameCelebrity
+from evaluation.EDA.game import Q20Game, Q20GameCelebrity
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
@@ -202,9 +202,6 @@ if __name__ == '__main__':
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
-        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
-        llm_config.modify_params = False
-
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/EDA/scripts/run_infer.sh

@@ -21,7 +21,7 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_openhands_version
+get_agent_version
 
 if [ -z "$DATASET" ]; then
   echo "Dataset not specified, use default 'things'"
@@ -34,13 +34,16 @@ if [ -z "$OPENAI_API_KEY" ]; then
   exit 1
 fi
 
+# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenHands
+# We need to track the version of Agent in the evaluation to make sure results are comparable
+AGENT_VERSION=v$(poetry run python -c "import openhands.agenthub; from openhands.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
 
 echo "AGENT: $AGENT"
-echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "AGENT_VERSION: $AGENT_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 echo "DATASET: $DATASET"
 
-COMMAND="poetry run python evaluation/benchmarks/EDA/run_infer.py \
+COMMAND="poetry run python evaluation/EDA/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --dataset $DATASET \
@@ -48,7 +51,7 @@ COMMAND="poetry run python evaluation/benchmarks/EDA/run_infer.py \
   --max-iterations 20 \
   --OPENAI_API_KEY $OPENAI_API_KEY \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note ${OPENHANDS_VERSION}_${DATASET}"
+  --eval-note ${AGENT_VERSION}_${DATASET}"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/README.md

@@ -6,9 +6,9 @@ This folder contains code and resources to run experiments and evaluations.
 
 ### Setup
 
-Before starting evaluation, follow the instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
+Before starting evaluation, follow the instructions here [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
 
-Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the [evaluation directory](#supported-benchmarks).
+Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the evaluation directory.
 Generally these will involve running `run_infer.py` to perform inference with the agents.
 
 ### Implementing and Evaluating an Agent
@@ -42,40 +42,32 @@ temperature = 0.0
 
 ## Supported Benchmarks
 
-The OpenHands evaluation harness supports a wide variety of benchmarks across [software engineering](#software-engineering), [web browsing](#web-browsing), [miscellaneous assistance](#misc-assistance), and [real-world](#real-world) tasks.
+The OpenHands evaluation harness supports a wide variety of benchmarks across software engineering, web browsing, and miscellaneous assistance tasks.
 
 ### Software Engineering
 
-- SWE-Bench: [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench)
-- HumanEvalFix: [`evaluation/benchmarks/humanevalfix`](./benchmarks/humanevalfix)
-- BIRD: [`evaluation/benchmarks/bird`](./benchmarks/bird)
-- BioCoder: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
-- ML-Bench: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
-- APIBench: [`evaluation/benchmarks/gorilla`](./benchmarks/gorilla/)
-- ToolQA: [`evaluation/benchmarks/toolqa`](./benchmarks/toolqa/)
-- AiderBench: [`evaluation/benchmarks/aider_bench`](./benchmarks/aider_bench/)
-- Commit0: [`evaluation/benchmarks/commit0_bench`](./benchmarks/commit0_bench/)
-- DiscoveryBench: [`evaluation/benchmarks/discoverybench`](./benchmarks/discoverybench/)
+- SWE-Bench: [`evaluation/swe_bench`](./swe_bench)
+- HumanEvalFix: [`evaluation/humanevalfix`](./humanevalfix)
+- BIRD: [`evaluation/bird`](./bird)
+- BioCoder: [`evaluation/ml_bench`](./ml_bench)
+- ML-Bench: [`evaluation/ml_bench`](./ml_bench)
+- APIBench: [`evaluation/gorilla`](./gorilla/)
+- ToolQA: [`evaluation/toolqa`](./toolqa/)
+- AiderBench: [`evaluation/aider_bench`](./aider_bench/)
 
 ### Web Browsing
 
-- WebArena: [`evaluation/benchmarks/webarena`](./benchmarks/webarena/)
-- MiniWob++: [`evaluation/benchmarks/miniwob`](./benchmarks/miniwob/)
-- Browsing Delegation: [`evaluation/benchmarks/browsing_delegation`](./benchmarks/browsing_delegation/)
+- WebArena: [`evaluation/webarena`](./webarena/)
+- MiniWob++: [`evaluation/miniwob`](./miniwob/)
 
 ### Misc. Assistance
 
-- GAIA: [`evaluation/benchmarks/gaia`](./benchmarks/gaia)
-- GPQA: [`evaluation/benchmarks/gpqa`](./benchmarks/gpqa)
-- AgentBench: [`evaluation/benchmarks/agent_bench`](./benchmarks/agent_bench)
-- MINT: [`evaluation/benchmarks/mint`](./benchmarks/mint)
-- Entity deduction Arena (EDA): [`evaluation/benchmarks/EDA`](./benchmarks/EDA)
-- ProofWriter: [`evaluation/benchmarks/logic_reasoning`](./benchmarks/logic_reasoning)
-- ScienceAgentBench: [`evaluation/benchmarks/scienceagentbench`](./benchmarks/scienceagentbench)
-
-### Real World
-
-- TheAgentCompany: [`evaluation/benchmarks/the_agent_company`](./benchmarks/the_agent_company)
+- GAIA: [`evaluation/gaia`](./gaia)
+- GPQA: [`evaluation/gpqa`](./gpqa)
+- AgentBench: [`evaluation/agent_bench`](./agent_bench)
+- MINT: [`evaluation/mint`](./mint)
+- Entity deduction Arena (EDA): [`evaluation/EDA`](./EDA)
+- ProofWriter: [`evaluation/logic_reasoning`](./logic_reasoning)
 
 ## Result Visualization
 
@@ -87,7 +79,7 @@ You can start your own fork of [our huggingface evaluation outputs](https://hugg
 
 To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/how-to/evaluation-harness). Briefly,
 
-- Each subfolder contains a specific benchmark or experiment. For example, [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench) should contain
+- Each subfolder contains a specific benchmark or experiment. For example, `evaluation/swe_bench` should contain
 all the preprocessing/evaluation/analysis scripts.
 - Raw data and experimental records should not be stored within this repo.
 - For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.

evaluation/agent_bench/README.md

@@ -4,12 +4,12 @@ This folder contains evaluation harness for evaluating agents on the [AgentBench
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
 
 ## Start the evaluation
 
 ```bash
-./evaluation/benchmarks/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
 ```
 
 - `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
@@ -25,7 +25,7 @@ in order to use `eval_limit`, you must also set `agent`.
 
 Following is the basic command to start the evaluation.
 
-You can update the arguments in the script `evaluation/benchmarks/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
+You can update the arguments in the script `evaluation/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
 
 - `--agent-cls`, the agent to use. For example, `CodeActAgent`.
 - `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
@@ -34,23 +34,5 @@ You can update the arguments in the script `evaluation/benchmarks/agent_bench/sc
 - `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
 
 ```bash
-./evaluation/benchmarks/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
+./evaluation/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
 ```
-
-## Run with Remote Runtime (experimental)
-
-You can run the evaluation using a remote runtime instead of a local Docker container. This is useful when you want to run the evaluation in a cloud environment or when you don't have Docker installed locally.
-
-To use the remote runtime, set the following environment variables:
-
-```bash
-# Required environment variables
-export ALLHANDS_API_KEY="your-api-key"  # Contact the team to get an API key
-export RUNTIME=remote
-export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
-
-# Run the evaluation
-./evaluation/benchmarks/agent_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 1
-```
-
-The remote runtime will build a container image and run the evaluation in a cloud environment. The results will be saved locally in the same way as when running with a local runtime.

evaluation/agent_bench/run_infer.py

@@ -7,7 +7,7 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.benchmarks.agent_bench.helper import (
+from evaluation.agent_bench.helper import (
     FAKE_RESPONSES,
     INST_SUFFIXES,
     compare_results,
@@ -43,16 +43,12 @@ def get_config(
     config = AppConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
-        runtime=os.environ.get('RUNTIME', 'eventstream'),
+        runtime='eventstream',
         max_iterations=metadata.max_iterations,
         sandbox=SandboxConfig(
-            base_container_image='python:3.12-slim',
+            base_container_image='python:3.12-bookworm',
             enable_auto_lint=True,
             use_host_network=False,
-            api_key=os.environ.get('ALLHANDS_API_KEY', None),
-            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_runtime_alive=False,
-            remote_runtime_init_timeout=3600,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -307,8 +303,6 @@ if __name__ == '__main__':
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
-        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
-        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/agent_bench/scripts/run_infer.sh

@@ -20,18 +20,18 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_openhands_version
+get_agent_version
 
 echo "AGENT: $AGENT"
-echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "AGENT_VERSION: $AGENT_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-COMMAND="export PYTHONPATH=evaluation/benchmarks/agent_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/agent_bench/run_infer.py \
+COMMAND="export PYTHONPATH=evaluation/agent_bench:\$PYTHONPATH && poetry run python evaluation/agent_bench/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 30 \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note $OPENHANDS_VERSION"
+  --eval-note $AGENT_VERSION"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/aider_bench/README.md

@@ -10,13 +10,13 @@ Hugging Face dataset based on the
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../../README.md#setup) to setup your local
+Please follow instruction [here](../README.md#setup) to setup your local
 development environment and LLM.
 
 ## Start the evaluation
 
 ```bash
-./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
 ```
 
 - `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
@@ -42,7 +42,7 @@ export SKIP_NUM=12 # skip the first 12 instances from the dataset
 Following is the basic command to start the evaluation.
 
 You can update the arguments in the script
-`evaluation/benchmarks/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
+`evaluation/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
 `--eval-num-workers` and so on:
 
 - `--agent-cls`, the agent to use. For example, `CodeActAgent`.
@@ -53,7 +53,7 @@ You can update the arguments in the script
 - `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`.
 
 ```bash
-./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
+./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
 ```
 
 ### Run Inference on `RemoteRuntime` (experimental)
@@ -61,25 +61,25 @@ You can update the arguments in the script
 This is in limited beta. Contact Xingyao over slack if you want to try this out!
 
 ```bash
-./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
 
 # Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
 export ALLHANDS_API_KEY="YOUR-API-KEY"
 export RUNTIME=remote
 export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
-./evaluation/benchmarks/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
+./evaluation/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
 ```
 
 ## Summarize Results
 
 ```bash
-poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
+poetry run python ./evaluation/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
 ```
 
 Full example:
 
 ```bash
-poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
+poetry run python ./evaluation/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
 ```
 
 This will list the instances that passed and the instances that failed. For each

evaluation/aider_bench/run_infer.py

@@ -7,7 +7,7 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.benchmarks.aider_bench.helper import (
+from evaluation.aider_bench.helper import (
     FAKE_RESPONSES,
     INST_SUFFIXES,
     INSTRUCTIONS_ADDENDUM,
@@ -279,8 +279,6 @@ if __name__ == '__main__':
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
-        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
-        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/aider_bench/scripts/run_infer.sh

@@ -21,13 +21,13 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_openhands_version
+get_agent_version
 
 echo "AGENT: $AGENT"
-echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "AGENT_VERSION: $AGENT_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-EVAL_NOTE=$OPENHANDS_VERSION
+EVAL_NOTE=$AGENT_VERSION
 
 # Default to NOT use unit tests.
 if [ -z "$USE_UNIT_TESTS" ]; then
@@ -39,7 +39,7 @@ if [ "$USE_UNIT_TESTS" = true ]; then
   EVAL_NOTE=$EVAL_NOTE-w-test
 fi
 
-COMMAND="export PYTHONPATH=evaluation/benchmarks/aider_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/aider_bench/run_infer.py \
+COMMAND="export PYTHONPATH=evaluation/aider_bench:\$PYTHONPATH && poetry run python evaluation/aider_bench/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 30 \

evaluation/benchmarks/commit0_bench/README.md

@@ -1,81 +0,0 @@
-# Commit0 Evaluation with OpenHands
-
-This folder contains the evaluation harness that we built on top of the original [Commit0](https://commit-0.github.io/) ([paper](TBD)).
-
-The evaluation consists of three steps:
-
-1. Environment setup: [install python environment](../../README.md#development-environment), [configure LLM config](../../README.md#configure-openhands-and-your-llm).
-2. [Run Evaluation](#run-inference-on-commit0-instances): Generate a edit patch for each Commit0 Repo, and get the evaluation results
-
-## Setup Environment and LLM Configuration
-
-Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
-
-## OpenHands Commit0 Instance-level Docker Support
-
-OpenHands supports using the Commit0 Docker for **[inference](#run-inference-on-commit0-instances).
-This is now the default behavior.
-
-## Run Inference on Commit0 Instances
-
-Make sure your Docker daemon is running, and you have ample disk space (at least 200-500GB, depends on the Commit0 set you are running on) for the [instance-level docker image](#openhands-commit0-instance-level-docker-support).
-
-When the `run_infer.sh` script is started, it will automatically pull the `lite` split in Commit0. For example, for instance ID `commit-0/minitorch`, it will try to pull our pre-build docker image `wentingzhao/minitorch` from DockerHub. This image will be used create an OpenHands runtime image where the agent will operate on.
-
-```bash
-./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
-
-# Example
-./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 16 100 8 wentingzhao/commit0_combined test
-```
-
-where `model_config` is mandatory, and the rest are optional.
-
-- `repo_split`, e.g. `lite`, is the split of the Commit0 dataset you would like to evaluate on. Available options are `lite`, `all` and each individual repo.
-- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`.
-- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
-like to evaluate. It could also be a release tag like `0.6.2`.
-- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
-default, the script evaluates the `lite` split of the Commit0 dataset (16 repos). Note:
-in order to use `eval_limit`, you must also set `agent`.
-- `max_iter`, e.g. `20`, is the maximum number of iterations for the agent to run. By
-default, it is set to 30.
-- `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
-default, it is set to 1.
-- `dataset`, a huggingface dataset name. e.g. `wentingzhao/commit0_combined`, specifies which dataset to evaluate on.
-- `dataset_split`, split for the huggingface dataset. Notice only `test` is supported for Commit0.
-
-Note that the `USE_INSTANCE_IMAGE` environment variable is always set to `true` for Commit0.
-
-Let's say you'd like to run 10 instances using `llm.eval_sonnet` and CodeActAgent,
-
-then your command would be:
-
-```bash
-./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
-```
-
-### Run Inference on `RemoteRuntime` (experimental)
-
-This is in limited beta. Contact Xingyao over slack if you want to try this out!
-
-```bash
-./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
-
-# Example - This runs evaluation on CodeActAgent for 10 instances on "wentingzhao/commit0_combined"'s test set, with max 30 iteration per instances, with 1 number of workers running in parallel
-ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev" EVAL_DOCKER_IMAGE_PREFIX="docker.io/wentingzhao" \
-./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
-```
-
-To clean-up all existing runtime you've already started, run:
-
-```bash
-ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/benchmarks/commit0_bench/scripts/cleanup_remote_runtime.sh
-```
-
-### Specify a subset of tasks to run infer
-
-If you would like to specify a list of tasks you'd like to benchmark on, you just need to pass selected repo through `repo_split` option.

evaluation/benchmarks/commit0_bench/run_infer.py

@@ -1,608 +0,0 @@
-import asyncio
-import json
-import os
-from collections import Counter
-from typing import Any
-
-import pandas as pd
-from commit0.harness.constants import SPLIT
-from datasets import load_dataset
-
-import openhands.agenthub
-from evaluation.utils.shared import (
-    EvalException,
-    EvalMetadata,
-    EvalOutput,
-    assert_and_raise,
-    codeact_user_response,
-    make_metadata,
-    prepare_dataset,
-    reset_logger_for_multiprocessing,
-    run_evaluation,
-    update_llm_config_for_completions_logging,
-)
-from openhands.controller.state.state import State
-from openhands.core.config import (
-    AgentConfig,
-    AppConfig,
-    SandboxConfig,
-    get_llm_config_arg,
-    get_parser,
-)
-from openhands.core.logger import openhands_logger as logger
-from openhands.core.main import create_runtime, run_controller
-from openhands.events.action import CmdRunAction, MessageAction
-from openhands.events.observation import CmdOutputObservation, ErrorObservation
-from openhands.events.serialization.event import event_to_dict
-from openhands.runtime.base import Runtime
-from openhands.utils.async_utils import call_async_from_sync
-from openhands.utils.shutdown_listener import sleep_if_should_continue
-
-USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
-USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
-RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-    'CodeActCommit0Agent': codeact_user_response,
-}
-
-
-def _get_commit0_workspace_dir_name(instance: pd.Series) -> str:
-    return instance['repo'].split('/')[1]
-
-
-def get_instruction(instance: pd.Series, metadata: EvalMetadata):
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-    # Prepare instruction
-    test_cmd = instance['test']['test_cmd']
-    test_dir = instance['test']['test_dir']
-    # Instruction based on Anthropic's official trajectory
-    # https://github.com/eschluntz/swe-bench-experiments/tree/main/evaluation/verified/20241022_tools_claude-3-5-sonnet-updated/trajs
-    instruction = (
-        '<uploaded_files>\n'
-        f'/workspace/{workspace_dir_name}\n'
-        '</uploaded_files>\n'
-        f"I've uploaded a python code repository in the directory {workspace_dir_name}. Here is your task:\n\n"
-        'Here is your task:\n\n'
-        '  You need to complete the implementations for all functions (i.e., those with pass\n'
-        '  statements) and pass the unit tests.\n\n'
-        '  Do not change the names of existing functions or classes, as they may be referenced\n'
-        '  from other code like unit tests, etc.\n\n'
-        '  When you generate code, you must maintain the original formatting of the function\n'
-        '  stubs (such as whitespaces), otherwise we will not able to search/replace blocks\n'
-        '  for code modifications, and therefore you will receive a score of 0 for your generated\n'
-        '  code.'
-        '\n\n'
-        'Here is the command to run the unit tests:\n'
-        '<test_command>\n'
-        f'{test_cmd} {test_dir}\n'
-        '</test_command>\n\n'
-        'Make a local git commit for each agent step for all code changes. If there is not change in current step, do not make a commit.'
-    )
-
-    if RUN_WITH_BROWSING:
-        instruction += (
-            '<IMPORTANT!>\n'
-            'You SHOULD NEVER attempt to browse the web. '
-            '</IMPORTANT!>\n'
-        )
-    return instruction
-
-
-# TODO: migrate all swe-bench docker to ghcr.io/openhands
-DOCKER_IMAGE_PREFIX = os.environ.get(
-    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/wentingzhao/'
-)
-logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
-
-
-def get_instance_docker_image(repo_name: str) -> str:
-    return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + repo_name).lower() + ':v0'
-
-
-def get_config(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-) -> AppConfig:
-    # COMMIT0_CONTAINER_IMAGE = 'wentingzhao/'
-    assert USE_INSTANCE_IMAGE
-    # We use a different instance image for the each instance of commit0 eval
-    repo_name = instance['repo'].split('/')[1]
-    base_container_image = get_instance_docker_image(repo_name)
-    logger.info(
-        f'Using instance container image: {base_container_image}. '
-        f'Please make sure this image exists. '
-        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
-    )
-    # else:
-    #     raise
-    # base_container_image = SWE_BENCH_CONTAINER_IMAGE
-    # logger.info(f'Using swe-bench container image: {base_container_image}')
-
-    config = AppConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
-        runtime=os.environ.get('RUNTIME', 'eventstream'),
-        sandbox=SandboxConfig(
-            base_container_image=base_container_image,
-            enable_auto_lint=True,
-            use_host_network=False,
-            # large enough timeout, since some testcases take very long to run
-            timeout=300,
-            api_key=os.environ.get('ALLHANDS_API_KEY', None),
-            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_runtime_alive=False,
-            remote_runtime_init_timeout=3600,
-        ),
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
-    config.set_llm_config(
-        update_llm_config_for_completions_logging(
-            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
-        )
-    )
-    agent_config = AgentConfig(
-        codeact_enable_jupyter=False,
-        codeact_enable_browsing=RUN_WITH_BROWSING,
-        codeact_enable_llm_editor=False,
-    )
-    config.set_agent_config(agent_config)
-    return config
-
-
-def initialize_runtime(
-    runtime: Runtime,
-    instance: pd.Series,  # this argument is not required
-):
-    """Initialize the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    """
-    logger.info('-' * 30)
-    logger.info('BEGIN Runtime Initialization Fn')
-    logger.info('-' * 30)
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-    obs: CmdOutputObservation
-
-    action = CmdRunAction(
-        command=f'git clone -b commit0_combined https://github.com/{instance["repo"]}.git'
-    )
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to git clone -b commit0_combined https://github.com/{instance["repo"]}.git: {str(obs)}',
-    )
-
-    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
-    )
-
-    action = CmdRunAction(command='git checkout -b openhands')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0, f'Failed to git checkout new branch openhands: {str(obs)}'
-    )
-
-    # Install commit0
-    action = CmdRunAction(command='/root/.cargo/bin/uv pip install commit0')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to install commit0: {str(obs)}',
-    )
-    logger.info('-' * 30)
-    logger.info('END Runtime Initialization Fn')
-    logger.info('-' * 30)
-
-
-def complete_runtime(
-    runtime: Runtime,
-    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
-) -> dict[str, Any]:
-    """Complete the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    If you need to do something in the sandbox to get the correctness metric after
-    the agent has run, modify this function.
-    """
-    logger.info('-' * 30)
-    logger.info('BEGIN Runtime Completion Fn')
-    logger.info('-' * 30)
-    obs: CmdOutputObservation
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-
-    action = CmdRunAction(command='git add .')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
-        f'Failed to git add -A: {str(obs)}',
-    )
-
-    action = CmdRunAction(command='git commit -m "openhands edits"')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation)
-        and (obs.exit_code == 0 or obs.exit_code == 1),
-        f'Failed to git commit -m "openhands": {str(obs)}',
-    )
-
-    # Generate diff patch compared to base commit, excluding spec.pdf.bz2 files
-    n_retries = 0
-    git_patch = None
-    while n_retries < 5:
-        action = CmdRunAction(
-            command=f"git diff {instance['base_commit']} HEAD -- . ':(exclude)spec.pdf.bz2'"
-        )
-        action.timeout = 600 + 100 * n_retries
-        logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = runtime.run_action(action)
-        # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-        n_retries += 1
-        if isinstance(obs, CmdOutputObservation):
-            if obs.exit_code == 0:
-                git_patch = obs.content.strip()
-                break
-            else:
-                logger.info('Failed to get git diff, retrying...')
-                sleep_if_should_continue(10)
-        elif isinstance(obs, ErrorObservation):
-            logger.error(f'Error occurred: {obs.content}. Retrying...')
-            sleep_if_should_continue(10)
-        else:
-            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
-
-    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
-
-    test_dir = instance['test']['test_dir']
-    action = CmdRunAction(
-        command=f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors {test_dir} > test_output.txt 2>&1"
-    )
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to run test command: {str(obs)}',
-    )
-    # Read test output
-    action = CmdRunAction(command='cat test_output.txt')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to read test output: {str(obs)}',
-    )
-    test_output = obs.content.strip()
-    # logger.info(f'Test output: {test_output}')
-
-    # Save pytest exit code
-    action = CmdRunAction(command='echo $?')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
-        f'Failed to save pytest exit code: {str(obs)}',
-    )
-    pytest_exit_code = obs.content.strip()
-    # logger.info(f'Pytest exit code: {pytest_exit_code}')
-
-    # Read the test report
-    action = CmdRunAction(command='cat report.json')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to read test report: {str(obs)}',
-    )
-    # Get test IDs from instance
-    repo_name = instance['repo'].split('/')[1]
-    repo_name = repo_name.replace('.', '-')
-    action = CmdRunAction(command=f'commit0 get-tests {repo_name}')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    test_ids = obs.content.strip().split('\n')
-
-    try:
-        report = json.loads(obs.content)
-        tests = {x['nodeid']: x['call'] for x in report['tests'] if 'call' in x}
-
-        # Calculate test statistics
-        status = []
-        runtimes = []
-        no_runs = 0
-
-        for test_id in test_ids:
-            if test_id in tests and tests[test_id] is not None:
-                status.append(tests[test_id]['outcome'])
-                runtimes.append(tests[test_id]['duration'])
-                no_runs += 1
-            else:
-                status.append('failed')
-                runtimes.append(0)
-
-        status_counts = Counter(status)
-        total_runtime = sum(runtimes) if no_runs > 0 else 0
-        num_passed = status_counts.get('passed', 0) + status_counts.get('xfail', 0)
-        passed_ratio = num_passed / len(status) if status else 0
-
-        eval_result = {
-            'name': workspace_dir_name,
-            'sum': total_runtime,
-            'passed': passed_ratio,
-            'num_passed': num_passed,
-            'num_tests': len(test_ids),
-        }
-
-    except json.JSONDecodeError:
-        logger.error('Failed to parse test report JSON')
-        eval_result = {
-            'name': workspace_dir_name,
-            'sum': 0,
-            'passed': 0,
-            'num_passed': 0,
-            'num_tests': len(test_ids),
-        }
-
-    # Create tarball of workspace
-    temp_zip = runtime.copy_from(f'/workspace/{workspace_dir_name}')
-
-    commit0_dir = os.path.dirname(__file__)
-    persistent_zip = os.path.join(commit0_dir, f'{workspace_dir_name}.zip')
-    with open(temp_zip, 'rb') as src, open(persistent_zip, 'wb') as dst:
-        dst.write(src.read())
-    zip_file = persistent_zip
-    return {
-        'eval_result': eval_result,
-        'git_patch': git_patch,
-        'test_output': test_output,
-        'pytest_exit_code': pytest_exit_code,
-        'zip_file': zip_file,
-    }
-
-
-def process_instance(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-    reset_logger: bool = True,
-) -> EvalOutput:
-    config = get_config(instance, metadata)
-    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
-    if reset_logger:
-        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
-        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
-    else:
-        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
-
-    runtime = create_runtime(config)
-    call_async_from_sync(runtime.connect)
-    try:
-        initialize_runtime(runtime, instance)
-
-        instruction = get_instruction(instance, metadata)
-
-        # Here's how you can run the agent (similar to the `main` function) and get the final task state
-        state: State | None = asyncio.run(
-            run_controller(
-                config=config,
-                initial_user_action=MessageAction(content=instruction),
-                runtime=runtime,
-                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
-                    metadata.agent_class
-                ],
-            )
-        )
-
-        # if fatal error, throw EvalError to trigger re-run
-        if (
-            state.last_error
-            and 'fatal error during agent execution' in state.last_error
-            and 'stuck in a loop' not in state.last_error
-        ):
-            raise EvalException('Fatal error detected: ' + state.last_error)
-
-        # ======= THIS IS Commit0 specific =======
-        # Get git patch
-        return_val = complete_runtime(runtime, instance)
-        eval_result = return_val['eval_result']
-        git_patch = return_val['git_patch']
-        test_output = return_val['test_output']
-        pytest_exit_code = return_val['pytest_exit_code']
-        zip_file = return_val['zip_file']
-
-        repo_name = instance['repo'].split('/')[1]
-        zip_dest = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}.zip'
-        )
-        patch_file = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_patch.diff'
-        )
-        test_output_file = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_test_output.txt'
-        )
-        pytest_exit_code_file = os.path.join(
-            metadata.eval_output_dir,
-            'repos',
-            repo_name,
-            f'{repo_name}_pytest_exit_code.txt',
-        )
-
-        os.makedirs(os.path.dirname(zip_dest), exist_ok=True)
-        os.rename(zip_file, zip_dest)
-
-        write_targets = [
-            (patch_file, git_patch),
-            (test_output_file, test_output),
-            (pytest_exit_code_file, pytest_exit_code),
-        ]
-
-        for write_target in write_targets:
-            with open(write_target[0], 'w') as f:
-                f.write(write_target[1])
-
-        logger.info(
-            f'Got evaluation result for repo {instance.instance_id}:\n--------\n{eval_result}\n--------'
-        )
-    finally:
-        runtime.close()
-    # ==========================================
-
-    # ======= Attempt to evaluate the agent's edits =======
-    # we use eval_infer.sh to evaluate the agent's edits, not here
-    # because the agent may alter the environment / testcases
-    test_result = {
-        'eval_result': eval_result,
-    }
-
-    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
-    histories = [event_to_dict(event) for event in state.history]
-    metrics = state.metrics.get() if state.metrics else None
-
-    # Save the output
-    output = EvalOutput(
-        instance_id=instance.instance_id,
-        instruction=instruction,
-        instance=instance.to_dict(),
-        test_result=test_result,
-        metadata=metadata,
-        history=histories,
-        metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
-    )
-    return output
-
-
-def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
-    """Setup Commit0 dataset based on split type.
-
-    Args:
-        dataset: Full Commit0 dataset
-        repo_split: Split type ('all', 'lite' or specific repo name)
-
-    Returns:
-        Filtered dataset based on split type
-    """
-
-    filtered_dataset = pd.concat(
-        [
-            dataset[dataset['repo'].str.split('/').str[1] == repo]
-            for repo in SPLIT.get(repo_split, [])
-        ]
-    )
-
-    # Drop setup column if it exists
-    if 'setup' in filtered_dataset.columns:
-        filtered_dataset = filtered_dataset.drop('setup', axis=1)
-
-    # Replace all forward slashes in instance_id with hyphens
-    filtered_dataset['instance_id'] = filtered_dataset['repo'].str.split('/').str[1]
-
-    return filtered_dataset
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--dataset',
-        type=str,
-        default='wentingzhao/commit0_combined',
-        help='dataset to evaluate on, only test split exists for this HF dataset',
-    )
-    parser.add_argument(
-        '--split',
-        type=str,
-        default='test',
-        help='this is the HF dataset split',
-    )
-    parser.add_argument(
-        '--repo-split',
-        type=str,
-        default='lite',
-        help='all, lite, or each repo name',
-    )
-    args, _ = parser.parse_known_args()
-
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenHands's repo
-    dataset = load_dataset(args.dataset, split=args.split)
-
-    commit0_datasets = commit0_setup(dataset.to_pandas(), args.repo_split)
-
-    logger.info(f'Loaded dataset {args.dataset} with reposplit {args.repo_split}')
-
-    llm_config = None
-    if args.llm_config:
-        llm_config = get_llm_config_arg(args.llm_config)
-        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
-        llm_config.modify_params = False
-        llm_config.log_completions = True
-
-    if llm_config is None:
-        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
-
-    details = {}
-    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
-
-    dataset_descrption = (
-        args.dataset.replace('/', '__') + '-' + args.repo_split.replace('/', '__')
-    )
-    metadata = make_metadata(
-        llm_config,
-        dataset_descrption,
-        args.agent_cls,
-        args.max_iterations,
-        args.eval_note,
-        args.eval_output_dir,
-        details=details,
-    )
-
-    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-
-    instances = prepare_dataset(commit0_datasets, output_file, args.eval_n_limit)
-
-    run_evaluation(
-        instances,
-        metadata,
-        output_file,
-        args.eval_num_workers,
-        process_instance,
-        timeout_seconds=120 * 60,  # 2 hour PER instance should be more than enough
-    )

evaluation/benchmarks/commit0_bench/scripts/run_infer.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-source "evaluation/utils/version_control.sh"
-
-REPO_SPLIT=$1
-MODEL_CONFIG=$2
-COMMIT_HASH=$3
-AGENT=$4
-EVAL_LIMIT=$5
-MAX_ITER=$6
-NUM_WORKERS=$7
-DATASET=$8
-SPLIT=$9
-N_RUNS=${10}
-
-if [ -z "$NUM_WORKERS" ]; then
-  NUM_WORKERS=1
-  echo "Number of workers not specified, use default $NUM_WORKERS"
-fi
-checkout_eval_branch
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-if [ -z "$MAX_ITER" ]; then
-  echo "MAX_ITER not specified, use default 100"
-  MAX_ITER=100
-fi
-
-if [ -z "$USE_INSTANCE_IMAGE" ]; then
-  echo "USE_INSTANCE_IMAGE not specified, use default true"
-  USE_INSTANCE_IMAGE=true
-fi
-
-if [ -z "$RUN_WITH_BROWSING" ]; then
-  echo "RUN_WITH_BROWSING not specified, use default false"
-  RUN_WITH_BROWSING=false
-fi
-
-
-if [ -z "$DATASET" ]; then
-  echo "DATASET not specified, use default wentingzhao/commit0_combined"
-  DATASET="wentingzhao/commit0_combined"
-fi
-
-if [ -z "$REPO_SPLIT" ]; then
-  echo "REPO_SPLIT not specified, use default lite"
-  REPO_SPLIT=0
-fi
-
-if [ -z "$SPLIT" ]; then
-  echo "HF SPLIT not specified, use default test"
-  SPLIT="test"
-fi
-
-export USE_INSTANCE_IMAGE=$USE_INSTANCE_IMAGE
-echo "USE_INSTANCE_IMAGE: $USE_INSTANCE_IMAGE"
-export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
-echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
-
-get_openhands_version
-
-echo "AGENT: $AGENT"
-echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-echo "DATASET: $DATASET"
-echo "HF SPLIT: $SPLIT"
-echo "REPO SPLIT: $REPO_SPLIT"
-
-# Default to NOT use Hint
-if [ -z "$USE_HINT_TEXT" ]; then
-  export USE_HINT_TEXT=false
-fi
-echo "USE_HINT_TEXT: $USE_HINT_TEXT"
-EVAL_NOTE="$OPENHANDS_VERSION"
-# if not using Hint, add -no-hint to the eval note
-if [ "$USE_HINT_TEXT" = false ]; then
-  EVAL_NOTE="$EVAL_NOTE-no-hint"
-fi
-
-if [ "$RUN_WITH_BROWSING" = true ]; then
-  EVAL_NOTE="$EVAL_NOTE-with-browsing"
-fi
-
-if [ -n "$EXP_NAME" ]; then
-  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
-fi
-
-function run_eval() {
-  local eval_note=$1
-  COMMAND="poetry run python evaluation/benchmarks/commit0_bench/run_infer.py \
-    --agent-cls $AGENT \
-    --llm-config $MODEL_CONFIG \
-    --max-iterations $MAX_ITER \
-    --eval-num-workers $NUM_WORKERS \
-    --eval-note $eval_note \
-    --dataset $DATASET \
-    --split $SPLIT \
-    --repo-split $REPO_SPLIT"
-
-  if [ -n "$EVAL_LIMIT" ]; then
-    echo "EVAL_LIMIT: $EVAL_LIMIT"
-    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-  fi
-
-  # Run the command
-  eval $COMMAND
-}
-
-unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
-if [ -z "$N_RUNS" ]; then
-  N_RUNS=1
-  echo "N_RUNS not specified, use default $N_RUNS"
-fi
-
-for i in $(seq 1 $N_RUNS); do
-  current_eval_note="$EVAL_NOTE-run_$i"
-  echo "EVAL_NOTE: $current_eval_note"
-  run_eval $current_eval_note
-done
-
-checkout_original_branch

evaluation/benchmarks/swe_bench/scripts/cleanup_remote_runtime.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-
-# API base URL
-BASE_URL="https://runtime.eval.all-hands.dev"
-
-# Get the list of runtimes
-response=$(curl --silent --location --request GET "${BASE_URL}/list" \
-  --header "X-API-Key: ${ALLHANDS_API_KEY}")
-
-n_runtimes=$(echo $response | jq -r '.total')
-echo "Found ${n_runtimes} runtimes. Stopping them..."
-
-runtime_ids=$(echo $response | jq -r '.runtimes | .[].runtime_id')
-
-# Function to stop a single runtime
-stop_runtime() {
-  local runtime_id=$1
-  local counter=$2
-  echo "Stopping runtime ${counter}/${n_runtimes}: ${runtime_id}"
-  curl --silent --location --request POST "${BASE_URL}/stop" \
-    --header "X-API-Key: ${ALLHANDS_API_KEY}" \
-    --header "Content-Type: application/json" \
-    --data-raw "{\"runtime_id\": \"${runtime_id}\"}"
-  echo
-}
-export -f stop_runtime
-export BASE_URL ALLHANDS_API_KEY n_runtimes
-
-# Use GNU Parallel to stop runtimes in parallel
-echo "$runtime_ids" | parallel -j 16 --progress stop_runtime {} {#}
-
-echo "All runtimes have been stopped."

evaluation/benchmarks/swe_bench/scripts/eval/summarize_outputs.py

@@ -1,299 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import glob
-import json
-import os
-from collections import Counter
-
-import pandas as pd
-import random
-import numpy as np
-
-from openhands.events.serialization import event_from_dict
-from openhands.events.utils import get_pairs_from_events
-
-ERROR_KEYWORDS = [
-    'Agent encountered an error while processing the last action',
-    'APIError',
-    'Action execution failed',
-    'litellm.Timeout: APITimeoutError',
-]
-
-
-def get_bootstrap_accuracy_error_bars(values: float | int | bool, num_samples: int = 1000, p_value=0.05) -> tuple[float, float]:
-    sorted_vals = np.sort(
-        [
-            np.mean(random.sample(values, len(values) // 2))
-            for _ in range(num_samples)
-        ]
-    )
-    bottom_idx = int(num_samples * p_value / 2)
-    top_idx = int(num_samples * (1.0 - p_value / 2))
-    return (sorted_vals[bottom_idx], sorted_vals[top_idx])
-
-
-def process_file(file_path):
-    with open(file_path, 'r') as file:
-        lines = file.readlines()
-
-    num_lines = len(lines)
-    num_error_lines = 0
-    num_agent_stuck_in_loop = 0
-    num_resolved = 0
-    resolved_arr = []
-    num_empty_patch = 0
-    num_unfinished_runs = 0
-    error_counter = Counter()
-    main_agent_cost = []
-    editor_cost = []
-    num_turns = []
-
-    for line in lines:
-        _d = json.loads(line)
-
-        if 'metrics' not in _d or _d['metrics'] is None:
-            # this is a failed run
-            num_unfinished_runs += 1
-            continue
-
-        # Cost
-        costs = _d['metrics'].get('costs', [])
-        _cur_main_agent_cost = 0
-        _cur_editor_cost = 0
-        for cost in costs:
-            if isinstance(cost, float):
-                # backward compatible
-                _cur_main_agent_cost += cost
-            else:
-                if 'draft_editor' in cost['model']:
-                    _cur_editor_cost += cost['cost']
-                else:
-                    _cur_main_agent_cost += cost['cost']
-
-        main_agent_cost.append(_cur_main_agent_cost)
-        editor_cost.append(_cur_editor_cost)
-
-        # Turn status
-        history = _d.get('history', [])
-        events = [event_from_dict(event) for event in history]
-        pairs = get_pairs_from_events(events)
-        num_turns.append(len(pairs))
-
-        # Patch & resolve status
-        patch = _d.get('test_result', {}).get('git_patch', '')
-        if patch == '':
-            num_empty_patch += 1
-            continue
-
-        report = _d.get('report', {}) or {}
-        resolved = report.get('resolved', False)
-        if resolved:
-            num_resolved += 1
-            resolved_arr.append(1)
-        else:
-            resolved_arr.append(0)
-
-        # Error
-        error = _d.get('error', None)
-
-        if error is not None and isinstance(error, str):
-            agent_stuck_in_loop = 'Agent got stuck in a loop' in error
-            contains_error = bool(error) and not agent_stuck_in_loop
-            if agent_stuck_in_loop:
-                error_counter['Agent got stuck in a loop'] += 1
-                num_agent_stuck_in_loop += 1
-            elif contains_error:
-                error_counter[error] += 1
-            continue
-
-        for keyword in ERROR_KEYWORDS:
-            if keyword in line:
-                error_counter[keyword] += 1
-                num_error_lines += 1
-                break
-
-    return {
-        'file_path': file_path,
-        'total_instances': num_lines,
-        'resolved': {
-            'count': num_resolved,
-            'percentage': (num_resolved / num_lines * 100) if num_lines > 0 else 0,
-            'ci': tuple(x * 100 for x in get_bootstrap_accuracy_error_bars(resolved_arr)),
-        },
-        'empty_patches': {
-            'count': num_empty_patch,
-            'percentage': (num_empty_patch / num_lines * 100) if num_lines > 0 else 0,
-        },
-        'unfinished_runs': {
-            'count': num_unfinished_runs,
-            'percentage': (num_unfinished_runs / num_lines * 100)
-            if num_lines > 0
-            else 0,
-        },
-        'errors': {
-            'total': num_error_lines,
-            'percentage': (num_error_lines / num_lines * 100) if num_lines > 0 else 0,
-            'stuck_in_loop': {
-                'count': num_agent_stuck_in_loop,
-                'percentage': (num_agent_stuck_in_loop / num_lines * 100)
-                if num_lines > 0
-                else 0,
-            },
-            'breakdown': {
-                str(error): {
-                    'count': count,
-                    'percentage': (count / num_lines * 100) if num_lines > 0 else 0,
-                }
-                for error, count in error_counter.items()
-            },
-        },
-        'costs': {
-            'main_agent': sum(main_agent_cost),
-            'editor': sum(editor_cost),
-            'total': sum(main_agent_cost) + sum(editor_cost),
-        },
-        'statistics': {
-            'avg_turns': sum(num_turns) / num_lines if num_lines > 0 else 0,
-            'costs': {
-                'main_agent': sum(main_agent_cost) / num_lines if num_lines > 0 else 0,
-                'editor': sum(editor_cost) / num_lines if num_lines > 0 else 0,
-                'total': (sum(main_agent_cost) + sum(editor_cost)) / num_lines
-                if num_lines > 0
-                else 0,
-            },
-        },
-    }
-
-
-def aggregate_directory(input_path) -> pd.DataFrame:
-    # Process all output.jsonl files in subdirectories
-    pattern = os.path.join(input_path, '**/output.jsonl')
-    files = glob.glob(pattern, recursive=True)
-    print(f'Processing {len(files)} files from directory {input_path}')
-
-    # Process each file silently and collect results
-    results = []
-    for file_path in files:
-        try:
-            result = process_file(file_path)
-            results.append(result)
-        except Exception as e:
-            print(f'Error processing {file_path}: {str(e)}')
-            import traceback
-
-            traceback.print_exc()
-            continue
-
-    # Convert results to pandas DataFrame and sort by resolve rate
-    df = pd.DataFrame(results)
-
-    # Extract directory name from file path
-    df['directory'] = df['file_path'].apply(
-        lambda x: os.path.basename(os.path.dirname(x))
-    )
-
-    df['resolve_rate'] = df['resolved'].apply(lambda x: x['percentage'])
-    df['resolve_rate_ci'] = df['resolved'].apply(lambda x: x['ci'])
-    df['empty_patch_rate'] = df['empty_patches'].apply(lambda x: x['percentage'])
-    df['unfinished_rate'] = df['unfinished_runs'].apply(lambda x: x['percentage'])
-    df['avg_turns'] = df['statistics'].apply(lambda x: x['avg_turns'])
-    df['error_rate'] = df['errors'].apply(lambda x: x['percentage'])
-    df['avg_cost'] = df['statistics'].apply(lambda x: x['costs']['total'])
-
-    df = df.sort_values('resolve_rate', ascending=False)
-
-    return df
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'input_path', type=str, help='The file or directory to summarize'
-    )
-    parser.add_argument(
-        '--output',
-        type=str,
-        help='Output JSONL file for results',
-        default='summary_results.jsonl',
-    )
-    args = parser.parse_args()
-
-    if os.path.isdir(args.input_path):
-        df = aggregate_directory(args.input_path)
-        # Create the summary string
-        columns = [
-            'directory',
-            'resolve_rate',
-            'empty_patch_rate',
-            'unfinished_rate',
-            'error_rate',
-            'avg_turns',
-            'avg_cost',
-            'total_instances',
-        ]
-        summary_str = df[columns].to_string(
-            float_format=lambda x: '{:.2f}'.format(x),
-            formatters={
-                'directory': lambda x: x[:90]
-            },  # Truncate directory names to 20 chars
-            index=False,
-        )
-
-        # Print to console
-        print('\nResults summary (sorted by resolve rate):')
-        print(summary_str)
-
-        # Save to text file
-        txt_output = args.output.rsplit('.', 1)[0] + '.txt'
-        with open(txt_output, 'w') as f:
-            f.write('Results summary (sorted by resolve rate):\n')
-            f.write(summary_str)
-
-        # Save
-        df.to_json(args.output, lines=True, orient='records')
-        df[columns].to_csv(args.output.rsplit('.', 1)[0] + '.csv', index=False)
-    else:
-        # Process single file with detailed output
-        results = []
-        try:
-            result = process_file(args.input_path)
-            results.append(result)
-
-            # Print detailed results for single file
-            print(f'\nResults for {args.input_path}:')
-            print(
-                f"Number of resolved: {result['resolved']['count']} / {result['total_instances']} ({result['resolved']['percentage']:.2f}% [{result['resolved']['ci'][0]:.2f}%, {result['resolved']['ci'][1]:.2f}%])"
-            )
-            print(
-                f"Number of empty patch: {result['empty_patches']['count']} / {result['total_instances']} ({result['empty_patches']['percentage']:.2f}%)"
-            )
-            print(
-                f"Number of error lines: {result['errors']['total']} / {result['total_instances']} ({result['errors']['percentage']:.2f}%)"
-            )
-            print(
-                f"Number of agent stuck in loop: {result['errors']['stuck_in_loop']['count']} / {result['total_instances']} ({result['errors']['stuck_in_loop']['percentage']:.2f}%)"
-            )
-            print(
-                f"Number of unfinished runs: {result['unfinished_runs']['count']} / {result['total_instances']} ({result['unfinished_runs']['percentage']:.2f}%)"
-            )
-            print(f"Total cost: {result['costs']['total']:.2f} USD")
-            print('## Statistics')
-            print(
-                f"Avg. num of turns per instance: {result['statistics']['avg_turns']:.2f}"
-            )
-            print(
-                f"Avg. agent cost per instance: {result['statistics']['costs']['main_agent']:.2f} USD"
-            )
-            print(
-                f"Avg. editor cost per instance: {result['statistics']['costs']['editor']:.2f} USD"
-            )
-            print(
-                f"Avg. total cost per instance: {result['statistics']['costs']['total']:.2f} USD"
-            )
-
-            print('## Detailed error breakdown:')
-            for error, data in result['errors']['breakdown'].items():
-                print(f"{error}: {data['count']} ({data['percentage']:.2f}%)")
-
-        except Exception as e:
-            print(f'Error processing {args.input_path}: {str(e)}')

evaluation/benchmarks/swe_bench/scripts/eval/verify_costs.py

@@ -1,104 +0,0 @@
-import argparse
-
-import pandas as pd
-
-from openhands.core.logger import openhands_logger as logger
-
-
-def verify_instance_costs(row: pd.Series) -> float:
-    """
-    Verifies that the accumulated_cost matches the sum of individual costs in metrics.
-    Also checks for duplicate consecutive costs which might indicate buggy counting.
-    If the consecutive costs are identical, the file is affected by this bug:
-    https://github.com/All-Hands-AI/OpenHands/issues/5383
-
-    Args:
-        row: DataFrame row containing instance data with metrics
-    Returns:
-        float: The verified total cost for this instance (corrected if needed)
-    """
-    try:
-        metrics = row.get('metrics')
-        if not metrics:
-            logger.warning(f"Instance {row['instance_id']}: No metrics found")
-            return 0.0
-
-        accumulated = metrics.get('accumulated_cost')
-        costs = metrics.get('costs', [])
-
-        if accumulated is None:
-            logger.warning(
-                f"Instance {row['instance_id']}: No accumulated_cost in metrics"
-            )
-            return 0.0
-
-        # Check for duplicate consecutive costs and systematic even-odd pairs
-        has_duplicate = False
-        all_pairs_match = True
-
-        # Check each even-odd pair (0-1, 2-3, etc.)
-        for i in range(0, len(costs) - 1, 2):
-            if abs(costs[i]['cost'] - costs[i + 1]['cost']) < 1e-6:
-                has_duplicate = True
-                logger.debug(
-                    f"Instance {row['instance_id']}: Possible buggy double-counting detected! "
-                    f"Steps {i} and {i+1} have identical costs: {costs[i]['cost']:.2f}"
-                )
-            else:
-                all_pairs_match = False
-                break
-
-        # Calculate total cost, accounting for buggy double counting if detected
-        if len(costs) >= 2 and has_duplicate and all_pairs_match:
-            paired_steps_cost = sum(
-                cost_entry['cost']
-                for cost_entry in costs[: -1 if len(costs) % 2 else None]
-            )
-            real_paired_cost = paired_steps_cost / 2
-
-            unpaired_cost = costs[-1]['cost'] if len(costs) % 2 else 0
-            total_cost = real_paired_cost + unpaired_cost
-
-        else:
-            total_cost = sum(cost_entry['cost'] for cost_entry in costs)
-
-        if not abs(total_cost - accumulated) < 1e-6:
-            logger.warning(
-                f"Instance {row['instance_id']}: Cost mismatch: "
-                f"accumulated: {accumulated:.2f}, sum of costs: {total_cost:.2f}, "
-            )
-
-        return total_cost
-
-    except Exception as e:
-        logger.error(
-            f"Error verifying costs for instance {row.get('instance_id', 'UNKNOWN')}: {e}"
-        )
-        return 0.0
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Verify costs in SWE-bench output file'
-    )
-    parser.add_argument(
-        'input_filepath', type=str, help='Path to the output.jsonl file'
-    )
-    args = parser.parse_args()
-
-    try:
-        # Load and verify the JSONL file
-        df = pd.read_json(args.input_filepath, lines=True)
-        logger.info(f'Loaded {len(df)} instances from {args.input_filepath}')
-
-        # Verify costs for each instance and sum up total
-        total_cost = df.apply(verify_instance_costs, axis=1).sum()
-        logger.info(f'Total verified cost across all instances: ${total_cost:.2f}')
-
-    except Exception as e:
-        logger.error(f'Failed to process file: {e}')
-        raise
-
-
-if __name__ == '__main__':
-    main()

evaluation/benchmarks/the_agent_company/README.md

@@ -1,43 +0,0 @@
-# The Agent Company Evaluation with OpenHands
-
-This folder contains the evaluation harness that we built on top of the original [The Agent Company](https://github.com/TheAgentCompany/TheAgentCompany/tree/main/evaluation) ([paper](https://arxiv.org/abs/2412.14161)).
-
-The evaluation consists of three steps:
-
-1. Environment setup: [install python environment](../../README.md#development-environment), [configure LLM config](../../README.md#configure-openhands-and-your-llm), [launch services](https://github.com/TheAgentCompany/TheAgentCompany/blob/main/docs/SETUP.md).
-2. [Run Evaluation](#run-inference-on-the-agent-company-instances): Run all tasks and get the evaluation results.
-
-## Setup Environment and LLM Configuration
-
-Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
-
-## Run Inference on The Agent Company Tasks
-
-When the `run_infer.sh` script is started, it will automatically pull all task images. Every task image will be used to create an OpenHands runtime image where the agent will operate on.
-
-```bash
-./evaluation/benchmarks/the_agent_company/scripts/run_infer.sh \
-  --agent-llm-config <agent-llm-config>  \
-  --env-llm-config <env-llm-config> \
-  --outputs-path <outputs-path> \
-  --server-hostname <server-hostname> \
-  --version <version>
-
-# Example
-./evaluation/benchmarks/the_agent_company/scripts/run_infer.sh \
-  --agent-llm-config claude-3-5-sonnet-20240620 \
-  --env-llm-config claude-3-5-sonnet-20240620 \
-  --outputs-path outputs \
-  --server-hostname localhost \
-  --version 1.0.0
-```
-
-- `agent-llm-config`: the config name for the agent LLM. This should match the config name in config.toml. This is the LLM used by the agent (e.g. CodeActAgent).
-- `env-llm-config`: the config name for the environment LLM. This should match the config name in config.toml. This is used by the chat bots (NPCs) and LLM-based evaluators.
-- `outputs-path`: the path to save trajectories and evaluation results.
-- `server-hostname`: the hostname of the server that hosts all the web services. It could be localhost if you are running the evaluation and services on the same machine. If the services are hosted on a remote machine, you must use the hostname of the remote machine rather than IP address.
-- `version`: the version of the task images to use. Currently, the only supported version is 1.0.0.
-
-The script is idempotent. If you run it again, it will resume from the last checkpoint. It would usually take a few days to finish evaluation.
-
-Note: the script will automatically skip a task if it encounters an error. This usually happens when the OpenHands runtime dies due to some unexpected errors. This means even if the script finishes, it might not have evaluated all tasks. You can manually resume the evaluation by running the script again.

evaluation/benchmarks/the_agent_company/browsing.py

@@ -1,273 +0,0 @@
-##################################################################################################
-# Adapted from https://github.com/TheAgentCompany/TheAgentCompany/blob/main/evaluation/browsing.py
-##################################################################################################
-
-import base64
-import os
-import re
-from dataclasses import dataclass
-from enum import Enum, auto
-from typing import Dict, List, Optional, Union
-
-from openhands.core.logger import openhands_logger as logger
-from openhands.events.action import BrowseInteractiveAction
-from openhands.events.observation import BrowserOutputObservation
-from openhands.runtime.base import Runtime
-
-
-class ActionType(Enum):
-    GOTO = auto()
-    FILL = auto()
-    CLICK = auto()
-    NOOP = auto()
-
-
-@dataclass
-class Selector:
-    """
-    Represents either a direct anchor ID or a descriptive selector
-    """
-
-    value: str
-    is_anchor: bool = False
-
-    def __str__(self) -> str:
-        return f'{self.value}'
-
-
-@dataclass
-class BrowserAction:
-    """Base class for all browser actions"""
-
-    action_type: ActionType
-
-    def to_instruction(self) -> str:
-        """Convert the action to a browser instruction string"""
-        raise NotImplementedError
-
-
-@dataclass
-class GotoAction(BrowserAction):
-    url: str
-
-    def __init__(self, url: str):
-        super().__init__(ActionType.GOTO)
-        self.url = url
-
-    def to_instruction(self) -> str:
-        return f'goto("{self.url}")'
-
-
-@dataclass
-class NoopAction(BrowserAction):
-    milliseconds: int
-
-    def __init__(self, milliseconds: int):
-        super().__init__(ActionType.NOOP)
-        self.milliseconds = milliseconds
-
-    def to_instruction(self) -> str:
-        return f'noop({self.milliseconds})'
-
-
-@dataclass
-class InputAction(BrowserAction):
-    selector: Selector
-    value: str
-
-    def __init__(self, selector: Union[str, Selector], value: str):
-        super().__init__(ActionType.FILL)
-        self.selector = (
-            selector if isinstance(selector, Selector) else Selector(selector)
-        )
-        self.value = value
-
-    def to_instruction(self) -> str:
-        return f'fill("{self.selector}", "{self.value}")'
-
-
-@dataclass
-class ClickAction(BrowserAction):
-    selector: Selector
-
-    def __init__(self, selector: Union[str, Selector]):
-        super().__init__(ActionType.CLICK)
-        self.selector = (
-            selector if isinstance(selector, Selector) else Selector(selector)
-        )
-
-    def to_instruction(self) -> str:
-        return f'click("{self.selector}")'
-
-
-def parse_content_to_elements(content: str) -> Dict[str, str]:
-    """Parse the observation content into a dictionary mapping anchors to their descriptions"""
-    elements = {}
-    current_anchor = None
-    description_lines = []
-
-    for line in content.split('\n'):
-        line = line.strip()
-        if not line:
-            continue
-
-        # Check for anchor line
-        anchor_match = re.match(r'\[(\d+)\](.*)', line)
-        if anchor_match:
-            # Save previous element if it exists
-            if current_anchor and description_lines:
-                elements[current_anchor] = ' '.join(description_lines)
-
-            # Start new element
-            current_anchor = anchor_match.group(1)
-            description_lines = [anchor_match.group(2).strip()]
-        else:
-            # Add to current description if we have an anchor
-            if current_anchor:
-                description_lines.append(line)
-
-    # Save last element
-    if current_anchor and description_lines:
-        elements[current_anchor] = ' '.join(description_lines)
-
-    return elements
-
-
-def find_matching_anchor(content: str, selector: str) -> Optional[str]:
-    """Find the anchor ID that matches the given selector description"""
-    elements = parse_content_to_elements(content)
-
-    # Clean up selector and create a pattern
-    selector = selector.lower().strip()
-
-    for anchor, description in elements.items():
-        description = description.lower().strip()
-        if selector in description:
-            return anchor
-
-    return None
-
-
-def resolve_action(action: BrowserAction, content: str) -> BrowserAction:
-    """
-    Resolve any descriptive selectors in the action to anchor IDs based on the content.
-    Returns a new action with resolved selectors.
-    """
-    if isinstance(action, (InputAction, ClickAction)):
-        if not action.selector.is_anchor:
-            anchor = find_matching_anchor(content, action.selector.value)
-            if anchor:
-                new_selector = Selector(anchor, is_anchor=True)
-                if isinstance(action, InputAction):
-                    return InputAction(new_selector, action.value)
-                else:
-                    return ClickAction(new_selector)
-            else:
-                logger.error(f'NO MATCH FOUND FOR SELECTOR, {action.selector}')
-                return None
-    return action
-
-
-def pre_login(
-    runtime: Runtime,
-    services: List[str],
-    save_screenshots=True,
-    screenshots_dir='screenshots',
-):
-    """
-    Logs in to all the websites that are needed for the evaluation.
-    Once logged in, the sessions would be cached in the browser, so OpenHands
-    agent doesn't need to log in to these websites again.
-    """
-    owncloud_login_actions = [
-        GotoAction('http://the-agent-company.com:8092'),
-        NoopAction(1000),
-        InputAction("textbox '', clickable, focused, required", 'theagentcompany'),
-        NoopAction(1000),
-        InputAction("textbox '', clickable, required", 'theagentcompany'),
-        NoopAction(1000),
-        ClickAction("button '', clickable"),
-        NoopAction(1000),
-    ]
-
-    rocketchat_login_actions = [
-        GotoAction('http://the-agent-company.com:3000'),
-        NoopAction(1000),
-        InputAction("textbox '', clickable, focused", 'theagentcompany'),
-        NoopAction(1000),
-        InputAction("textbox '', clickable", 'theagentcompany'),
-        NoopAction(1000),
-        ClickAction("button 'Login', clickable"),
-    ]
-
-    gitlab_login_actions = [
-        GotoAction('http://the-agent-company.com:8929/users/sign_in'),
-        NoopAction(1000),
-        InputAction("textbox 'Username or primary email'", 'root'),
-        NoopAction(1000),
-        InputAction("textbox 'Password'", 'theagentcompany'),
-        NoopAction(1000),
-        ClickAction("button 'Sign in', clickable"),
-    ]
-
-    # devnote: plane reset is not stable, and sometimes it fails to launch
-    # in which case the login action will fail, and then we would skip the task
-    plane_login_actions = [
-        GotoAction('http://the-agent-company.com:8091'),
-        NoopAction(1000),
-        InputAction(
-            "textbox 'Email', clickable, focused",
-            'agent@company.com',
-        ),
-        NoopAction(1000),
-        ClickAction("button 'Continue'"),
-        NoopAction(1000),
-        InputAction("textbox 'Enter password', clickable", 'theagentcompany'),
-        NoopAction(1000),
-        ClickAction("button 'Go to workspace'"),
-    ]
-
-    all_login_actions = [
-        ('owncloud', owncloud_login_actions),
-        ('rocketchat', rocketchat_login_actions),
-        ('gitlab', gitlab_login_actions),
-        ('plane', plane_login_actions),
-    ]
-
-    for website_name, login_actions in all_login_actions:
-        if website_name not in services:
-            logger.info(
-                f"Skipping login for {website_name} because it's not in the list of services to reset"
-            )
-            continue
-
-        if save_screenshots:
-            directory = os.path.join(screenshots_dir, website_name)
-            if not os.path.exists(directory):
-                os.makedirs(directory)
-            image_id = 0
-        obs: BrowserOutputObservation = None
-        for action in login_actions:
-            # Resolve any descriptive selectors to anchor IDs
-            if obs:
-                action = resolve_action(action, obs.get_agent_obs_text())
-
-            if not action:
-                logger.error(f'FAILED TO RESOLVE ACTION, {action}')
-                raise Exception(
-                    'FAILED TO RESOLVE ACTION, maybe the service is not available'
-                )
-
-            # Convert the action to an instruction string
-            instruction = action.to_instruction()
-
-            browser_action = BrowseInteractiveAction(browser_actions=instruction)
-            browser_action.timeout = 10000
-            logger.info(browser_action, extra={'msg_type': 'ACTION'})
-            obs: BrowserOutputObservation = runtime.run_action(browser_action)
-            logger.debug(obs, extra={'msg_type': 'OBSERVATION'})
-            if save_screenshots:
-                image_data = base64.b64decode(obs.screenshot)
-                with open(os.path.join(directory, f'{image_id}.png'), 'wb') as file:
-                    file.write(image_data)
-                    image_id += 1

evaluation/benchmarks/the_agent_company/run_infer.py

@@ -1,319 +0,0 @@
-##################################################################################################
-# Adapted from https://github.com/TheAgentCompany/TheAgentCompany/blob/main/evaluation/run_eval.py
-##################################################################################################
-
-import asyncio
-import base64
-import json
-import os
-import shutil
-import tempfile
-from typing import List
-
-import yaml
-from browsing import pre_login
-
-from openhands.controller.state.state import State
-from openhands.core.config import (
-    AppConfig,
-    LLMConfig,
-    SandboxConfig,
-    get_llm_config_arg,
-    get_parser,
-)
-from openhands.core.logger import openhands_logger as logger
-from openhands.core.main import create_runtime, run_controller
-from openhands.events.action import CmdRunAction, MessageAction
-from openhands.events.observation import BrowserOutputObservation, CmdOutputObservation
-from openhands.runtime.base import Runtime
-from openhands.utils.async_utils import call_async_from_sync
-
-
-def get_config(
-    base_container_image: str,
-    task_short_name: str,
-    mount_path_on_host: str,
-    llm_config: LLMConfig,
-) -> AppConfig:
-    config = AppConfig(
-        run_as_openhands=False,
-        max_budget_per_task=4,
-        max_iterations=100,
-        trajectories_path=os.path.join(
-            mount_path_on_host, f'traj_{task_short_name}.json'
-        ),
-        sandbox=SandboxConfig(
-            base_container_image=base_container_image,
-            enable_auto_lint=True,
-            # using host network to access the host machine from the container
-            use_host_network=True,
-            # large enough timeout, since some testcases take very long to run
-            timeout=300,
-            api_key=os.environ.get('ALLHANDS_API_KEY', None),
-        ),
-        # we mount trajectories path so that trajectories, generated by OpenHands
-        # controller, can be accessible to the evaluator file in the runtime container
-        workspace_mount_path=mount_path_on_host,
-        workspace_mount_path_in_sandbox='/outputs',
-    )
-    config.set_llm_config(llm_config)
-    return config
-
-
-def load_dependencies(runtime: Runtime) -> List[str]:
-    """
-    Every task has a dependencies.yml file, which lists all the services that the
-    task depends on. This function loads the file and returns all dependent service names.
-    """
-    command = 'cat /utils/dependencies.yml'
-    action = CmdRunAction(command=command)
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs: CmdOutputObservation = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert obs.exit_code == 0
-    dependencies = yaml.safe_load(obs.content)
-    if dependencies is None:
-        dependencies = []
-    return dependencies
-
-
-def init_task_env(runtime: Runtime, hostname: str, env_llm_config: LLMConfig):
-    command = (
-        f'SERVER_HOSTNAME={hostname} '
-        f'LITELLM_API_KEY={env_llm_config.api_key} '
-        f'LITELLM_BASE_URL={env_llm_config.base_url} '
-        f'LITELLM_MODEL={env_llm_config.model} '
-        'bash /utils/init.sh'
-    )
-    action = CmdRunAction(command=command)
-    action.timeout = 900
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert obs.exit_code == 0
-
-
-def codeact_user_response(state: State) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have solved the task, please finish the interaction.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
-    )
-
-    if state.history:
-        # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
-        user_msgs = [
-            event
-            for event in state.history
-            if isinstance(event, MessageAction) and event.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # let the agent know that it can give up when it has tried 3 times
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-
-
-def run_solver(
-    runtime: Runtime,
-    task_name: str,
-    config: AppConfig,
-    dependencies: List[str],
-    save_final_state: bool,
-    state_dir: str,
-    save_screenshots: bool,
-    screenshots_dir: str,
-) -> State:
-    instruction = 'Complete the task in /instruction/task.md'
-
-    if 'gitlab' in dependencies:
-        instruction += "\n\nGitlab username is 'root' and password is 'theagentcompany'"
-
-    state: State | None = asyncio.run(
-        run_controller(
-            config=config,
-            sid=task_name,
-            initial_user_action=MessageAction(content=instruction),
-            runtime=runtime,
-            fake_user_response_fn=codeact_user_response,
-        )
-    )
-    logger.info(state)
-
-    if save_screenshots:
-        screenshots_dir = os.path.join(screenshots_dir, task_name)
-        os.makedirs(screenshots_dir, exist_ok=True)
-        for image_id, obs in enumerate(state.history):
-            if isinstance(obs, BrowserOutputObservation):
-                image_data = base64.b64decode(obs.screenshot)
-                with open(
-                    os.path.join(screenshots_dir, f'{image_id}.png'), 'wb'
-                ) as file:
-                    file.write(image_data)
-
-    if save_final_state:
-        os.makedirs(state_dir, exist_ok=True)
-        with open(os.path.join(state_dir, f'state_{task_name}.json'), 'w') as file:
-            json.dump(str(state), file)
-
-    return state
-
-
-def run_evaluator(
-    runtime: Runtime, env_llm_config: LLMConfig, trajectory_path: str, result_path: str
-):
-    command = (
-        f'LITELLM_API_KEY={env_llm_config.api_key} '
-        f'LITELLM_BASE_URL={env_llm_config.base_url} '
-        f'LITELLM_MODEL={env_llm_config.model} '
-        f"DECRYPTION_KEY='theagentcompany is all you need' "  # Hardcoded Key
-        f'python_default /utils/eval.py --trajectory_path {trajectory_path} --result_path {result_path}'
-    )
-    action = CmdRunAction(command=command)
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert obs.exit_code == 0
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--task-image-name',
-        type=str,
-        default='ghcr.io/theagentcompany/example-image:1.0.0',
-        help='Task image name',
-    )
-    parser.add_argument(
-        '--outputs-path',
-        type=str,
-        default='./outputs',
-        help='Folder path to save trajectories and evaluation results',
-    )
-    parser.add_argument(
-        '--server-hostname',
-        type=str,
-        default='localhost',
-        help='Server hostname, e.g. localhost to access the host machine from the container, '
-        'assuming the task docker container is run with `--network host` flag',
-    )
-    parser.add_argument(
-        '--agent-llm-config',
-        type=str,
-        default=None,
-        help='LLM config for agent',
-    )
-    parser.add_argument(
-        '--env-llm-config',
-        type=str,
-        default=None,
-        help='LLM config for evaluation environment (NPC & llm-based evaluator)',
-    )
-    args, _ = parser.parse_known_args()
-
-    agent_llm_config: LLMConfig | None = None
-    if args.agent_llm_config:
-        agent_llm_config = get_llm_config_arg(args.agent_llm_config)
-
-    if agent_llm_config is None:
-        raise ValueError(
-            f'Could not find LLM config for agent: --agent-llm-config {args.agent_llm_config}'
-        )
-
-    if agent_llm_config.api_key is None:
-        raise ValueError('LLM API key is not set for agent')
-
-    env_llm_config: LLMConfig | None = None
-    if args.env_llm_config:
-        env_llm_config = get_llm_config_arg(args.env_llm_config)
-
-    if env_llm_config is None:
-        raise ValueError(
-            f'Could not find LLM config for evaluation environment: --env-llm-config {args.env_llm_config}'
-        )
-
-    if env_llm_config.api_key is None:
-        raise ValueError('LLM API key is not set for evaluation environment')
-
-    task_short_name = args.task_image_name.split('/')[-1].split(':')[0]
-    logger.info(
-        f'Task image name is {args.task_image_name}, short name is {task_short_name}'
-    )
-
-    # mount a temporary directory to pass trajectory from host to container, and to
-    # pass the evaluation result from container to host
-    # 1) trajectory is dumped by OpenHands library (on host machine), but it's needed by
-    # evaluator (in container), so we mount a temporary directory to pass it in
-    # 2) evaluation result is written by evaluator (in container), but we need to persist
-    # it on host machine, so we mount a temporary directory to pass it out
-    if os.getenv('TMPDIR') and os.path.exists(os.getenv('TMPDIR')):
-        temp_dir = os.path.abspath(os.getenv('TMPDIR'))
-    else:
-        temp_dir = tempfile.mkdtemp()
-    config: AppConfig = get_config(
-        args.task_image_name, task_short_name, temp_dir, agent_llm_config
-    )
-    runtime: Runtime = create_runtime(config)
-    call_async_from_sync(runtime.connect)
-
-    init_task_env(runtime, args.server_hostname, env_llm_config)
-
-    dependencies = load_dependencies(runtime)
-    logger.info(f'Service dependencies: {dependencies}')
-
-    try:
-        pre_login(
-            runtime,
-            dependencies,
-            save_screenshots=True,
-            screenshots_dir=os.path.join(
-                os.path.abspath(args.outputs_path), 'screenshots'
-            ),
-        )
-    except Exception as e:
-        logger.error(f'Failed to pre-login: {e}')
-
-        # before giving up, let's try to init and login again
-        init_task_env(runtime, args.server_hostname, env_llm_config)
-        pre_login(
-            runtime,
-            dependencies,
-            save_screenshots=True,
-            screenshots_dir=os.path.join(
-                os.path.abspath(args.outputs_path), 'screenshots'
-            ),
-        )
-
-    state = run_solver(
-        runtime,
-        task_short_name,
-        config,
-        dependencies,
-        save_final_state=True,
-        state_dir=os.path.abspath(args.outputs_path),
-        save_screenshots=True,
-        screenshots_dir=os.path.join(os.path.abspath(args.outputs_path), 'screenshots'),
-    )
-
-    # this path is the absolute path in the runtime container
-    trajectory_path = f'/outputs/traj_{task_short_name}.json'
-    result_path = f'/outputs/eval_{task_short_name}.json'
-
-    run_evaluator(runtime, env_llm_config, trajectory_path, result_path)
-
-    # finally, move trajectory file and evaluation result from mount path on host (temp dir) to outputs path
-    shutil.move(
-        os.path.join(temp_dir, f'traj_{task_short_name}.json'),
-        os.path.join(
-            os.path.abspath(args.outputs_path), f'traj_{task_short_name}.json'
-        ),
-    )
-    shutil.move(
-        os.path.join(temp_dir, f'eval_{task_short_name}.json'),
-        os.path.join(
-            os.path.abspath(args.outputs_path), f'eval_{task_short_name}.json'
-        ),
-    )

evaluation/benchmarks/the_agent_company/scripts/run_infer.sh

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-##################################################################################################
-# Adapted from https://github.com/TheAgentCompany/TheAgentCompany/blob/main/evaluation/run_eval.sh
-##################################################################################################
-
-# Exit on any error would be useful for debugging
-if [ -n "$DEBUG" ]; then
-    set -e
-fi
-
-# AGENT_LLM_CONFIG is the config name for the agent LLM
-# In config.toml, you should have a section with the name
-# [llm.<AGENT_LLM_CONFIG>], e.g. [llm.agent]
-AGENT_LLM_CONFIG="agent"
-
-# ENV_LLM_CONFIG is the config name for the environment LLM,
-# used by the NPCs and LLM-based evaluators.
-# In config.toml, you should have a section with the name
-# [llm.<ENV_LLM_CONFIG>], e.g. [llm.env]
-ENV_LLM_CONFIG="env"
-
-# OUTPUTS_PATH is the path to save trajectories and evaluation results
-OUTPUTS_PATH="outputs"
-
-# SERVER_HOSTNAME is the hostname of the server that hosts all the web services,
-# including RocketChat, ownCloud, GitLab, and Plane.
-SERVER_HOSTNAME="localhost"
-
-# VERSION is the version of the task images to use
-# If a task doesn't have a published image with this version, it will be skipped
-# 12/15/2024: this is for forward compatibility, in the case where we add new tasks
-# after the 1.0.0 release
-VERSION="1.0.0"
-
-# Parse command line arguments
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --agent-llm-config)
-            AGENT_LLM_CONFIG="$2"
-            shift 2
-            ;;
-        --env-llm-config)
-            ENV_LLM_CONFIG="$2"
-            shift 2
-            ;;
-        --outputs-path)
-            OUTPUTS_PATH="$2"
-            shift 2
-            ;;
-        --server-hostname)
-            SERVER_HOSTNAME="$2"
-            shift 2
-            ;;
-        --version)
-            VERSION="$2"
-            shift 2
-            ;;
-        *)
-            echo "Unknown argument: $1"
-            exit 1
-            ;;
-    esac
-done
-
-# Convert outputs_path to absolute path
-if [[ ! "$OUTPUTS_PATH" = /* ]]; then
-    # If path is not already absolute (doesn't start with /), make it absolute
-    OUTPUTS_PATH="$(cd "$(dirname "$OUTPUTS_PATH")" 2>/dev/null && pwd)/$(basename "$OUTPUTS_PATH")"
-fi
-
-echo "Using agent LLM config: $AGENT_LLM_CONFIG"
-echo "Using environment LLM config: $ENV_LLM_CONFIG"
-echo "Outputs path: $OUTPUTS_PATH"
-echo "Server hostname: $SERVER_HOSTNAME"
-echo "Version: $VERSION"
-
-echo "Downloading tasks.md..."
-rm -f tasks.md
-wget https://github.com/TheAgentCompany/TheAgentCompany/releases/download/${VERSION}/tasks.md
-
-while IFS= read -r task_image; do
-    docker pull $task_image
-
-    # Remove prefix using ## to remove longest matching pattern from start
-    task_name=${task_image##ghcr.io/theagentcompany/}
-
-    # Remove suffix using % to remove shortest matching pattern from end
-    task_name=${task_name%-image:*}
-    echo "Use task image $task_image, task name $task_name..."
-
-    # Check if evaluation file exists
-    if [ -f "$OUTPUTS_PATH/eval_${task_name}-image.json" ]; then
-        echo "Skipping $task_name - evaluation file already exists"
-        continue
-    fi
-
-    export PYTHONPATH=evaluation/benchmarks/the_agent_company:\$PYTHONPATH && \
-        poetry run python run_infer.py \
-            --agent-llm-config "$AGENT_LLM_CONFIG" \
-            --env-llm-config "$ENV_LLM_CONFIG" \
-            --outputs-path "$OUTPUTS_PATH" \
-            --server-hostname "$SERVER_HOSTNAME" \
-            --task-image-name "$task_image"
-
-    # Prune unused images and volumes
-    docker image rm "$task_image"
-    docker images "ghcr.io/all-hands-ai/runtime" -q | xargs -r docker rmi -f
-    docker volume prune -f
-    docker system prune -f
-done < tasks.md
-
-rm tasks.md
-
-echo "All evaluation completed successfully!"

evaluation/biocoder/README.md

@@ -4,14 +4,13 @@ Implements evaluation of agents on BioCoder from the BioCoder benchmark introduc
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
 
 ## BioCoder Docker Image
 
 In the openhands branch of the Biocoder repository, we have slightly modified our original Docker image to work with the OpenHands environment. In the Docker image are testing scripts (`/testing/start_test_openhands.py` and aux files in `/testing_files/`) to assist with evaluation. Additionally, we have installed all dependencies, including OpenJDK, mamba (with Python 3.6), and many system libraries. Notably, we have **not** packaged all repositories into the image, so they are downloaded at runtime.
 
 **Before first execution, pull our Docker image with the following command**
-
 ```bash
 docker pull public.ecr.aws/i5g0m1f6/eval_biocoder:v1.0
 ```
@@ -20,8 +19,9 @@ To reproduce this image, please see the Dockerfile_Openopenhands in the `biocode
 
 ## Start the evaluation
 
+
 ```bash
-./evaluation/benchmarks/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
 ```
 
 where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@@ -43,12 +43,11 @@ with current OpenHands version, then your command would be:
 ## Examples
 
 ```bash
-./evaluation/benchmarks/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
+./evaluation/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
 ```
 
 ## Reference
-
-```bibtex
+```
 @misc{tang2024biocoder,
       title={BioCoder: A Benchmark for Bioinformatics Code Generation with Large Language Models},
       author={Xiangru Tang and Bill Qian and Rick Gao and Jiakang Chen and Xinyun Chen and Mark Gerstein},

evaluation/biocoder/run_infer.py

@@ -8,7 +8,7 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.benchmarks.biocoder.utils import BiocoderData
+from evaluation.biocoder.utils import BiocoderData
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
@@ -328,8 +328,6 @@ if __name__ == '__main__':
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
-        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
-        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/biocoder/scripts/run_infer.sh

@@ -21,19 +21,19 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_openhands_version
+get_agent_version
 
 echo "AGENT: $AGENT"
-echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "AGENT_VERSION: $AGENT_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 echo "DATASET: $DATASET"
 
-COMMAND="poetry run python evaluation/benchmarks/biocoder/run_infer.py \
+COMMAND="poetry run python evaluation/biocoder/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 10 \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note ${OPENHANDS_VERSION}_${DATASET}"
+  --eval-note ${AGENT_VERSION}_${DATASET}"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"