Merge branch 'main' into fix-git-coauthorship-cli-runtime

Co-authored-by: openhands <openhands@all-hands.dev>

.github/CODEOWNERS

@@ -2,7 +2,7 @@
 # See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
 # Frontend code owners
-/frontend/ @rbren @amanape
+/frontend/ @amanape
 /openhands-ui/ @amanape
 
 # Evaluation code owners

.github/dependabot.yml

@@ -10,6 +10,9 @@ updates:
       pre-commit:
         patterns:
           - "pre-commit"
+      browsergym:
+        patterns:
+          - "browsergym*"
       mcp-packages:
         patterns:
           - "mcp"

.github/scripts/update_pr_description.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# This script updates the PR description with commands to run the PR locally
+# It adds both Docker and uvx commands
+
+# Get the branch name for the PR
+BRANCH_NAME=$(gh pr view "$PR_NUMBER" --json headRefName --jq .headRefName)
+
+# Define the Docker command
+DOCKER_RUN_COMMAND="docker run -it --rm \
+  -p 3000:3000 \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  --add-host host.docker.internal:host-gateway \
+  -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:${SHORT_SHA}-nikolaik \
+  --name openhands-app-${SHORT_SHA} \
+  docker.all-hands.dev/all-hands-ai/openhands:${SHORT_SHA}"
+
+# Define the uvx command
+UVX_RUN_COMMAND="uvx --python 3.12 --from git+https://github.com/All-Hands-AI/OpenHands@${BRANCH_NAME} openhands"
+
+# Get the current PR body
+PR_BODY=$(gh pr view "$PR_NUMBER" --json body --jq .body)
+
+# Prepare the new PR body with both commands
+if echo "$PR_BODY" | grep -q "To run this PR locally, use the following command:"; then
+  # For existing PR descriptions, use a more robust approach
+  # Split the PR body at the "To run this PR locally" section and replace everything after it
+  BEFORE_SECTION=$(echo "$PR_BODY" | sed '/To run this PR locally, use the following command:/,$d')
+  NEW_PR_BODY=$(cat <<EOF
+${BEFORE_SECTION}
+
+To run this PR locally, use the following command:
+
+GUI with Docker:
+\`\`\`
+${DOCKER_RUN_COMMAND}
+\`\`\`
+
+CLI with uvx:
+\`\`\`
+${UVX_RUN_COMMAND}
+\`\`\`
+EOF
+)
+else
+  # For new PR descriptions: use heredoc safely without indentation
+  NEW_PR_BODY=$(cat <<EOF
+$PR_BODY
+
+---
+
+To run this PR locally, use the following command:
+
+GUI with Docker:
+\`\`\`
+${DOCKER_RUN_COMMAND}
+\`\`\`
+
+CLI with uvx:
+\`\`\`
+${UVX_RUN_COMMAND}
+\`\`\`
+EOF
+)
+fi
+
+# Update the PR description
+echo "Updating PR description with Docker and uvx commands"
+gh pr edit "$PR_NUMBER" --body "$NEW_PR_BODY"

.github/workflows/e2e-tests.yml

@@ -0,0 +1,227 @@
+name: End-to-End Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    branches:
+      - main
+      - develop
+  workflow_dispatch:
+
+jobs:
+  e2e-tests:
+    if: contains(github.event.pull_request.labels.*.name, 'end-to-end') || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    env:
+      GITHUB_REPO_NAME: ${{ github.repository }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install poetry via pipx
+        uses: abatilo/actions-poetry@v4
+        with:
+          poetry-version: 2.1.3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'poetry'
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 xauth xvfb libgbm1 libasound2t64 netcat-openbsd
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+          cache-dependency-path: 'frontend/package-lock.json'
+
+      - name: Setup environment for end-to-end tests
+        run: |
+          # Create test results directory
+          mkdir -p test-results
+
+          # Create downloads directory for OpenHands (use a directory in the home folder)
+          mkdir -p $HOME/downloads
+          sudo chown -R $USER:$USER $HOME/downloads
+          sudo chmod -R 755 $HOME/downloads
+
+      - name: Build OpenHands
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          INSTALL_DOCKER: 1
+          RUNTIME: docker
+          FRONTEND_PORT: 12000
+          FRONTEND_HOST: 0.0.0.0
+          BACKEND_HOST: 0.0.0.0
+          BACKEND_PORT: 3000
+          ENABLE_BROWSER: true
+          INSTALL_PLAYWRIGHT: 1
+        run: |
+          # Fix poetry.lock file if needed
+          echo "Fixing poetry.lock file if needed..."
+          poetry lock
+
+          # Build OpenHands using make build
+          echo "Running make build..."
+          make build
+
+          # Install Chromium Headless Shell for Playwright (needed for pytest-playwright)
+          echo "Installing Chromium Headless Shell for Playwright..."
+          poetry run playwright install chromium-headless-shell
+
+          # Verify Playwright browsers are installed (for e2e tests only)
+          echo "Verifying Playwright browsers installation for e2e tests..."
+          BROWSER_CHECK=$(poetry run python tests/e2e/check_playwright.py 2>/dev/null)
+
+          if [ "$BROWSER_CHECK" != "chromium_found" ]; then
+            echo "ERROR: Chromium browser not found or not working for e2e tests"
+            echo "$BROWSER_CHECK"
+            exit 1
+          else
+            echo "Playwright browsers are properly installed for e2e tests."
+          fi
+
+          # Docker runtime will handle workspace directory creation
+
+          # Start the application using make run with custom parameters and reduced logging
+          echo "Starting OpenHands using make run..."
+          # Set environment variables to reduce logging verbosity
+          export PYTHONUNBUFFERED=1
+          export LOG_LEVEL=WARNING
+          export UVICORN_LOG_LEVEL=warning
+          export OPENHANDS_LOG_LEVEL=WARNING
+          FRONTEND_PORT=12000 FRONTEND_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0 make run > /tmp/openhands-e2e-test.log 2>&1 &
+
+          # Store the PID of the make run process
+          MAKE_PID=$!
+          echo "OpenHands started with PID: $MAKE_PID"
+
+          # Wait for the application to start
+          echo "Waiting for OpenHands to start..."
+          max_attempts=15
+          attempt=1
+
+          while [ $attempt -le $max_attempts ]; do
+            echo "Checking if OpenHands is running (attempt $attempt of $max_attempts)..."
+
+            # Check if the process is still running
+            if ! ps -p $MAKE_PID > /dev/null; then
+              echo "ERROR: OpenHands process has terminated unexpectedly"
+              echo "Last 50 lines of the log:"
+              tail -n 50 /tmp/openhands-e2e-test.log
+              exit 1
+            fi
+
+            # Check if frontend port is open
+            if nc -z localhost 12000; then
+              # Verify we can get HTML content
+              if curl -s http://localhost:12000 | grep -q "<html"; then
+                echo "SUCCESS: OpenHands is running and serving HTML content on port 12000"
+                break
+              else
+                echo "Port 12000 is open but not serving HTML content yet"
+              fi
+            else
+              echo "Frontend port 12000 is not open yet"
+            fi
+
+            # Show log output on each attempt
+            echo "Recent log output:"
+            tail -n 20 /tmp/openhands-e2e-test.log
+
+            # Wait before next attempt
+            echo "Waiting 10 seconds before next check..."
+            sleep 10
+            attempt=$((attempt + 1))
+
+            # Exit if we've reached the maximum number of attempts
+            if [ $attempt -gt $max_attempts ]; then
+              echo "ERROR: OpenHands failed to start after $max_attempts attempts"
+              echo "Last 50 lines of the log:"
+              tail -n 50 /tmp/openhands-e2e-test.log
+              exit 1
+            fi
+          done
+
+          # Final verification that the app is running
+          if ! nc -z localhost 12000 || ! curl -s http://localhost:12000 | grep -q "<html"; then
+            echo "ERROR: OpenHands is not running properly on port 12000"
+            echo "Last 50 lines of the log:"
+            tail -n 50 /tmp/openhands-e2e-test.log
+            exit 1
+          fi
+
+          # Print success message
+          echo "OpenHands is running successfully on port 12000"
+
+      - name: Run end-to-end tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+        run: |
+          # Check if the application is running
+          if ! nc -z localhost 12000; then
+            echo "ERROR: OpenHands is not running on port 12000"
+            echo "Last 50 lines of the log:"
+            tail -n 50 /tmp/openhands-e2e-test.log
+            exit 1
+          fi
+
+          # Run the tests with detailed output
+          cd tests/e2e
+          poetry run python -m pytest \
+            test_settings.py::test_github_token_configuration \
+            test_conversation.py::test_conversation_start \
+            test_browsing_catchphrase.py::test_browsing_catchphrase \
+            -v --no-header --capture=no --timeout=900
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: tests/e2e/test-results/
+          retention-days: 30
+
+      - name: Upload OpenHands logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: openhands-logs
+          path: |
+            /tmp/openhands-e2e-test.log
+            /tmp/openhands-e2e-build.log
+            /tmp/openhands-backend.log
+            /tmp/openhands-frontend.log
+            /tmp/backend-health-check.log
+            /tmp/frontend-check.log
+            /tmp/vite-config.log
+            /tmp/makefile-contents.log
+          retention-days: 30
+
+      - name: Cleanup
+        if: always()
+        run: |
+          # Stop OpenHands processes
+          echo "Stopping OpenHands processes..."
+          pkill -f "python -m openhands.server" || true
+          pkill -f "npm run dev" || true
+          pkill -f "make run" || true
+
+          # Print process status for debugging
+          echo "Checking if any OpenHands processes are still running:"
+          ps aux | grep -E "openhands|npm run dev" || true

.github/workflows/fe-unit-tests.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: blacksmith-4vcpu-ubuntu-2204
     strategy:
       matrix:
-        node-version: 22
+        node-version: [22]
       fail-fast: true
     steps:
       - name: Checkout

.github/workflows/ghcr-build.yml

@@ -225,7 +225,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --durations=10
+          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
         env:
           DEBUG: "1"
 
@@ -284,7 +284,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --durations=10
+          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
         env:
           DEBUG: "1"
 
@@ -332,29 +332,5 @@ jobs:
           SHORT_SHA: ${{ steps.short_sha.outputs.SHORT_SHA }}
         shell: bash
         run: |
-          echo "updating PR description"
-          DOCKER_RUN_COMMAND="docker run -it --rm \
-            -p 3000:3000 \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            --add-host host.docker.internal:host-gateway \
-            -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:$SHORT_SHA-nikolaik \
-            --name openhands-app-$SHORT_SHA \
-            docker.all-hands.dev/all-hands-ai/openhands:$SHORT_SHA"
-
-          PR_BODY=$(gh pr view $PR_NUMBER --json body --jq .body)
-
-          if echo "$PR_BODY" | grep -q "To run this PR locally, use the following command:"; then
-            UPDATED_PR_BODY=$(echo "${PR_BODY}" | sed -E "s|docker run -it --rm.*|$DOCKER_RUN_COMMAND|")
-          else
-            UPDATED_PR_BODY="${PR_BODY}
-
-          ---
-
-          To run this PR locally, use the following command:
-          \`\`\`
-          $DOCKER_RUN_COMMAND
-          \`\`\`"
-          fi
-
-          echo "updated body: $UPDATED_PR_BODY"
-          gh pr edit $PR_NUMBER --body "$UPDATED_PR_BODY"
+          echo "Updating PR description with Docker and uvx commands"
+          bash ${GITHUB_WORKSPACE}/.github/scripts/update_pr_description.sh

.github/workflows/lint-fix.yml

@@ -29,6 +29,12 @@ jobs:
         run: |
           cd frontend
           npm install --frozen-lockfile
+      - name: Generate i18n and route types
+        run: |
+          cd frontend
+          npm run make-i18n
+          npx react-router typegen || true
+
       - name: Fix frontend lint issues
         run: |
           cd frontend
@@ -45,7 +51,7 @@ jobs:
           git config --local user.email "openhands@all-hands.dev"
           git config --local user.name "OpenHands Bot"
           git add -A
-          git commit -m "🤖 Auto-fix frontend linting issues"
+          git commit -m "🤖 Auto-fix frontend linting issues" --no-verify
           git push
 
   # Python lint fixes
@@ -87,5 +93,5 @@ jobs:
           git config --local user.email "openhands@all-hands.dev"
           git config --local user.name "OpenHands Bot"
           git add -A
-          git commit -m "🤖 Auto-fix Python linting issues"
+          git commit -m "🤖 Auto-fix Python linting issues" --no-verify
           git push

.github/workflows/py-tests.yml

@@ -48,11 +48,9 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Unit Tests
-        run: poetry run pytest --forked -n auto -svv ./tests/unit
+        run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest --forked -n auto -svv ./tests/unit
       - name: Run Runtime Tests with CLIRuntime
-        run: TEST_RUNTIME=cli poetry run pytest -svv tests/runtime/test_bash.py
-      - name: Run E2E Tests
-        run: poetry run pytest -svv tests/e2e
+        run: PYTHONPATH=".:$PYTHONPATH" TEST_RUNTIME=cli poetry run pytest -svv tests/runtime/test_bash.py
 
   # Run specific Windows python tests
   test-on-windows:
@@ -75,11 +73,13 @@ jobs:
       - name: Install Python dependencies using Poetry
         run: poetry install --with dev,test,runtime
       - name: Run Windows unit tests
-        run: poetry run pytest -svv tests/unit/test_windows_bash.py
+        run: poetry run pytest -svv tests/unit/runtime/utils/test_windows_bash.py
         env:
+          PYTHONPATH: ".;$env:PYTHONPATH"
           DEBUG: "1"
       - name: Run Windows runtime tests with LocalRuntime
         run: $env:TEST_RUNTIME="local"; poetry run pytest -svv tests/runtime/test_bash.py
         env:
+          PYTHONPATH: ".;$env:PYTHONPATH"
           TEST_RUNTIME: local
           DEBUG: "1"

.github/workflows/run-eval.yml

@@ -1,56 +1,135 @@
-# Run evaluation on a PR
+# Run evaluation on a PR, after releases, or manually
 name: Run Eval
 
-# Runs when a PR is labeled with one of the "run-eval-" labels
+# Runs when a PR is labeled with one of the "run-eval-" labels, after releases, or manually triggered
 on:
   pull_request:
     types: [labeled]
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to evaluate'
+        required: true
+        default: 'main'
+      eval_instances:
+        description: 'Number of evaluation instances'
+        required: true
+        default: '50'
+        type: choice
+        options:
+          - '1'
+          - '2'
+          - '50'
+          - '100'
+      reason:
+        description: 'Reason for manual trigger'
+        required: false
+        default: ''
+
+env:
+  # Environment variable for the master GitHub issue number where all evaluation results will be commented
+  # This should be set to the issue number where you want all evaluation results to be posted
+  MASTER_EVAL_ISSUE_NUMBER: ${{ vars.MASTER_EVAL_ISSUE_NUMBER || '0' }}
 
 jobs:
   trigger-job:
     name: Trigger remote eval job
-    if: ${{ github.event.label.name == 'run-eval-1' || github.event.label.name == 'run-eval-2' || github.event.label.name == 'run-eval-50' || github.event.label.name == 'run-eval-100' }}
+    if: ${{ (github.event_name == 'pull_request' && (github.event.label.name == 'run-eval-1' || github.event.label.name == 'run-eval-2' || github.event.label.name == 'run-eval-50' || github.event.label.name == 'run-eval-100')) || github.event_name == 'release' || github.event_name == 'workflow_dispatch' }}
     runs-on: blacksmith-4vcpu-ubuntu-2204
 
     steps:
-      - name: Checkout PR branch
+      - name: Checkout branch
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref }}
+          ref: ${{ github.event_name == 'pull_request' && github.head_ref || (github.event_name == 'workflow_dispatch' && github.event.inputs.branch) || github.ref }}
 
-      - name: Trigger remote job
-        env:
-          PR_BRANCH: ${{ github.head_ref }}
+      - name: Set evaluation parameters
+        id: eval_params
         run: |
           REPO_URL="https://github.com/${{ github.repository }}"
           echo "Repository URL: $REPO_URL"
-          echo "PR Branch: $PR_BRANCH"
 
-          if [[ "${{ github.event.label.name }}" == "run-eval-1" ]]; then
-            EVAL_INSTANCES="1"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-2" ]]; then
-            EVAL_INSTANCES="2"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-50" ]]; then
+          # Determine branch based on trigger type
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            EVAL_BRANCH="${{ github.head_ref }}"
+            echo "PR Branch: $EVAL_BRANCH"
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            EVAL_BRANCH="${{ github.event.inputs.branch }}"
+            echo "Manual Branch: $EVAL_BRANCH"
+          else
+            # For release events, use the tag name or main branch
+            EVAL_BRANCH="${{ github.ref_name }}"
+            echo "Release Branch/Tag: $EVAL_BRANCH"
+          fi
+
+          # Determine evaluation instances based on trigger type
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            if [[ "${{ github.event.label.name }}" == "run-eval-1" ]]; then
+              EVAL_INSTANCES="1"
+            elif [[ "${{ github.event.label.name }}" == "run-eval-2" ]]; then
+              EVAL_INSTANCES="2"
+            elif [[ "${{ github.event.label.name }}" == "run-eval-50" ]]; then
+              EVAL_INSTANCES="50"
+            elif [[ "${{ github.event.label.name }}" == "run-eval-100" ]]; then
+              EVAL_INSTANCES="100"
+            fi
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            EVAL_INSTANCES="${{ github.event.inputs.eval_instances }}"
+          else
+            # For release events, default to 50 instances
             EVAL_INSTANCES="50"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-100" ]]; then
-            EVAL_INSTANCES="100"
+          fi
+
+          echo "Evaluation instances: $EVAL_INSTANCES"
+          echo "repo_url=$REPO_URL" >> $GITHUB_OUTPUT
+          echo "eval_branch=$EVAL_BRANCH" >> $GITHUB_OUTPUT
+          echo "eval_instances=$EVAL_INSTANCES" >> $GITHUB_OUTPUT
+
+      - name: Trigger remote job
+        run: |
+          # Determine PR number for the remote evaluation system
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            PR_NUMBER="${{ github.event.pull_request.number }}"
+          else
+            # For non-PR triggers, use the master issue number as PR number
+            PR_NUMBER="${{ env.MASTER_EVAL_ISSUE_NUMBER }}"
           fi
 
           curl -X POST \
             -H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
             -H "Accept: application/vnd.github+json" \
-            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
+            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${{ steps.eval_params.outputs.repo_url }}\", \"github-branch\": \"${{ steps.eval_params.outputs.eval_branch }}\", \"pr-number\": \"${PR_NUMBER}\", \"eval-instances\": \"${{ steps.eval_params.outputs.eval_instances }}\"}}" \
             https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
 
           # Send Slack message
-          PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
-          slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            TRIGGER_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
+            slack_text="PR $TRIGGER_URL has triggered evaluation on ${{ steps.eval_params.outputs.eval_instances }} instances..."
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            TRIGGER_URL="https://github.com/${{ github.repository }}/releases/tag/${{ github.ref_name }}"
+            slack_text="Release $TRIGGER_URL has triggered evaluation on ${{ steps.eval_params.outputs.eval_instances }} instances..."
+          else
+            TRIGGER_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            slack_text="Manual trigger (${{ github.event.inputs.reason || 'No reason provided' }}) has triggered evaluation on ${{ steps.eval_params.outputs.eval_instances }} instances for branch ${{ steps.eval_params.outputs.eval_branch }}..."
+          fi
+
           curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
             https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
 
-      - name: Comment on PR
+      - name: Comment on issue/PR
         uses: KeisukeYamashita/create-comment@v1
         with:
+          # For PR triggers, comment on the PR. For other triggers, comment on the master issue
+          number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || env.MASTER_EVAL_ISSUE_NUMBER }}
           unique: false
           comment: |
-            Running evaluation on the PR. Once eval is done, the results will be posted.
+            **Evaluation Triggered**
+
+            **Trigger:** ${{ github.event_name == 'pull_request' && format('Pull Request #{0}', github.event.pull_request.number) || (github.event_name == 'release' && 'Release') || format('Manual Trigger: {0}', github.event.inputs.reason || 'No reason provided') }}
+            **Branch:** ${{ steps.eval_params.outputs.eval_branch }}
+            **Instances:** ${{ steps.eval_params.outputs.eval_instances }}
+            **Commit:** ${{ github.sha }}
+
+            Running evaluation on the specified branch. Once eval is done, the results will be posted here.

.github/workflows/stale.yml

@@ -12,11 +12,11 @@ jobs:
     steps:
       - uses: actions/stale@v9
         with:
-          stale-issue-message: 'This issue is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
-          stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
-          days-before-stale: 30
+          stale-issue-message: 'This issue is stale because it has been open for 40 days with no activity. Remove the stale label or leave a comment, otherwise it will be closed in 10 days.'
+          stale-pr-message: 'This PR is stale because it has been open for 40 days with no activity. Remove the stale label or leave a comment, otherwise it will be closed in 10 days.'
+          days-before-stale: 40
           exempt-issue-labels: 'roadmap'
-          close-issue-message: 'This issue was closed because it has been stalled for over 30 days with no activity.'
-          close-pr-message: 'This PR was closed because it has been stalled for over 30 days with no activity.'
-          days-before-close: 7
+          close-issue-message: 'This issue was automatically closed due to 50 days of inactivity. We do this to help keep the issues somewhat manageable and focus on active issues.'
+          close-pr-message: 'This PR was closed because it had no activity for 50 days. If you feel this was closed in error, and you would like to continue the PR, please resubmit or let us know.'
+          days-before-close: 10
           operations-per-run: 150

.github/workflows/welcome-good-first-issue.yml

@@ -0,0 +1,50 @@
+name: Welcome Good First Issue
+
+on:
+  issues:
+    types: [labeled]
+
+permissions:
+  issues: write
+
+jobs:
+  comment-on-good-first-issue:
+    if: github.event.label.name == 'good first issue'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if welcome comment already exists
+        id: check_comment
+        uses: actions/github-script@v7
+        with:
+          result-encoding: string
+          script: |
+            const issueNumber = context.issue.number;
+            const comments = await github.rest.issues.listComments({
+              ...context.repo,
+              issue_number: issueNumber
+            });
+
+            const alreadyCommented = comments.data.some(
+              (comment) =>
+                comment.body.includes('<!-- auto-comment:good-first-issue -->')
+            );
+
+            return alreadyCommented ? 'true' : 'false';
+
+      - name: Leave welcome comment
+        if: steps.check_comment.outputs.result == 'false'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const repoUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}`;
+
+            await github.rest.issues.createComment({
+              ...context.repo,
+              issue_number: context.issue.number,
+              body: "🙌 **Hey there, future contributor!** 🙌\n\n" +
+                    "This issue has been labeled as **good first issue**, which means it's a great place to get started with the OpenHands project.\n\n" +
+                    "If you're interested in working on it, feel free to! No need to ask for permission.\n\n" +
+                    "Be sure to check out our [development setup guide](" + repoUrl + "/blob/main/Development.md) to get your environment set up, and follow our [contribution guidelines](" + repoUrl + "/blob/main/CONTRIBUTING.md) when you're ready to submit a fix.\n\n" +
+                    "🙌 Happy hacking! 🙌\n\n" +
+                    "<!-- auto-comment:good-first-issue -->"
+            });

.gitignore

@@ -254,3 +254,8 @@ containers/runtime/Dockerfile
 containers/runtime/project.tar.gz
 containers/runtime/code
 **/node_modules/
+
+# test results
+test-results
+
+.eval_sessions

.openhands/microagents/repo.md

@@ -15,8 +15,6 @@ make build && make run FRONTEND_PORT=12000 FRONTEND_HOST=0.0.0.0 BACKEND_HOST=0.
 
 IMPORTANT: Before making any changes to the codebase, ALWAYS run `make install-pre-commit-hooks` to ensure pre-commit hooks are properly installed.
 
-
-
 Before pushing any changes, you MUST ensure that any lint errors or simple test errors have been fixed.
 
 * If you've made changes to the backend, you should run `pre-commit run --config ./dev_config/python/.pre-commit-config.yaml` (this will run on staged files).
@@ -32,6 +30,12 @@ then re-run the command to ensure it passes. Common issues include:
 - Trailing whitespace
 - Missing newlines at end of files
 
+## Git Best Practices
+
+- Prefer specific `git add <filename>` instead of `git add .` to avoid accidentally staging unintended files
+- Be especially careful with `git reset --hard` after staging files, as it will remove accidentally staged files
+- When remote has new changes, use `git fetch upstream && git rebase upstream/<branch>` on the same branch
+
 ## Repository Structure
 Backend:
 - Located in the `openhands` directory
@@ -83,6 +87,8 @@ VSCode Extension:
 
 If you are starting a pull request (PR), please follow the template in `.github/pull_request_template.md`.
 
+If you need to add labels when opening a PR, check the existing labels defined on that repository and select from existing ones. Do not invent your own labels.
+
 ## Implementation Details
 
 These details may or may not be useful for your current task.
@@ -138,6 +144,35 @@ Your specialized knowledge and instructions here...
      - Add the setting to the `Settings` model in `openhands/storage/data_models/settings.py`
      - Update any relevant backend code to apply the setting (e.g., in session creation)
 
+#### Settings UI Patterns:
+
+There are two main patterns for saving settings in the OpenHands frontend:
+
+**Pattern 1: Entity-based Resources (Immediate Save)**
+- Used for: API Keys, Secrets, MCP Servers
+- Behavior: Changes are saved immediately when user performs actions (add/edit/delete)
+- Implementation:
+  - No "Save Changes" button
+  - No local state management or `isDirty` tracking
+  - Uses dedicated mutation hooks for each operation (e.g., `use-add-mcp-server.ts`, `use-delete-mcp-server.ts`)
+  - Each mutation triggers immediate API call with query invalidation for UI updates
+  - Example: MCP settings, API Keys & Secrets tabs
+- Benefits: Simpler UX, no risk of losing changes, consistent with modern web app patterns
+
+**Pattern 2: Form-based Settings (Manual Save)**
+- Used for: Application settings, LLM configuration
+- Behavior: Changes are accumulated locally and saved when user clicks "Save Changes"
+- Implementation:
+  - Has "Save Changes" button that becomes enabled when changes are detected
+  - Uses local state management with `isDirty` tracking
+  - Uses `useSaveSettings` hook to save all changes at once
+  - Example: LLM tab, Application tab
+- Benefits: Allows bulk changes, explicit save action, can validate all fields before saving
+
+**When to use each pattern:**
+- Use Pattern 1 (Immediate Save) for entity management where each item is independent
+- Use Pattern 2 (Manual Save) for configuration forms where settings are interdependent or need validation
+
 ### Adding New LLM Models
 
 To add a new LLM model to OpenHands, you need to update multiple files across both frontend and backend:

.openhands/pre-commit.sh

@@ -1,56 +1,158 @@
 #!/bin/bash
 
 echo "Running OpenHands pre-commit hook..."
+echo "This hook runs selective linting based on changed files."
 
 # Store the exit code to return at the end
 # This allows us to be additive to existing pre-commit hooks
 EXIT_CODE=0
 
-# Check if frontend directory has changed
-frontend_changes=$(git diff --cached --name-only | grep "^frontend/")
-if [ -n "$frontend_changes" ]; then
-    echo "Frontend changes detected. Running frontend checks..."
+# Get the list of staged files
+STAGED_FILES=$(git diff --cached --name-only)
 
-    # Check if frontend directory exists
-    if [ -d "frontend" ]; then
-        # Change to frontend directory
-        cd frontend || exit 1
+# Check if any files match specific patterns
+has_frontend_changes=false
+has_backend_changes=false
+has_vscode_changes=false
 
-        # Run lint:fix
-        echo "Running npm lint:fix..."
-        npm run lint:fix
+# Check each file individually to avoid issues with grep
+for file in $STAGED_FILES; do
+    if [[ $file == frontend/* ]]; then
+        has_frontend_changes=true
+    elif [[ $file == openhands/* || $file == evaluation/* || $file == tests/* ]]; then
+        has_backend_changes=true
+        # Check for VSCode extension changes (subset of backend changes)
+        if [[ $file == openhands/integrations/vscode/* ]]; then
+            has_vscode_changes=true
+        fi
+    fi
+done
+
+echo "Analyzing changes..."
+echo "- Frontend changes: $has_frontend_changes"
+echo "- Backend changes: $has_backend_changes"
+echo "- VSCode extension changes: $has_vscode_changes"
+
+# Run frontend linting if needed
+if [ "$has_frontend_changes" = true ]; then
+    # Check if we're in a CI environment or if frontend dependencies are missing
+    if [ -n "$CI" ] || ! command -v react-router &> /dev/null || ! command -v vitest &> /dev/null; then
+        echo "Skipping frontend checks (CI environment or missing dependencies detected)."
+        echo "WARNING: Frontend files have changed but frontend checks are being skipped."
+        echo "Please run 'make lint-frontend' manually before submitting your PR."
+    else
+        echo "Running frontend linting..."
+        make lint-frontend
         if [ $? -ne 0 ]; then
             echo "Frontend linting failed. Please fix the issues before committing."
             EXIT_CODE=1
+        else
+            echo "Frontend linting checks passed!"
         fi
 
-        # Run build
-        echo "Running npm build..."
-        npm run build
-        if [ $? -ne 0 ]; then
-            echo "Frontend build failed. Please fix the issues before committing."
-            EXIT_CODE=1
-        fi
+        # Run additional frontend checks
+        if [ -d "frontend" ]; then
+            echo "Running additional frontend checks..."
+            cd frontend || exit 1
 
-        # Run tests
-        echo "Running npm test..."
-        npm test
-        if [ $? -ne 0 ]; then
-            echo "Frontend tests failed. Please fix the failing tests before committing."
-            EXIT_CODE=1
-        fi
+            # Run build
+            echo "Running npm build..."
+            npm run build
+            if [ $? -ne 0 ]; then
+                echo "Frontend build failed. Please fix the issues before committing."
+                EXIT_CODE=1
+            fi
 
-        # Return to the original directory
-        cd ..
+            # Run tests
+            echo "Running npm test..."
+            npm test
+            if [ $? -ne 0 ]; then
+                echo "Frontend tests failed. Please fix the failing tests before committing."
+                EXIT_CODE=1
+            fi
 
-        if [ $EXIT_CODE -eq 0 ]; then
-            echo "Frontend checks passed!"
+            cd ..
         fi
-    else
-        echo "Frontend directory not found. Skipping frontend checks."
     fi
 else
-    echo "No frontend changes detected. Skipping frontend checks."
+    echo "Skipping frontend checks (no frontend changes detected)."
+fi
+
+# Run backend linting if needed
+if [ "$has_backend_changes" = true ]; then
+    echo "Running backend linting..."
+    make lint-backend
+    if [ $? -ne 0 ]; then
+        echo "Backend linting failed. Please fix the issues before committing."
+        EXIT_CODE=1
+    else
+        echo "Backend linting checks passed!"
+    fi
+else
+    echo "Skipping backend checks (no backend changes detected)."
+fi
+
+# Run VSCode extension checks if needed
+if [ "$has_vscode_changes" = true ]; then
+    # Check if we're in a CI environment
+    if [ -n "$CI" ]; then
+        echo "Skipping VSCode extension checks (CI environment detected)."
+        echo "WARNING: VSCode extension files have changed but checks are being skipped."
+        echo "Please run VSCode extension checks manually before submitting your PR."
+    else
+        echo "Running VSCode extension checks..."
+        if [ -d "openhands/integrations/vscode" ]; then
+            cd openhands/integrations/vscode || exit 1
+
+            echo "Running npm lint:fix..."
+            npm run lint:fix
+            if [ $? -ne 0 ]; then
+                echo "VSCode extension linting failed. Please fix the issues before committing."
+                EXIT_CODE=1
+            else
+                echo "VSCode extension linting passed!"
+            fi
+
+            echo "Running npm typecheck..."
+            npm run typecheck
+            if [ $? -ne 0 ]; then
+                echo "VSCode extension type checking failed. Please fix the issues before committing."
+                EXIT_CODE=1
+            else
+                echo "VSCode extension type checking passed!"
+            fi
+
+            echo "Running npm compile..."
+            npm run compile
+            if [ $? -ne 0 ]; then
+                echo "VSCode extension compilation failed. Please fix the issues before committing."
+                EXIT_CODE=1
+            else
+                echo "VSCode extension compilation passed!"
+            fi
+
+            cd ../../..
+        fi
+    fi
+else
+    echo "Skipping VSCode extension checks (no VSCode extension changes detected)."
+fi
+
+# If no specific code changes detected, run basic checks
+if [ "$has_frontend_changes" = false ] && [ "$has_backend_changes" = false ]; then
+    echo "No specific code changes detected. Running basic checks..."
+    if [ -n "$STAGED_FILES" ]; then
+        # Run only basic pre-commit hooks for non-code files
+        poetry run pre-commit run --files $(echo "$STAGED_FILES" | tr '\n' ' ') --hook-stage commit --config ./dev_config/python/.pre-commit-config.yaml
+        if [ $? -ne 0 ]; then
+            echo "Basic checks failed. Please fix the issues before committing."
+            EXIT_CODE=1
+        else
+            echo "Basic checks passed!"
+        fi
+    else
+        echo "No files changed. Skipping basic checks."
+    fi
 fi
 
 # Run any existing pre-commit hooks that might have been installed by the user

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: ^(docs/|modules/|python/|openhands-ui/|third_party/)
+      - id: end-of-file-fixer
+        exclude: ^(docs/|modules/|python/|openhands-ui/|third_party/)
+      - id: check-yaml
+        args: ["--allow-multiple-documents"]
+      - id: debug-statements
+
+  - repo: https://github.com/tox-dev/pyproject-fmt
+    rev: v2.5.1
+    hooks:
+      - id: pyproject-fmt
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.24.1
+    hooks:
+      - id: validate-pyproject
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.11.8
+    hooks:
+      # Run the linter.
+      - id: ruff
+        entry: ruff check --config dev_config/python/ruff.toml
+        types_or: [python, pyi, jupyter]
+        args: [--fix, --unsafe-fixes]
+        exclude: third_party/
+      # Run the formatter.
+      - id: ruff-format
+        entry: ruff format --config dev_config/python/ruff.toml
+        types_or: [python, pyi, jupyter]
+        exclude: third_party/
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [types-requests, types-setuptools, types-pyyaml, types-toml, types-docker, types-Markdown, pydantic, lxml]
+        # To see gaps add `--html-report mypy-report/`
+        entry: mypy --config-file dev_config/python/mypy.ini openhands/
+        always_run: true
+        pass_filenames: false

CREDITS.md

@@ -16,7 +16,7 @@ OpenHands includes and adapts the following open source projects. We are gratefu
    - License: Apache License 2.0
    - Description: AI pair programming tool. OpenHands has adapted and integrated its linter module for code-related tasks in [`agentskills utilities`](https://github.com/All-Hands-AI/OpenHands/tree/main/openhands/runtime/plugins/agent_skills/utils/aider)
 
-#### [Browser-Use](https://github.com/browser-use/browser-use)
+#### [BrowserGym](https://github.com/ServiceNow/BrowserGym)
    - License: Apache License 2.0
    - Description: Adapted in implementing the browsing agent
 

Development.md

@@ -34,7 +34,7 @@ _Dev Container: Reopen in Container_ command from the Command Palette
 
 #### Develop without sudo access
 
-If you want to develop without system admin/sudo access to upgrade/install `Python` and/or `NodeJs`, you can use
+If you want to develop without system admin/sudo access to upgrade/install `Python` and/or `NodeJS`, you can use
 `conda` or `mamba` to manage the packages for you:
 
 ```bash
@@ -71,7 +71,7 @@ This command will prompt you to enter the LLM API key, model name, and other var
 tailored to your specific needs. Note that the model name will apply only when you run headless. If you use the UI,
 please set the model in the UI.
 
-Note: If you have previously run OpenHands using the docker command, you may have already set some environmental
+Note: If you have previously run OpenHands using the docker command, you may have already set some environment
 variables in your terminal. The final configurations are set from highest to lowest priority:
 Environment variables > config.toml variables > default variables
 
@@ -154,12 +154,12 @@ poetry run pytest ./tests/unit/test_*.py
 1. Add your dependency in `pyproject.toml` or use `poetry add xxx`.
 2. Update the poetry.lock file via `poetry lock --no-update`.
 
-### 9. Use existing Docker image
+### 10. Use existing Docker image
 
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker
 container image by setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
 
-Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.49-nikolaik`
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.54-nikolaik`
 
 ## Develop inside Docker container
 

Makefile

@@ -3,10 +3,10 @@ SHELL=/usr/bin/env bash
 
 # Variables
 BACKEND_HOST ?= "127.0.0.1"
-BACKEND_PORT = 3000
+BACKEND_PORT ?= 3000
 BACKEND_HOST_PORT = "$(BACKEND_HOST):$(BACKEND_PORT)"
 FRONTEND_HOST ?= "127.0.0.1"
-FRONTEND_PORT = 3001
+FRONTEND_PORT ?= 3001
 DEFAULT_WORKSPACE_DIR = "./workspace"
 DEFAULT_MODEL = "gpt-4o"
 CONFIG_FILE = config.toml
@@ -174,7 +174,7 @@ install-python-dependencies:
 	fi
 	@echo "$(GREEN)Python dependencies installed successfully.$(RESET)"
 
-install-frontend-dependencies:
+install-frontend-dependencies: check-npm check-nodejs
 	@echo "$(YELLOW)Setting up frontend environment...$(RESET)"
 	@echo "$(YELLOW)Detect Node.js version...$(RESET)"
 	@cd frontend && node ./scripts/detect-node-version.js
@@ -182,17 +182,17 @@ install-frontend-dependencies:
 	@cd frontend && npm install
 	@echo "$(GREEN)Frontend dependencies installed successfully.$(RESET)"
 
-install-pre-commit-hooks:
+install-pre-commit-hooks: check-python check-poetry install-python-dependencies
 	@echo "$(YELLOW)Installing pre-commit hooks...$(RESET)"
 	@git config --unset-all core.hooksPath || true
 	@poetry run pre-commit install --config $(PRE_COMMIT_CONFIG_PATH)
 	@echo "$(GREEN)Pre-commit hooks installed successfully.$(RESET)"
 
-lint-backend:
+lint-backend: install-pre-commit-hooks
 	@echo "$(YELLOW)Running linters...$(RESET)"
 	@poetry run pre-commit run --all-files --show-diff-on-failure --config $(PRE_COMMIT_CONFIG_PATH)
 
-lint-frontend:
+lint-frontend: install-frontend-dependencies
 	@echo "$(YELLOW)Running linters for frontend...$(RESET)"
 	@cd frontend && npm run lint
 

README.md

@@ -52,37 +52,63 @@ which comes with $20 in free credits for new users.
 
 ## 💻 Running OpenHands Locally
 
-OpenHands can also run on your local system using Docker.
-See the [Running OpenHands](https://docs.all-hands.dev/usage/installation) guide for
-system requirements and more information.
+### Option 1: CLI Launcher (Recommended)
 
-> [!WARNING]
-> On a public network? See our [Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)
-> to secure your deployment by restricting network binding and implementing additional security measures.
+The easiest way to run OpenHands locally is using the CLI launcher with [uv](https://docs.astral.sh/uv/). This provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers.
 
+**Install uv** (if you haven't already):
+
+See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+**Launch OpenHands**:
+```bash
+# Launch the GUI server
+uvx --python 3.12 --from openhands-ai openhands serve
+
+# Or launch the CLI
+uvx --python 3.12 --from openhands-ai openhands
+```
+
+You'll find OpenHands running at [http://localhost:3000](http://localhost:3000) (for GUI mode)!
+
+### Option 2: Docker
+
+<details>
+<summary>Click to expand Docker command</summary>
+
+You can also run OpenHands directly with Docker:
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49
+    docker.all-hands.dev/all-hands-ai/openhands:0.54
 ```
 
+</details>
+
 > **Note**: If you used OpenHands before version 0.44, you may want to run `mv ~/.openhands-state ~/.openhands` to migrate your conversation history to the new location.
 
-You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
+> [!WARNING]
+> On a public network? See our [Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)
+> to secure your deployment by restricting network binding and implementing additional security measures.
+
+### Getting Started
 
 When you open the application, you'll be asked to choose an LLM provider and add an API key.
 [Anthropic's Claude Sonnet 4](https://www.anthropic.com/api) (`anthropic/claude-sonnet-4-20250514`)
 works best, but you have [many options](https://docs.all-hands.dev/usage/llms).
 
+See the [Running OpenHands](https://docs.all-hands.dev/usage/installation) guide for
+system requirements and more information.
+
 ## 💡 Other ways to run OpenHands
 
 > [!WARNING]
@@ -93,8 +119,8 @@ works best, but you have [many options](https://docs.all-hands.dev/usage/llms).
 > [OpenHands Cloud Helm Chart](https://github.com/all-Hands-AI/OpenHands-cloud)
 
 You can [connect OpenHands to your local filesystem](https://docs.all-hands.dev/usage/runtimes/docker#connecting-to-your-filesystem),
-run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/usage/how-to/headless-mode),
 interact with it via a [friendly CLI](https://docs.all-hands.dev/usage/how-to/cli-mode),
+run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/usage/how-to/headless-mode),
 or run it on tagged issues with [a github action](https://docs.all-hands.dev/usage/how-to/github-action).
 
 Visit [Running OpenHands](https://docs.all-hands.dev/usage/installation) for more information and setup instructions.

README_CN.md

@@ -51,17 +51,17 @@ OpenHands也可以使用Docker在本地系统上运行。
 
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49
+    docker.all-hands.dev/all-hands-ai/openhands:0.54
 ```
 
 > **注意**: 如果您在0.44版本之前使用过OpenHands，您可能需要运行 `mv ~/.openhands-state ~/.openhands` 来将对话历史迁移到新位置。

README_JA.md

@@ -42,17 +42,17 @@ OpenHandsはDockerを利用してローカル環境でも実行できます。
 > 公共ネットワークで実行していますか？[Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)を参照して、ネットワークバインディングの制限や追加のセキュリティ対策を実施してください。
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49
+    docker.all-hands.dev/all-hands-ai/openhands:0.54
 ```
 
 **注**: バージョン0.44以前のOpenHandsを使用していた場合は、会話履歴を移行するために `mv ~/.openhands-state ~/.openhands` を実行してください。

browser_refactor_gotchas.md

@@ -1,228 +0,0 @@
-# Browser Refactoring Gotchas and Findings
-
-## Initial Exploration
-
-### Current Browser Integration Points Found
-
-1. **Core Browser Environment**: `openhands/runtime/browser/browser_use_env.py` ✅
-2. **Action Definitions**: `openhands/events/action/browse.py`
-3. **Observation Definitions**: `openhands/events/observation/browse.py`
-4. **Agent Implementations**:
-   - `openhands/agenthub/browsing_agent/`
-   - `openhands/agenthub/visualbrowsing_agent/`
-   - `openhands/agenthub/codeact_agent/tools/browser.py`
-5. **Configuration**: `openhands/core/config/sandbox_config.py` ✅
-6. **Evaluation Benchmarks**: Various evaluation scripts ✅
-
-### Key Findings
-
-- Browser-Use uses direct Playwright-based browser control
-- Multiprocessing architecture with pipe communication maintained
-- Rich observation structure with screenshots, DOM, accessibility tree
-- Multiple evaluation modes (webarena, miniwob, visualwebarena) - needs Browser-Use implementation
-
-## Paradigm Shift: Browser-Use vs Browser-Gym
-
-### Browser-Gym Approach (Previous)
-- **Accessibility Tree Based**: Rich accessibility tree with semantic element identification
-- **BID System**: Elements identified by unique BIDs (Browser ID) with semantic properties
-- **Tree Updates**: Accessibility tree updates after form interactions to reflect state changes
-- **Semantic Parsing**: Agents parse accessibility tree to understand page structure
-
-### Browser-Use Approach (New)
-- **Index-Based Selection**: Elements identified by numeric indices representing position
-- **Visual + Text Analysis**: Agent uses screenshots and text content to understand pages
-- **No Accessibility Tree**: No complex accessibility tree parsing required
-- **Simpler but Robust**: More reliable element selection through positioning
-
-### Why This Matters
-The test failures we were seeing were because we were trying to force Browser-Use into Browser-Gym's mold. Instead, we need to:
-1. **Accept Browser-Use's different approach** - it's designed to be simpler and more robust
-2. **Update our tests** to work with Browser-Use's observation model
-3. **Use Browser-Use's native capabilities** rather than trying to replicate accessibility trees
-
-### Current Implementation Analysis
-
-**Browser Environment (`browser_use_env.py`):** ✅ COMPLETED
-- Uses multiprocessing with pipe communication between agent and browser processes
-- Supports evaluation modes with different Browser-Use environments
-- Handles screenshots, DOM extraction, accessibility tree, and text content
-- Uses direct Browser-Use interface with step() method
-
-**Action Execution Flow:** ✅ COMPLETED
-1. `ActionExecutor` initializes `BrowserUseEnv` in `_init_browser_async()`
-2. Browser actions are executed via `browse()` utility function
-3. Actions are converted to Browser-Use action models or string actions for compatibility
-4. Browser-Use environment executes actions and returns observations
-5. Observations are converted to `BrowserOutputObservation` format
-
-**Key Observation Fields:** ✅ COMPLETED
-- `url`, `screenshot`, `screenshot_path`, `set_of_marks`
-- `dom_object`, `axtree_object`, `extra_element_properties`
-- `text_content`, `open_pages_urls`, `active_page_index`
-- `last_browser_action`, `last_browser_action_error`, `focused_element_bid`
-
-## Implementation Notes
-
-### Phase 1: Core Browser Environment Replacement ✅ COMPLETED
-
-**Completed Steps:**
-1. ✅ Examine current browser environment implementation
-2. ✅ Research Browser-Use library structure and APIs
-3. ✅ Create new `browser_use_env.py` with equivalent functionality
-4. ✅ Implement observation adapter
-5. ✅ **REVISED**: Remove action mapper - use Browser-Use actions directly
-6. ✅ Test the new implementation
-7. ✅ Update action execution server to use new environment
-
-### Phase 2: Adapt to Browser-Use's Approach 🔄 IN PROGRESS
-
-**Completed Steps:**
-1. ✅ **Remove Form State Tracking**: Removed form state tracking from BrowserUseEnv
-2. ✅ **Simplify Accessibility Tree**: Removed form state dependency from observation adapter
-3. ✅ **Update Tests**: Modified tests to work with Browser-Use's approach instead of expecting accessibility tree updates
-
-**Current Work:**
-- Adapting tests to check actual behavior (form submission, page changes) rather than accessibility tree updates
-- Simplifying element identification to work with Browser-Use's index-based approach
-
-### Browser-Use Library Analysis ✅ COMPLETED
-
-**Key Components Found:**
-- `BrowserSession`: Main browser interface with methods like `navigate()`, `take_screenshot()`, `get_page_info()`, `go_back()`, `go_forward()`
-- `Controller`: Action execution interface with `act()` method
-- Action Models: Structured actions like `GoToUrlAction`, `ClickElementAction`, `InputTextAction`
-
-**Available Actions:**
-- `GoToUrlAction`: `url`, `new_tab` fields
-- `ClickElementAction`: `index` field
-- `InputTextAction`: `index`, `text` fields
-- `ScrollAction`, `SearchGoogleAction`, `UploadFileAction`, etc.
-
-**Key Differences from Previous Browser Environment:**
-- Browser-Use uses structured action models instead of string-based actions
-- Actions can be executed via Controller.act() method OR direct BrowserSession methods
-- BrowserSession provides rich state information via get_* methods
-- No gymnasium dependency - direct Playwright-based control
-- **✅ Direct Navigation Methods**: `go_back()`, `go_forward()`, `navigate()` available directly on BrowserSession
-
-### Gotchas to Watch For
-
-1. **Action Mapping Complexity**: Previous browser environment and Browser-Use have different action models ✅ RESOLVED
-2. **Multiprocessing Architecture**: Need to maintain pipe communication for compatibility ✅ MAINTAINED
-3. **Observation Structure**: Must maintain exact field names for backward compatibility ✅ MAINTAINED
-4. **Evaluation Compatibility**: Critical for maintaining benchmark functionality ✅ RESOLVED
-5. **Browser-Use Installation**: Need to install and understand Browser-Use library first ✅ COMPLETED
-6. **Paradigm Shift**: Adapting from accessibility tree to index-based approach 🔄 MITIGATING
-
-### Important Implementation Details
-
-**Current Action Format:** ✅ COMPLETED
-- Previous browser environment used string-based actions like `goto("url")`, `click("bid")`, `fill("bid", "text")`
-- Actions are executed via `browser.step(action_str)` method
-- Successfully mapped these to Browser-Use's action format
-
-**Current Observation Format:** ✅ COMPLETED
-- Rich observation dict with screenshots, DOM, accessibility tree
-- Base64 encoded images
-- Text content extracted from HTML
-- Error handling and status reporting
-
-**Browser-Use Native Approach:** 🔄 ADAPTING
-- Index-based element selection instead of BID-based
-- Visual and text analysis for page understanding
-- Simplified accessibility tree (basic HTML parsing only)
-- Focus on actual behavior rather than accessibility tree updates
-
-## Progress Tracking
-
-- [x] Phase 1: Core Browser Environment Replacement ✅ COMPLETED
-  - [x] Create observation adapter (`observation_adapter.py`)
-  - [x] Create Browser-Use environment (`browser_use_env.py`)
-  - [x] **REVISED**: Remove action mapper, integrate Browser-Use actions directly
-  - [x] **✅ Test the new implementation** - All navigation tests passing
-  - [x] **✅ Fix async handling** - All async operations properly awaited
-  - [x] **✅ Fix go_back/go_forward** - Using direct BrowserSession methods
-  - [x] **✅ Update action execution server** - Action execution server updated to use new environment
-- [x] Phase 2: Adapt to Browser-Use's Approach 🔄 IN PROGRESS
-  - [x] **✅ Remove form state tracking** - Removed from BrowserUseEnv and observation adapter
-  - [x] **✅ Simplify accessibility tree** - Removed form state dependency
-  - [x] **✅ Update tests** - Modified to work with Browser-Use's approach
-  - [ ] **🔄 Simplify element identification** - Remove BID dependency, use index-based approach
-- [ ] Phase 3: Action and Observation Updates
-- [ ] Phase 4: Agent Updates
-- [x] Phase 5: Configuration and Infrastructure ✅ COMPLETED
-  - [x] **✅ Update configuration** - Sandbox config updated to use browser_use_config
-  - [x] **✅ Update action execution server** - All browser environment integration updated
-  - [x] **✅ Update command generation** - Command generation updated for Browser-Use
-- [x] Phase 6: Evaluation and Testing ✅ COMPLETED
-  - [x] **✅ Remove browsergym dependencies** - All browsergym references removed from codebase
-  - [x] **✅ Update evaluation scripts** - All evaluation scripts updated to work with Browser-Use
-  - [x] **✅ Update documentation** - All documentation updated to reflect Browser-Use
-- [x] Phase 7: Dependencies and Cleanup ✅ COMPLETED
-  - [x] **✅ Remove browsergym dependencies** - All browsergym references removed from codebase
-  - [x] **✅ Update evaluation scripts** - All evaluation scripts updated to work with Browser-Use
-  - [x] **✅ Update documentation** - All documentation updated to reflect Browser-Use
-
-## Implementation Notes
-
-### Created Files
-
-1. **`openhands/runtime/browser/observation_adapter.py`** ✅
-   - Converts Browser-Use observations to OpenHands format
-   - Maintains compatibility with existing BrowserOutputObservation structure
-   - Handles screenshots, HTML content, and page structure
-
-2. **`openhands/runtime/browser/browser_use_env.py`** ✅
-   - Drop-in replacement for previous browser environment
-   - Maintains same interface (step(), check_alive(), close())
-   - Uses multiprocessing architecture for compatibility
-   - Integrates Browser-Use BrowserSession and Controller
-   - **REVISED**: Supports both string actions (backward compatibility) and direct Browser-Use action models
-
-### Key Implementation Decisions
-
-1. **REVISED**: **Hybrid Action Support**: Support both string actions (backward compatibility) and direct Browser-Use action models
-2. **Observation Structure**: Maintained exact field names for backward compatibility
-3. **Multiprocessing**: Kept the same pipe-based communication for compatibility
-4. **Error Handling**: Implemented comprehensive error handling and fallbacks
-5. **Complete Replacement**: Remove previous browser environment entirely, no feature flags or dual support
-6. **✅ Direct Method Usage**: Use BrowserSession methods directly (go_back, go_forward, navigate) instead of controller when possible
-7. **✅ Async-First Design**: All Browser-Use operations properly awaited and handled asynchronously
-8. **🔄 Browser-Use Native**: Adapt to Browser-Use's index-based approach instead of forcing Browser-Gym patterns
-
-### Known Limitations
-
-1. **🔄 Element Identification**: Need to replace BID system with Browser-Use's element indexing
-2. **✅ Accessibility Tree**: Simplified implementation - basic HTML parsing only
-3. **✅ Async Operations**: All async operations properly handled and awaited
-4. **✅ Evaluation Support**: Basic evaluation support implemented - needs testing
-5. **Action Interface**: Need to update all agents to use Browser-Use action models instead of strings
-6. **✅ Navigation Actions**: All navigation actions (goto, go_back, go_forward) working correctly
-
-### Test Results
-
-**✅ Successful Tests:**
-- Browser-Use action model creation and validation
-- Action string parsing for backward compatibility
-- Environment initialization and basic communication
-- Alive check functionality
-- **✅ Navigation actions**: `goto()`, `go_back()`, `go_forward()` all working correctly
-- **✅ No-op actions**: `noop()` with wait times working correctly
-- **✅ Simple browsing**: Basic URL navigation working correctly
-
-**🔧 Fixed Issues:**
-- **✅ Async operations**: Properly awaited all async calls in Browser-Use environment
-- **✅ Navigation actions**: Fixed `go_back()` and `go_forward()` by using direct `BrowserSession` methods instead of controller
-- **✅ Screenshot capture**: Async handling implemented correctly
-- **✅ Page content retrieval**: Working correctly with proper async handling
-- **🔄 Form interaction tests**: Updated to work with Browser-Use's approach instead of expecting accessibility tree updates
-
-**Next Steps:**
-- ✅ **COMPLETED**: Update action execution server to use new environment
-- ✅ **COMPLETED**: Remove all browsergym references from codebase
-- ✅ **COMPLETED**: Remove form state tracking and simplify accessibility tree
-- 🔄 **IN PROGRESS**: Update tests to work with Browser-Use's native capabilities
-- Continue with Phase 3 (action/observation updates)
-- Update agents to use Browser-Use action models
-- Update evaluation scripts and benchmarks

browser_refactor_plan.md

@@ -1,413 +0,0 @@
-# Browser Refactoring Plan: Replacing Previous Browser Environment with Browser-Use
-
-## Overview
-
-This document outlines the plan to refactor OpenHands' browser functionality from the previous browser environment to Browser-Use library. The goal is to replace the current browser environment implementation with Browser-Use's low-level APIs while maintaining all existing functionality.
-
-## Key Architectural Difference: Browser-Use vs Browser-Gym
-
-### Browser-Gym Approach (Previous)
-- **Accessibility Tree Based**: Rich accessibility tree with semantic element identification
-- **BID System**: Elements identified by unique BIDs (Browser ID) with semantic properties
-- **Tree Updates**: Accessibility tree updates after form interactions to reflect state changes
-- **Semantic Parsing**: Agents parse accessibility tree to understand page structure
-
-### Browser-Use Approach (New)
-- **Index-Based Selection**: Elements identified by numeric indices representing position
-- **Visual + Text Analysis**: Agent uses screenshots and text content to understand pages
-- **No Accessibility Tree**: No complex accessibility tree parsing required
-- **Simpler but Robust**: More reliable element selection through positioning
-
-### Why This Matters
-The test failures we're seeing are because we're trying to force Browser-Use into Browser-Gym's mold. Instead, we need to:
-1. **Accept Browser-Use's different approach** - it's designed to be simpler and more robust
-2. **Update our tests** to work with Browser-Use's observation model
-3. **Use Browser-Use's native capabilities** rather than trying to replicate accessibility trees
-
-## Current Architecture Analysis
-
-### Current Browser Integration Points
-
-1. **Core Browser Environment** (`openhands/runtime/browser/browser_use_env.py`) ✅ COMPLETED
-   - Uses Browser-Use's direct browser control interface
-   - Supports evaluation modes (webarena, miniwob, visualwebarena) - needs implementation
-   - Multiprocessing architecture with pipe communication
-   - Handles screenshots, DOM extraction, and accessibility tree
-
-2. **Action Definitions** (`openhands/events/action/browse.py`)
-   - `BrowseURLAction`: Simple URL navigation
-   - `BrowseInteractiveAction`: Full browser action support
-   - Includes `browsergym_send_msg_to_user` field (needs removal)
-
-3. **Observation Definitions** (`openhands/events/observation/browse.py`)
-   - `BrowserOutputObservation`: Rich observation data
-   - Includes screenshots, DOM objects, accessibility tree, etc.
-
-4. **Agent Implementations**
-   - `BrowsingAgent` (`openhands/agenthub/browsing_agent/`)
-   - `VisualBrowsingAgent` (`openhands/agenthub/visualbrowsing_agent/`)
-   - `CodeActAgent` browser tool (`openhands/agenthub/codeact_agent/tools/browser.py`)
-
-5. **Configuration** (`openhands/core/config/sandbox_config.py`) ✅ COMPLETED
-   - `browser_use_config` configuration option
-
-6. **Evaluation Benchmarks** ✅ COMPLETED
-   - WebArena, MiniWoB, VisualWebArena evaluation scripts updated
-   - Success rate calculation scripts updated
-
-## Browser-Use Library Analysis
-
-### Key Components
-
-1. **Controller Service** (`browser_use/controller/service.py`)
-   - Action registry system
-   - Built-in actions: search_google, go_to_url, click_element, input_text, etc.
-   - Extensible action system
-
-2. **Action Models** (`browser_use/controller/views.py`)
-   - Structured action parameters
-   - Type-safe action definitions
-
-3. **Browser Session** (`browser_use/browser/`)
-   - Playwright-based browser control
-   - Tab management
-   - Page navigation and interaction
-
-4. **Types** (`browser_use/browser/types.py`)
-   - Unified Playwright/Patchright types
-   - Page, Browser, ElementHandle abstractions
-
-## Refactoring Strategy
-
-### Phase 1: Core Browser Environment Replacement ✅ COMPLETED
-
-#### 1.1 Create New Browser Environment ✅
-- **File**: `openhands/runtime/browser/browser_use_env.py` ✅
-- **Purpose**: Replace `browser_env.py` with Browser-Use implementation ✅
-- **Key Changes**:
-  - Remove gymnasium dependency ✅
-  - Use Browser-Use's BrowserSession directly ✅
-  - Maintain multiprocessing architecture for compatibility ✅
-  - Implement equivalent observation structure ✅
-
-#### 1.2 Browser-Use Action Integration ✅
-- **Purpose**: Use Browser-Use's native action system directly ✅
-- **Strategy**:
-  - **REVISED**: Support both string actions (backward compatibility) and Browser-Use action models ✅
-  - Use Browser-Use's structured action models directly ✅
-  - **✅ Direct Method Usage**: Use BrowserSession methods directly for navigation (go_back, go_forward, navigate) ✅
-
-#### 1.3 Observation Adapter ✅
-- **File**: `openhands/runtime/browser/observation_adapter.py` ✅
-- **Purpose**: Convert Browser-Use observations to OpenHands format ✅
-- **Key Features**:
-  - Screenshot capture and base64 encoding ✅
-  - DOM extraction and flattening ✅
-  - Accessibility tree generation ✅
-  - Error handling and status reporting ✅
-
-### Phase 2: Adapt to Browser-Use's Approach 🔄 IN PROGRESS
-
-#### 2.1 Remove Accessibility Tree Dependency
-- **Purpose**: Stop trying to replicate Browser-Gym's accessibility tree functionality
-- **Strategy**:
-  - Remove form state tracking (it's a workaround for Browser-Gym's approach)
-  - Simplify accessibility tree generation to basic HTML parsing
-  - Focus on Browser-Use's native capabilities (screenshots, text content, element indices)
-
-#### 2.2 Update Tests for Browser-Use's Model
-- **Purpose**: Make tests work with Browser-Use's observation model
-- **Strategy**:
-  - Update form interaction tests to check actual behavior (form submission, page changes)
-  - Remove expectations about accessibility tree updates after form interactions
-  - Test Browser-Use's native capabilities instead of Browser-Gym's features
-
-#### 2.3 Simplify Element Identification
-- **Purpose**: Use Browser-Use's index-based approach
-- **Strategy**:
-  - Remove BID-based element identification
-  - Use element indices for interaction
-  - Update agents to work with index-based selection
-
-### Phase 3: Action and Observation Updates
-
-#### 3.1 Update Action Definitions
-- **File**: `openhands/events/action/browse.py`
-- **Changes**:
-  - Remove `browsergym_send_msg_to_user` field
-  - Update to use Browser-Use action models directly
-  - Replace string-based actions with structured Browser-Use actions
-
-#### 3.2 Update Observation Definitions
-- **File**: `openhands/events/observation/browse.py`
-- **Changes**:
-  - Ensure compatibility with new observation structure
-  - Add any Browser-Use specific fields
-  - Maintain existing field names for compatibility
-
-### Phase 4: Agent Updates
-
-#### 4.1 Update BrowsingAgent
-- **File**: `openhands/agenthub/browsing_agent/browsing_agent.py`
-- **Changes**:
-  - Remove BrowserGym HighLevelActionSet dependency
-  - Implement Browser-Use action generation using structured action models
-  - Update response parsing for Browser-Use action format
-
-#### 4.2 Update VisualBrowsingAgent
-- **File**: `openhands/agenthub/visualbrowsing_agent/visualbrowsing_agent.py`
-- **Changes**:
-  - Similar updates to BrowsingAgent
-  - Ensure visual capabilities are maintained
-
-#### 4.3 Update CodeActAgent Browser Tool
-- **File**: `openhands/agenthub/codeact_agent/tools/browser.py`
-- **Changes**:
-  - Replace BrowserGym action descriptions with Browser-Use action models
-  - Update tool parameter descriptions to match Browser-Use action fields
-  - Maintain existing API for tool calls
-
-### Phase 5: Configuration and Infrastructure ✅ COMPLETED
-
-#### 5.1 Update Configuration ✅ COMPLETED
-- **File**: `openhands/core/config/sandbox_config.py`
-- **Changes**:
-  - Replace `browsergym_eval_env` with `browser_use_config` ✅
-  - Add Browser-Use specific configuration options ✅
-  - Remove BrowserGym configuration entirely ✅
-- **Status**: ✅ COMPLETED - Configuration updated
-
-#### 5.2 Update Action Execution Server ✅ COMPLETED
-- **File**: `openhands/runtime/action_execution_server.py`
-- **Changes**:
-  - Replace BrowserEnv with BrowserUseEnv ✅
-  - Update initialization parameters ✅
-  - Maintain existing API ✅
-- **Status**: ✅ COMPLETED - All browser environment integration updated
-
-#### 5.3 Update Command Generation ✅ COMPLETED
-- **File**: `openhands/runtime/utils/command.py`
-- **Changes**:
-  - Replace browsergym arguments with browser-use arguments ✅
-  - Update startup command generation ✅
-- **Status**: ✅ COMPLETED - Command generation updated
-
-### Phase 6: Evaluation and Testing ✅ COMPLETED
-
-#### 6.1 Update Evaluation Scripts ✅ COMPLETED
-- **Files**:
-  - `evaluation/benchmarks/webarena/run_infer.py`
-  - `evaluation/benchmarks/miniwob/run_infer.py`
-  - `evaluation/benchmarks/visualwebarena/run_infer.py`
-- **Changes**:
-  - Remove BrowserGym imports ✅
-  - Update evaluation environment setup ✅
-  - Maintain evaluation metrics and success rate calculations ✅
-
-#### 6.2 Update Success Rate Scripts ✅ COMPLETED
-- **Files**:
-  - `evaluation/benchmarks/webarena/get_success_rate.py`
-  - `evaluation/benchmarks/miniwob/get_avg_reward.py`
-  - `evaluation/benchmarks/visualwebarena/get_success_rate.py`
-- **Changes**:
-  - Remove BrowserGym environment registration ✅
-  - Update metric calculation logic ✅
-
-### Phase 7: Dependencies and Cleanup ✅ COMPLETED
-
-#### 7.1 Update Dependencies ✅ COMPLETED
-- **File**: `pyproject.toml`
-- **Changes**:
-  - Remove BrowserGym dependencies ✅
-  - Add Browser-Use dependency ✅
-- **Status**: ✅ COMPLETED
-
-#### 7.2 Cleanup Imports ✅ COMPLETED
-- **Files**: All files with BrowserGym imports
-- **Changes**:
-  - Remove all `browsergym` imports ✅
-  - Update import statements to use Browser-Use ✅
-  - Remove unused imports ✅
-
-## Implementation Details
-
-### Browser-Use Integration Architecture ✅ IMPLEMENTED
-
-```python
-# New Browser Environment Structure ✅ IMPLEMENTED
-class BrowserUseEnv:
-    def __init__(self, browser_use_config: Optional[str] = None):
-        self.browser_session: BrowserSession
-        self.observation_adapter: ObservationAdapter
-
-    async def execute_action_async(self, browser_session: BrowserSession, controller: Controller, action: Union[str, Any]) -> Dict[str, Any]:
-        # 1. Execute Browser-Use action directly ✅
-        # 2. Get observation from BrowserSession ✅
-        # 3. Convert observation to OpenHands format ✅
-        # 4. Return observation dict ✅
-
-        # Key improvements:
-        # - Direct BrowserSession method usage for navigation (go_back, go_forward, navigate)
-        # - Proper async handling for all operations
-        # - Backward compatibility with string actions
-```
-
-### Browser-Use Action Integration ✅ IMPLEMENTED
-
-```python
-# Direct Browser-Use Action Usage ✅ IMPLEMENTED
-from browser_use.controller.service import GoToUrlAction, ClickElementAction, InputTextAction
-
-# Instead of string parsing, use structured actions directly ✅
-goto_action = GoToUrlAction(url="https://example.com", new_tab=False)
-click_action = ClickElementAction(index=123)
-input_action = InputTextAction(index=456, text="Hello World")
-
-# ✅ HYBRID APPROACH: Support both structured actions and string actions
-# String actions for backward compatibility:
-# goto("https://example.com") -> GoToUrlAction(url="https://example.com", new_tab=False)
-# go_back() -> await browser_session.go_back()
-# go_forward() -> await browser_session.go_forward()
-
-# ✅ Direct BrowserSession method usage for navigation:
-await browser_session.go_back()      # Direct method call
-await browser_session.go_forward()   # Direct method call
-await browser_session.navigate(url)  # Direct method call
-```
-
-### Observation Structure Compatibility
-
-```python
-# Maintain existing observation structure
-{
-    'url': str,
-    'screenshot': str,  # base64 encoded
-    'screenshot_path': str | None,
-    'dom_object': dict,
-    'axtree_object': dict,  # Simplified - basic HTML parsing only
-    'text_content': str,
-    'open_pages_urls': list[str],
-    'active_page_index': int,
-    'last_browser_action': str,
-    'last_browser_action_error': str,
-    'focused_element_bid': str,
-    # ... other existing fields
-}
-```
-
-## Migration Strategy
-
-### Direct Replacement
-1. **Complete Removal**: Remove BrowserGym entirely and replace with Browser-Use
-2. **No Feature Flags**: No dual support period - direct replacement
-3. **Structured Actions**: Use Browser-Use's native action models throughout
-4. **Adapt to Browser-Use's Approach**: Accept that Browser-Use works differently than Browser-Gym
-
-### Testing Strategy
-1. **Unit Tests**: Test each component individually
-2. **Integration Tests**: Test browser environment end-to-end
-3. **Evaluation Tests**: Ensure evaluation benchmarks still work
-4. **Performance Tests**: Compare performance between implementations
-5. **Browser-Use Native Tests**: Test Browser-Use's actual capabilities, not Browser-Gym's features
-
-### Rollback Plan
-1. **Git Revert**: Use git revert to rollback to previous BrowserGym implementation
-2. **Version Tagging**: Tag releases before and after migration
-3. **Documentation**: Clear migration instructions
-
-## Timeline
-
-### Week 1-2: Core Environment ✅ COMPLETED
-- ✅ Implement BrowserUseEnv
-- ✅ Create action mapper and observation adapter
-- ✅ Basic functionality testing
-- ✅ Fix async handling and navigation actions
-
-### Week 3-4: Adapt to Browser-Use's Approach 🔄 IN PROGRESS
-- Remove accessibility tree dependency
-- Update tests for Browser-Use's model
-- Simplify element identification
-
-### Week 5-6: Agent Updates
-- Update BrowsingAgent and VisualBrowsingAgent
-- Update CodeActAgent browser tool
-- Agent functionality testing
-
-### Week 7-8: Infrastructure ✅ COMPLETED
-- ✅ Update configuration and command generation
-- ✅ Update action execution server
-- ✅ Integration testing
-
-### Week 9-10: Evaluation ✅ COMPLETED
-- ✅ Update evaluation scripts
-- ✅ Update success rate calculations
-- ✅ Remove all browsergym dependencies
-- ✅ Update documentation
-
-### Week 11-12: Cleanup and Polish ✅ COMPLETED
-- ✅ Remove remaining browsergym references
-- ✅ Clean up imports and unused code
-- ✅ Final testing and documentation
-
-## Risk Assessment
-
-### High Risk
-1. **Action Mapping Complexity**: BrowserGym and Browser-Use have different action models ✅ RESOLVED
-2. **Evaluation Compatibility**: Ensuring evaluation benchmarks work correctly ✅ RESOLVED
-3. **Performance Impact**: Browser-Use might have different performance characteristics
-4. **Paradigm Shift**: Adapting from accessibility tree to index-based approach 🔄 MITIGATING
-
-### Medium Risk
-1. **API Changes**: Browser-Use API might change during development
-2. **Dependency Conflicts**: Potential conflicts with existing dependencies
-3. **Testing Coverage**: Ensuring all edge cases are covered
-
-### Low Risk
-1. **Documentation Updates**: Updating documentation and examples
-2. **Configuration Changes**: Updating configuration files
-
-### ✅ Mitigated Risks
-1. **✅ Async Operations**: All async operations properly handled and tested
-2. **✅ Navigation Actions**: go_back, go_forward, goto all working correctly
-3. **✅ Backward Compatibility**: String actions still supported for smooth transition
-4. **✅ Core Functionality**: Basic browsing and navigation fully functional
-
-## Success Criteria
-
-1. **Functional Parity**: All existing browser functionality works with Browser-Use
-2. **Performance**: Browser-Use implementation performs at least as well as BrowserGym
-3. **Evaluation**: All evaluation benchmarks pass with similar or better results
-4. **Stability**: No regressions in browser functionality
-5. **Maintainability**: Cleaner, more maintainable codebase
-6. **Browser-Use Native**: Fully leverage Browser-Use's capabilities instead of forcing Browser-Gym patterns
-
-### ✅ Achieved Milestones
-1. **✅ Core Navigation**: goto, go_back, go_forward actions working correctly
-2. **✅ Basic Browsing**: Simple URL navigation and page content retrieval working
-3. **✅ Async Operations**: All async operations properly handled
-4. **✅ Backward Compatibility**: String-based actions still supported
-5. **✅ Error Handling**: Robust error handling and fallbacks implemented
-
-## Conclusion
-
-This refactoring plan provides a comprehensive approach to replacing BrowserGym with Browser-Use while maintaining all existing functionality. The phased approach ensures minimal disruption and allows for thorough testing at each stage. The focus on backward compatibility and gradual migration reduces risk and ensures a smooth transition.
-
-**Key Insight**: Browser-Use uses a fundamentally different approach than Browser-Gym. Instead of trying to replicate Browser-Gym's accessibility tree functionality, we should embrace Browser-Use's simpler but more robust index-based approach.
-
-### ✅ Phase 1, Phase 5, Phase 6, and Phase 7 Successfully Completed
-
-Phase 1, Phase 5, Phase 6, and Phase 7 of the refactoring have been successfully completed with all core browser environment functionality, infrastructure updates, and browsergym removal working correctly:
-
-- **✅ BrowserUseEnv Implementation**: Fully functional drop-in replacement for previous browser environment
-- **✅ Navigation Actions**: goto, go_back, go_forward all working correctly
-- **✅ Async Operations**: All async operations properly handled and tested
-- **✅ Backward Compatibility**: String-based actions still supported
-- **✅ Error Handling**: Robust error handling and fallbacks implemented
-- **✅ Action Execution Server**: Updated to use BrowserUseEnv with proper parameter naming
-- **✅ Configuration**: Updated sandbox config to use browser_use_config
-- **✅ Command Generation**: Updated to use Browser-Use arguments
-- **✅ Browsergym Removal**: All browsergym dependencies and references completely removed from codebase
-- **✅ Evaluation Scripts**: All evaluation scripts updated to work with Browser-Use
-- **✅ Documentation**: All documentation updated to reflect Browser-Use
-
-**🔄 Current Priority**: Phase 2 - Adapt to Browser-Use's approach by removing accessibility tree dependency and updating tests to work with Browser-Use's native capabilities.

build_vscode.py

@@ -93,8 +93,7 @@ def build_vscode_extension():
 
 
 def build(setup_kwargs):
-    """
-    This function is called by Poetry during the build process.
+    """This function is called by Poetry during the build process.
     `setup_kwargs` is a dictionary that will be passed to `setuptools.setup()`.
     """
     print('--- Running custom Poetry build script (build_vscode.py) ---')

config.template.toml

@@ -308,7 +308,8 @@ classpath = "my_package.my_module.MyCustomAgent"
 # Environment variables to set at the launch of the runtime
 #runtime_startup_env_vars = {}
 
-# browser_use_config = ""
+# BrowserGym environment to use for evaluation
+#browsergym_eval_env = ""
 
 # Platform to use for building the runtime image (e.g., "linux/amd64")
 #platform = ""

containers/app/Dockerfile

@@ -21,7 +21,7 @@ ENV POETRY_NO_INTERACTION=1 \
     POETRY_CACHE_DIR=/tmp/poetry_cache
 
 RUN apt-get update -y \
-    && apt-get install -y curl make git build-essential \
+    && apt-get install -y curl make git build-essential jq gettext \
     && python3 -m pip install poetry --break-system-packages
 
 COPY pyproject.toml poetry.lock ./
@@ -58,8 +58,8 @@ RUN sed -i 's/^UID_MIN.*/UID_MIN 499/' /etc/login.defs
 # Default is 60000, but we've seen up to 200000
 RUN sed -i 's/^UID_MAX.*/UID_MAX 1000000/' /etc/login.defs
 
-RUN groupadd app
-RUN useradd -l -m -u $OPENHANDS_USER_ID -s /bin/bash openhands && \
+RUN groupadd --gid $OPENHANDS_USER_ID app
+RUN useradd -l -m -u $OPENHANDS_USER_ID --gid $OPENHANDS_USER_ID -s /bin/bash openhands && \
     usermod -aG app openhands && \
     usermod -aG sudo openhands && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

containers/app/entrypoint.sh

@@ -23,6 +23,18 @@ if [ -z "$WORKSPACE_MOUNT_PATH" ]; then
   unset WORKSPACE_BASE
 fi
 
+if [[ "$INSTALL_THIRD_PARTY_RUNTIMES" == "true" ]]; then
+  echo "Downloading and installing third_party_runtimes..."
+  echo "Warning: Third-party runtimes are provided as-is, not actively supported and may be removed in future releases."
+
+  if pip install 'openhands-ai[third_party_runtimes]' -qqq 2> >(tee /dev/stderr); then
+    echo "third_party_runtimes installed successfully."
+  else
+    echo "Failed to install third_party_runtimes." >&2
+    exit 1
+  fi
+fi
+
 if [[ "$SANDBOX_USER_ID" -eq 0 ]]; then
   echo "Running OpenHands as root"
   export RUN_AS_OPENHANDS=false

containers/dev/compose.yml

@@ -12,7 +12,7 @@ services:
       - SANDBOX_API_HOSTNAME=host.docker.internal
       - DOCKER_HOST_ADDR=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.49-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.54-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

dev_config/python/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies:
-          [types-requests, types-setuptools, types-pyyaml, types-toml, types-docker, pydantic, lxml]
+          [types-requests, types-setuptools, types-pyyaml, types-toml, types-docker, types-Markdown, pydantic, lxml]
         # To see gaps add `--html-report mypy-report/`
         entry: mypy --config-file dev_config/python/mypy.ini openhands/
         always_run: true

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik}
       #- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of ~/.openhands for this user
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/docs.json

@@ -37,7 +37,16 @@
                   "usage/cloud/bitbucket-installation",
                   "usage/cloud/github-installation",
                   "usage/cloud/gitlab-installation",
-                  "usage/cloud/slack-installation"
+                  "usage/cloud/slack-installation",
+                  {
+                    "group": "Project Management Tools",
+                    "pages": [
+                      "usage/cloud/project-management/overview",
+                      "usage/cloud/project-management/jira-integration",
+                      "usage/cloud/project-management/jira-dc-integration",
+                      "usage/cloud/project-management/linear-integration"
+                    ]
+                  }
                 ]
               },
               "usage/cloud/cloud-ui",
@@ -62,6 +71,7 @@
                     {
                       "group": "Providers",
                       "pages": [
+                        "usage/llms/openhands-llms",
                         "usage/llms/azure-llms",
                         "usage/llms/google-llms",
                         "usage/llms/groq",
@@ -69,7 +79,6 @@
                         "usage/llms/litellm-proxy",
                         "usage/llms/moonshot",
                         "usage/llms/openai-llms",
-                        "usage/llms/openhands-llms",
                         "usage/llms/openrouter"
                       ]
                     }

docs/usage/architecture/backend.mdx

@@ -2,55 +2,102 @@
 title: Backend Architecture
 ---
 
-<div style={{ textAlign: 'center' }}>
-  <img src="https://github.com/All-Hands-AI/OpenHands/assets/16201837/97d747e3-29d8-4ccb-8d34-6ad1adb17f38" alt="OpenHands System Architecture Diagram Jul 4 2024" />
-  <p><em>OpenHands System Architecture Diagram (July 4, 2024)</em></p>
-</div>
 
 This is a high-level overview of the system architecture. The system is divided into two main components: the frontend and the backend. The frontend is responsible for handling user interactions and displaying the results. The backend is responsible for handling the business logic and executing the agents.
 
-# Frontend architecture
+# System overview
 
-![system_architecture.svg](/static/img/system_architecture.svg)
+```mermaid
+flowchart LR
+  U["User"] --> FE["Frontend (SPA)"]
+  FE -- "HTTP/WS" --> BE["OpenHands Backend"]
+  BE --> ES["EventStream"]
+  BE --> ST["Storage"]
+  BE --> RT["Runtime Interface"]
+  BE --> LLM["LLM Providers"]
+
+  subgraph Runtime
+    direction TB
+    RT --> DRT["Docker Runtime"]
+    RT --> LRT["Local Runtime"]
+    RT --> RRT["Remote Runtime"]
+    DRT --> AES["Action Execution Server"]
+    LRT --> AES
+    RRT --> AES
+    AES --> Bash["Bash Session"]
+    AES --> Jupyter["Jupyter Plugin"]
+    AES --> Browser["BrowserEnv"]
+  end
+```
 
 This Overview is simplified to show the main components and their interactions. For a more detailed view of the backend architecture, see the Backend Architecture section below.
 
 # Backend Architecture
 
-_**Disclaimer**: The backend architecture is a work in progress and is subject to change. The following diagram shows the current architecture of the backend based on the commit that is shown in the footer of the diagram._
 
-![backend_architecture.svg](/static/img/backend_architecture.svg)
+```mermaid
+classDiagram
+  class Agent {
+    <<abstract>>
+    +sandbox_plugins: list[PluginRequirement]
+  }
+  class CodeActAgent {
+    +tools
+  }
+  Agent <|-- CodeActAgent
+
+  class EventStream
+  class Observation
+  class Action
+  Action --> Observation
+  Agent --> EventStream
+
+  class Runtime {
+    +connect()
+    +send_action_for_execution()
+  }
+  class ActionExecutionClient {
+    +_send_action_server_request()
+  }
+  class DockerRuntime
+  class LocalRuntime
+  class RemoteRuntime
+  Runtime <|-- ActionExecutionClient
+  ActionExecutionClient <|-- DockerRuntime
+  ActionExecutionClient <|-- LocalRuntime
+  ActionExecutionClient <|-- RemoteRuntime
+
+  class ActionExecutionServer {
+    +/execute_action
+    +/alive
+  }
+  class BashSession
+  class JupyterPlugin
+  class BrowserEnv
+  ActionExecutionServer --> BashSession
+  ActionExecutionServer --> JupyterPlugin
+  ActionExecutionServer --> BrowserEnv
+
+  Agent --> Runtime
+  Runtime ..> ActionExecutionServer : REST
+```
 
 <details>
   <summary>Updating this Diagram</summary>
   <div>
-    The generation of the backend architecture diagram is partially automated.
-    The diagram is generated from the type hints in the code using the py2puml
-    tool. The diagram is then manually reviewed, adjusted and exported to PNG
-    and SVG.
+    We maintain architecture diagrams inline with Mermaid in this MDX.
 
-    ## Prerequisites
-
-    - Running python environment in which openhands is executable
-    (according to the instructions in the README.md file in the root of the repository)
-    - [py2puml](https://github.com/lucsorel/py2puml) installed
-
-## Steps
-
-1.  Autogenerate the diagram by running the following command from the root of the repository:
-    `py2puml openhands openhands > docs/architecture/backend_architecture.puml`
-
-2.  Open the generated file in a PlantUML editor, e.g. Visual Studio Code with the PlantUML extension or [PlantText](https://www.planttext.com/)
-
-3.  Review the generated PUML and make all necessary adjustments to the diagram (add missing parts, fix mistakes, improve positioning).
-    _py2puml creates the diagram based on the type hints in the code, so missing or incorrect type hints may result in an incomplete or incorrect diagram._
-
-4.  Review the diff between the new and the previous diagram and manually check if the changes are correct.
-    _Make sure not to remove parts that were manually added to the diagram in the past and are still relevant._
-
-5.  Add the commit hash of the commit that was used to generate the diagram to the diagram footer.
-
-6.  Export the diagram as PNG and SVG files and replace the existing diagrams in the `docs/architecture` directory. This can be done with (e.g. [PlantText](https://www.planttext.com/))
+    Guidance:
+    - Edit the Mermaid blocks directly (flowchart/classDiagram).
+    - Quote labels and edge text for GitHub preview compatibility.
+    - Keep relationships concise and reflect stable abstractions (agents, runtime client/server, plugins).
+    - Verify accuracy against code:
+      - openhands/runtime/impl/action_execution/action_execution_client.py
+      - openhands/runtime/impl/docker/docker_runtime.py
+      - openhands/runtime/impl/local/local_runtime.py
+      - openhands/runtime/action_execution_server.py
+      - openhands/runtime/plugins/*
+    - Build docs locally or view on GitHub to confirm diagrams render.
 
   </div>
 </details>

docs/usage/architecture/runtime.mdx

@@ -52,7 +52,7 @@ graph TD
 2. Image Building: OpenHands builds a new Docker image (the "OH runtime image") based on the user-provided image. This new image includes OpenHands-specific code, primarily the "runtime client"
 3. Container Launch: When OpenHands starts, it launches a Docker container using the OH runtime image
 4. Action Execution Server Initialization: The action execution server initializes an `ActionExecutor` inside the container, setting up necessary components like a bash shell and loading any specified plugins
-5. Communication: The OpenHands backend (`openhands/runtime/impl/eventstream/eventstream_runtime.py`) communicates with the action execution server over RESTful API, sending actions and receiving observations
+5. Communication: The OpenHands backend (client: `openhands/runtime/impl/action_execution/action_execution_client.py`; runtimes: `openhands/runtime/impl/docker/docker_runtime.py`, `openhands/runtime/impl/local/local_runtime.py`) communicates with the action execution server over RESTful API, sending actions and receiving observations
 6. Action Execution: The runtime client receives actions from the backend, executes them in the sandboxed environment, and sends back observations
 7. Observation Return: The action execution server sends execution results back to the OpenHands backend as observations
 
@@ -72,7 +72,7 @@ Check out the [relevant code](https://github.com/All-Hands-AI/OpenHands/blob/mai
 ### Image Tagging System
 
 OpenHands uses a three-tag system for its runtime images to balance reproducibility with flexibility.
-Tags may be in one of 2 formats:
+The tags are:
 
 - **Versioned Tag**: `oh_v{openhands_version}_{base_image}` (e.g.: `oh_v0.9.9_nikolaik_s_python-nodejs_t_python3.12-nodejs22`)
 - **Lock Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`)
@@ -119,18 +119,52 @@ This tagging approach allows OpenHands to efficiently manage both development an
 2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images)
 3. The **lock** tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image, dependency, and OpenHands version combination
 
+## Volume mounts: named volumes and overlay
+
+OpenHands supports both bind mounts and Docker named volumes in SandboxConfig.volumes:
+
+- Bind mount: "/abs/host/path:/container/path[:mode]"
+- Named volume: "volume:<name>:/container/path[:mode]" or any non-absolute host spec treated as a named volume
+
+Overlay mode (copy-on-write layer) is supported for bind mounts by appending ":overlay" to the mode (e.g., ":ro,overlay").
+To enable overlay COW, set SANDBOX_VOLUME_OVERLAYS to a writable host directory; per-container upper/work dirs are created under it. If SANDBOX_VOLUME_OVERLAYS is unset, overlay mounts are skipped.
+
+Implementation references:
+- openhands/runtime/impl/docker/docker_runtime.py (named volumes in _build_docker_run_args; overlay mounts in _process_overlay_mounts)
+- openhands/core/config/sandbox_config.py (volumes field)
+
+
 ## Runtime Plugin System
 
-The OpenHands Runtime supports a plugin system that allows for extending functionality and customizing the runtime environment. Plugins are initialized when the runtime client starts up.
+The OpenHands Runtime supports a plugin system that allows for extending functionality and customizing the runtime environment. Plugins are initialized when the action execution server starts up inside the runtime.
 
-Check [an example of Jupyter plugin here](https://github.com/All-Hands-AI/OpenHands/blob/ecf4aed28b0cf7c18d4d8ff554883ba182fc6bdd/openhands/runtime/plugins/jupyter/__init__.py#L21-L55) if you want to implement your own plugin.
+## Ports and URLs
 
-*More details about the Plugin system are still under construction - contributions are welcomed!*
+- Host port allocation uses file-locked ranges for stability and concurrency:
+  - Main runtime port: find_available_port_with_lock on configured range
+  - VSCode port: SandboxConfig.sandbox.vscode_port if provided, else find_available_port_with_lock in VSCODE_PORT_RANGE
+  - App ports: two additional ranges for plugin/web apps
+- DOCKER_HOST_ADDR (if set) adjusts how URLs are formed for LocalRuntime/Docker environments.
+- VSCode URL is exposed with a connection token from the action execution server endpoint /vscode/connection_token and rendered as:
+  - Docker/Local: http://localhost:{port}/?tkn={token}&folder={workspace_mount_path_in_sandbox}
+  - RemoteRuntime: scheme://vscode-{host}/?tkn={token}&folder={workspace_mount_path_in_sandbox}
+
+References:
+- openhands/runtime/impl/docker/docker_runtime.py (port ranges, locking, DOCKER_HOST_ADDR, vscode_url)
+- openhands/runtime/impl/local/local_runtime.py (vscode_url factory)
+- openhands/runtime/impl/remote/remote_runtime.py (vscode_url mapping)
+- openhands/runtime/action_execution_server.py (/vscode/connection_token)
+
+
+Examples:
+- Jupyter: openhands/runtime/plugins/jupyter/__init__.py (JupyterPlugin, Kernel Gateway)
+- VS Code: openhands/runtime/plugins/vscode/* (VSCodePlugin, exposes tokenized URL)
+- Agent Skills: openhands/runtime/plugins/agent_skills/*
 
 Key aspects of the plugin system:
 
 1. Plugin Definition: Plugins are defined as Python classes that inherit from a base `Plugin` class
-2. Plugin Registration: Available plugins are registered in an `ALL_PLUGINS` dictionary
+2. Plugin Registration: Available plugins are registered in `openhands/runtime/plugins/__init__.py` via `ALL_PLUGINS`
 3. Plugin Specification: Plugins are associated with `Agent.sandbox_plugins: list[PluginRequirement]`. Users can specify which plugins to load when initializing the runtime
-4. Initialization: Plugins are initialized asynchronously when the runtime client starts
-5. Usage: The runtime client can use initialized plugins to extend its capabilities (e.g., the JupyterPlugin for running IPython cells)
+4. Initialization: Plugins are initialized asynchronously when the runtime starts and are accessible to actions
+5. Usage: Plugins extend capabilities (e.g., Jupyter for IPython cells); the server exposes any web endpoints (ports) via host port mapping

docs/usage/cloud/bitbucket-installation.mdx

@@ -8,6 +8,29 @@ description: This guide walks you through the process of installing OpenHands Cl
 
 - Signed in to [OpenHands Cloud](https://app.all-hands.dev) with [a Bitbucket account](/usage/cloud/openhands-cloud).
 
+## IP Whitelisting
+
+If your Bitbucket Cloud instance has IP restrictions, you'll need to whitelist the following IP addresses to allow OpenHands to access your repositories:
+
+### Core App IP
+```
+34.68.58.200
+```
+
+### Runtime IPs
+```
+34.10.175.217
+34.136.162.246
+34.45.0.142
+34.28.69.126
+35.224.240.213
+34.70.174.52
+34.42.4.87
+35.222.133.153
+34.29.175.97
+34.60.55.59
+```
+
 ## Adding Bitbucket Repository Access
 
 Upon signing into OpenHands Cloud with a Bitbucket account, OpenHands will have access to your repositories.

docs/usage/cloud/project-management/jira-dc-integration.mdx

@@ -0,0 +1,126 @@
+---
+title: Jira Data Center Integration (Beta)
+description: Complete guide for setting up Jira Data Center integration with OpenHands Cloud, including service account creation, personal access token generation, webhook configuration, and workspace integration setup.
+---
+
+# Jira Data Center Integration
+
+## Platform Configuration
+
+### Step 1: Create Service Account
+
+1. **Access User Management**
+   - Log in to Jira Data Center as administrator
+   - Go to **Administration** > **User Management**
+
+2. **Create User**
+   - Click **Create User**
+   - Username: `openhands-agent`
+   - Full Name: `OpenHands Agent`
+   - Email: `openhands@yourcompany.com` (replace with your preferred service account email)
+   - Password: Set a secure password
+   - Click **Create**
+
+3. **Assign Permissions**
+   - Add user to appropriate groups
+   - Ensure access to relevant projects
+   - Grant necessary project permissions
+
+### Step 2: Generate API Token
+
+1. **Personal Access Tokens**
+   - Log in as the service account
+   - Go to **Profile** > **Personal Access Tokens**
+   - Click **Create token**
+   - Name: `OpenHands Cloud Integration`
+   - Expiry: Set appropriate expiration (recommend 1 year)
+   - Click **Create**
+   - **Important**: Copy and store the token securely
+
+### Step 3: Configure Webhook
+
+1. **Create Webhook**
+   - Go to **Administration** > **System** > **WebHooks**
+   - Click **Create a WebHook**
+   - **Name**: `OpenHands Cloud Integration`
+   - **URL**: `https://app.all-hands.dev/integration/jira-dc/events`
+   - Set a suitable webhook secret
+   - **Issue related events**: Select the following:
+     - Issue updated
+     - Comment created
+   - **JQL Filter**: Leave empty (or customize as needed)
+   - Click **Create**
+   - **Important**: Copy and store the webhook secret securely (you'll need this for workspace integration)
+
+---
+
+## Workspace Integration
+
+### Step 1: Log in to OpenHands Cloud
+
+1. **Navigate and Authenticate**
+   - Go to [OpenHands Cloud](https://app.all-hands.dev/)
+   - Sign in with your Git provider (GitHub, GitLab, or BitBucket)
+   - **Important:** Make sure you're signing in with the same Git provider account that contains the repositories you want the OpenHands agent to work on.
+
+### Step 2: Configure Jira Data Center Integration
+
+1. **Access Integration Settings**
+   - Navigate to **Settings** > **Integrations**
+   - Locate **Jira Data Center** section
+
+2. **Configure Workspace**
+   - Click **Configure** button
+   - Enter your workspace name and click **Connect**
+      - If no integration exists, you'll be prompted to enter additional credentials required for the workspace integration:
+         - **Webhook Secret**: The webhook secret from Step 3 above
+         - **Service Account Email**: The service account email from Step 1 above
+         - **Service Account API Key**: The personal access token from Step 2 above
+         - Ensure **Active** toggle is enabled
+
+<Note>
+Workspace name is the host name of your Jira Data Center instance.
+
+Eg: http://jira.all-hands.dev/projects/OH/issues/OH-77
+
+Here the workspace name is **jira.all-hands.dev**.
+</Note>
+
+3. **Complete OAuth Flow**
+   - You'll be redirected to Jira Data Center to complete OAuth verification
+   - Grant the necessary permissions to verify your workspace access. If you have access to multiple workspaces, select the correct one that you initially provided
+   - If successful, you will be redirected back to the **Integrations** settings in the OpenHands Cloud UI
+
+### Managing Your Integration
+
+**Edit Configuration:**
+- Click the **Edit** button next to your configured platform
+- Update any necessary credentials or settings
+- Click **Update** to apply changes
+- You will need to repeat the OAuth flow as before
+- **Important:** Only the original user who created the integration can see the edit view
+
+**Unlink Workspace:**
+- In the edit view, click **Unlink** next to the workspace name
+- This will deactivate your workspace link
+- **Important:** If the original user who configured the integration chooses to unlink their integration, any users currently linked to that integration will also be unlinked, and the workspace integration will be deactivated. The integration can only be reactivated by the original user.
+
+### Screenshots
+
+<AccordionGroup>
+<Accordion title="Workspace link flow">
+![workspace-link.png](/static/img/jira-dc-user-link.png)
+</Accordion>
+
+<Accordion title="Workspace Configure flow">
+![workspace-link.png](/static/img/jira-dc-admin-configure.png)
+</Accordion>
+
+<Accordion title="Edit view as a user">
+![workspace-link.png](/static/img/jira-dc-user-unlink.png)
+</Accordion>
+
+<Accordion title="Edit view as the workspace creator">
+![workspace-link.png](/static/img/jira-dc-admin-edit.png)
+</Accordion>
+</AccordionGroup>

docs/usage/cloud/project-management/jira-integration.mdx

@@ -0,0 +1,130 @@
+---
+title: Jira Cloud Integration
+description: Complete guide for setting up Jira Cloud integration with OpenHands Cloud, including service account creation, API token generation, webhook configuration, and workspace integration setup.
+---
+
+# Jira Cloud Integration
+
+## Platform Configuration
+
+### Step 1: Create Service Account
+
+1. **Navigate to User Management**
+   - Go to [Atlassian Admin](https://admin.atlassian.com/)
+   - Select your organization
+   - Go to **Directory** > **Users**
+
+2. **Create OpenHands Service Account**
+   - Click **Service accounts**
+   - Click **Create a service account**
+   - Name: `OpenHands Agent`
+   - Click **Next**
+   - Select **User** role for Jira app
+   - Click **Create**
+
+### Step 2: Generate API Token
+
+1. **Access Service Account Configuration**
+   - Locate the created service account from above step and click on it
+   - Click **Create API token**
+   - Set the expiry to 365 days (maximum allowed value)
+   - Click **Next**
+   - In **Select token scopes** screen, filter by following values
+      - App: Jira
+      - Scope type: Classic
+      - Scope actions: Write, Read
+   - Select `read:jira-work` and `write:jira-work` scopes
+   - Click **Next**
+   - Review and create API token
+   - **Important**: Copy and securely store the token immediately
+
+### Step 3: Configure Webhook
+
+1. **Navigate to Webhook Settings**
+   - Go to **Jira Settings** > **System** > **WebHooks**
+   - Click **Create a WebHook**
+
+2. **Configure Webhook**
+   - **Name**: `OpenHands Cloud Integration`
+   - **Status**: Enabled
+   - **URL**: `https://app.all-hands.dev/integration/jira/events`
+   - **Issue related events**: Select the following:
+     - Issue updated
+     - Comment created
+   - **JQL Filter**: Leave empty (or customize as needed)
+   - Click **Create**
+   - **Important**: Copy and store the webhook secret securely (you'll need this for workspace integration)
+
+---
+
+## Workspace Integration
+
+### Step 1: Log in to OpenHands Cloud
+
+1. **Navigate and Authenticate**
+   - Go to [OpenHands Cloud](https://app.all-hands.dev/)
+   - Sign in with your Git provider (GitHub, GitLab, or BitBucket)
+   - **Important:** Make sure you're signing in with the same Git provider account that contains the repositories you want the OpenHands agent to work on.
+
+### Step 2: Configure Jira Integration
+
+1. **Access Integration Settings**
+   - Navigate to **Settings** > **Integrations**
+   - Locate **Jira Cloud** section
+
+2. **Configure Workspace**
+   - Click **Configure** button
+   - Enter your workspace name and click **Connect**
+   - **Important:** Make sure you enter the full workspace name, eg: **yourcompany.atlassian.net**
+      - If no integration exists, you'll be prompted to enter additional credentials required for the workspace integration:
+         - **Webhook Secret**: The webhook secret from Step 3 above
+         - **Service Account Email**: The service account email from Step 1 above
+         - **Service Account API Key**: The API token from Step 2 above
+         - Ensure **Active** toggle is enabled
+
+<Note>
+Workspace name is the host name when accessing a resource in Jira Cloud.
+
+Eg: https://all-hands.atlassian.net/browse/OH-55
+
+Here the workspace name is **all-hands**.
+</Note>
+
+3. **Complete OAuth Flow**
+   - You'll be redirected to Jira Cloud to complete OAuth verification
+   - Grant the necessary permissions to verify your workspace access.
+   - If successful, you will be redirected back to the **Integrations** settings in the OpenHands Cloud UI
+
+### Managing Your Integration
+
+**Edit Configuration:**
+- Click the **Edit** button next to your configured platform
+- Update any necessary credentials or settings
+- Click **Update** to apply changes
+- You will need to repeat the OAuth flow as before
+- **Important:** Only the original user who created the integration can see the edit view
+
+**Unlink Workspace:**
+- In the edit view, click **Unlink** next to the workspace name
+- This will deactivate your workspace link
+- **Important:** If the original user who configured the integration chooses to unlink their integration, any users currently linked to that workspace integration will also be unlinked, and the workspace integration will be deactivated. The integration can only be reactivated by the original user.
+
+### Screenshots
+
+<AccordionGroup>
+<Accordion title="Workspace link flow">
+![workspace-link.png](/static/img/jira-user-link.png)
+</Accordion>
+
+<Accordion title="Workspace Configure flow">
+![workspace-link.png](/static/img/jira-admin-configure.png)
+</Accordion>
+
+<Accordion title="Edit view as a user">
+![workspace-link.png](/static/img/jira-user-unlink.png)
+</Accordion>
+
+<Accordion title="Edit view as the workspace creator">
+![workspace-link.png](/static/img/jira-admin-edit.png)
+</Accordion>
+</AccordionGroup>

docs/usage/cloud/project-management/linear-integration.mdx

@@ -0,0 +1,130 @@
+---
+title: Linear Integration
+description: Complete guide for setting up Linear integration with OpenHands Cloud, including service account creation, API key generation, webhook configuration, and workspace integration setup.
+---
+
+# Linear Integration
+
+## Platform Configuration
+
+### Step 1: Create Service Account
+
+1. **Access Team Settings**
+   - Log in to Linear as a team admin
+   - Go to **Settings** > **Members**
+
+2. **Invite Service Account**
+   - Click **Invite members**
+   - Email: `openhands@yourcompany.com` (replace with your preferred service account email)
+   - Role: **Member** (with appropriate team access)
+   - Send invitation
+
+3. **Complete Setup**
+   - Accept invitation from the service account email
+   - Complete profile setup
+   - Ensure access to relevant teams/workspaces
+
+### Step 2: Generate API Key
+
+1. **Access API Settings**
+   - Log in as the service account
+   - Go to **Settings** > **Security & access**
+
+2. **Create Personal API Key**
+   - Click **Create new key**
+   - Name: `OpenHands Cloud Integration`
+   - Scopes: Select the following:
+     - `Read` - Read access to issues and comments
+     - `Create comments` - Ability to create or update comments
+   - Select the teams you want to provide access to, or allow access for all teams you have permissions for
+   - Click **Create**
+   - **Important**: Copy and store the API key securely
+
+### Step 3: Configure Webhook
+
+1. **Access Webhook Settings**
+   - Go to **Settings** > **API** > **Webhooks**
+   - Click **New webhook**
+
+2. **Configure Webhook**
+   - **Label**: `OpenHands Cloud Integration`
+   - **URL**: `https://app.all-hands.dev/integration/linear/events`
+   - **Resource types**: Select:
+     - `Comment` - For comment events
+     - `Issue` - For issue updates (label changes)
+   - Select the teams you want to provide access to, or allow access for all public teams
+   - Click **Create webhook**
+   - **Important**: Copy and store the webhook secret securely (you'll need this for workspace integration)
+
+---
+
+## Workspace Integration
+
+### Step 1: Log in to OpenHands Cloud
+
+1. **Navigate and Authenticate**
+   - Go to [OpenHands Cloud](https://app.all-hands.dev/)
+   - Sign in with your Git provider (GitHub, GitLab, or BitBucket)
+   - **Important:** Make sure you're signing in with the same Git provider account that contains the repositories you want the OpenHands agent to work on.
+
+### Step 2: Configure Linear Integration
+
+1. **Access Integration Settings**
+   - Navigate to **Settings** > **Integrations**
+   - Locate **Linear** section
+
+2. **Configure Workspace**
+   - Click **Configure** button
+   - Enter your workspace name and click **Connect**
+      - If no integration exists, you'll be prompted to enter additional credentials required for the workspace integration:
+         - **Webhook Secret**: The webhook secret from Step 3 above
+         - **Service Account Email**: The service account email from Step 1 above
+         - **Service Account API Key**: The API key from Step 2 above
+         - Ensure **Active** toggle is enabled
+
+<Note>
+Workspace name is the identifier after the host name when accessing a resource in Linear.
+
+Eg: https://linear.app/allhands/issue/OH-37
+
+Here the workspace name is **allhands**.
+</Note>
+
+3. **Complete OAuth Flow**
+   - You'll be redirected to Linear to complete OAuth verification
+   - Grant the necessary permissions to verify your workspace access. If you have access to multiple workspaces, select the correct one that you initially provided
+   - If successful, you will be redirected back to the **Integrations** settings in the OpenHands Cloud UI
+
+### Managing Your Integration
+
+**Edit Configuration:**
+- Click the **Edit** button next to your configured platform
+- Update any necessary credentials or settings
+- Click **Update** to apply changes
+- You will need to repeat the OAuth flow as before
+- **Important:** Only the original user who created the integration can see the edit view
+
+**Unlink Workspace:**
+- In the edit view, click **Unlink** next to the workspace name
+- This will deactivate your workspace link
+- **Important:** If the original user who configured the integration chooses to unlink their integration, any users currently linked to that integration will also be unlinked, and the workspace integration will be deactivated. The integration can only be reactivated by the original user.
+
+### Screenshots
+
+<AccordionGroup>
+<Accordion title="Workspace link flow">
+![workspace-link.png](/static/img/linear-user-link.png)
+</Accordion>
+
+<Accordion title="Workspace Configure flow">
+![workspace-link.png](/static/img/linear-admin-configure.png)
+</Accordion>
+
+<Accordion title="Edit view as a user">
+![workspace-link.png](/static/img/linear-admin-edit.png)
+</Accordion>
+
+<Accordion title="Edit view as the workspace creator">
+![workspace-link.png](/static/img/workspace-admin-edit.png)
+</Accordion>
+</AccordionGroup>

docs/usage/cloud/project-management/overview.mdx

@@ -0,0 +1,80 @@
+---
+title: Project Management Tool Integrations
+description: Overview of OpenHands Cloud integrations with project management platforms including Jira Cloud, Jira Data Center, and Linear. Learn about setup requirements, usage methods, and troubleshooting.
+---
+
+# Project Management Tool Integrations
+
+## Overview
+
+OpenHands Cloud integrates with project management platforms (Jira Cloud, Jira Data Center, and Linear) to enable AI-powered task delegation. Users can invoke the OpenHands agent by:
+- Adding `@openhands` in ticket comments
+- Adding the `openhands` label to tickets
+
+## Prerequisites
+
+Integration requires two levels of setup:
+1. **Platform Configuration** - Administrative setup of service accounts and webhooks on your project management platform (see individual platform documentation below)
+2. **Workspace Integration** - Self-service configuration through the OpenHands Cloud UI to link your OpenHands account to the target workspace
+
+### Platform-Specific Setup Guides:
+- [Jira Cloud Integration](./jira-integration.md)
+- [Jira Data Center Integration](./jira-dc-integration.md)
+- [Linear Integration](./linear-integration.md)
+
+## Usage
+
+Once both the platform configuration and workspace integration are completed, users can trigger the OpenHands agent within their project management platforms using two methods:
+
+### Method 1: Comment Mention
+Add a comment to any issue with `@openhands` followed by your task description:
+```
+@openhands Please implement the user authentication feature described in this ticket
+```
+
+### Method 2: Label-based Delegation
+Add the label `openhands` to any issue. The OpenHands agent will automatically process the issue based on its description and requirements.
+
+### Git Repository Detection
+
+The OpenHands agent needs to identify which Git repository to work with when processing your issues. Here's how to ensure proper repository detection:
+
+#### Specifying the Target Repository
+
+**Required:** Include the target Git repository in your issue description or comment to ensure the agent works with the correct codebase.
+
+**Supported Repository Formats:**
+- Full HTTPS URL: `https://github.com/owner/repository.git`
+- GitHub URL without .git: `https://github.com/owner/repository`
+- Owner/repository format: `owner/repository`
+
+#### Platform-Specific Behavior
+
+**Linear Integration:** When GitHub integration is enabled for your Linear workspace with issue sync activated, the target repository is automatically detected from the linked GitHub issue. Manual specification is not required in this configuration.
+
+**Jira Integrations:** Always include the repository information in your issue description or `@openhands` comment to ensure proper repository detection.
+
+## Troubleshooting
+
+### Platform Configuration Issues
+- **Webhook not triggering**: Verify the webhook URL is correct and the proper event types are selected (Comment, Issue updated)
+- **API authentication failing**: Check API key/token validity and ensure required scopes are granted. If your current API token is expired, make sure to update it in the respective integration settings
+- **Permission errors**: Ensure the service account has access to relevant projects/teams and appropriate permissions
+
+### Workspace Integration Issues
+- **Workspace linking requests credentials**: If there are no active workspace integrations for the workspace you specified, you need to configure it first. Contact your platform administrator that you want to integrate with (eg: Jira, Linear)
+- **Integration not found**: Verify the workspace name matches exactly and that platform configuration was completed first
+- **OAuth flow fails**: Make sure that you're authorizing with the correct account with proper workspace access
+
+### General Issues
+- **Agent not responding**: Check webhook logs in your platform settings and verify service account status
+- **Authentication errors**: Verify Git provider permissions and OpenHands Cloud access
+- **Agent fails to identify git repo**: Ensure you're signing in with the same Git provider account that contains the repositories you want OpenHands to work on
+- **Partial functionality**: Ensure both platform configuration and workspace integration are properly completed
+
+### Getting Help
+For additional support, contact OpenHands Cloud support with:
+- Your integration platform (Linear, Jira Cloud, or Jira Data Center)
+- Workspace name
+- Error logs from webhook/integration attempts
+- Screenshots of configuration settings (without sensitive credentials)

docs/usage/cloud/slack-installation.mdx

@@ -12,6 +12,10 @@ description: This guide walks you through installing the OpenHands Slack app.
   allowFullScreen>
 </iframe>
 
+<Info>
+OpenHands utilizes a large language model (LLM), which may generate responses that are inaccurate or incomplete. While we strive for accuracy, OpenHands' outputs are not guaranteed to be correct, and we encourage users to validate critical information independently.
+</Info>
+
 ## Prerequisites
 
 - Access to OpenHands Cloud.
@@ -24,7 +28,7 @@ description: This guide walks you through installing the OpenHands Slack app.
   **This step is for Slack admins/owners**
 
   1. Make sure you have permissions to install Apps to your workspace.
-  2. Click the button below to install OpenHands Slack App <a target="_blank" href="https://slack.com/oauth/v2/authorize?client_id=7477886716822.8729519890534&scope=app_mentions:read,chat:write,users:read,channels:history,groups:history,mpim:history,im:history&user_scope=channels:history,groups:history,im:history,mpim:history"><img alt="Add to Slack" height="40" width="139" src="https://platform.slack-edge.com/img/add_to_slack.png" srcSet="https://platform.slack-edge.com/img/add_to_slack.png 1x, https://platform.slack-edge.com/img/add_to_slack@2x.png 2x" /></a>
+  2. Click the button below to install OpenHands Slack App <a target="_blank" href="https://slack.com/oauth/v2/authorize?client_id=7477886716822.8729519890534&scope=app_mentions:read,channels:history,chat:write,groups:history,im:history,mpim:history,users:read&user_scope="><img alt="Add to Slack" height="40" width="139" src="https://platform.slack-edge.com/img/add_to_slack.png" srcSet="https://platform.slack-edge.com/img/add_to_slack.png 1x, https://platform.slack-edge.com/img/add_to_slack@2x.png 2x" /></a>
   3. In the top right corner, select the workspace to install the OpenHands Slack app.
   4. Review permissions and click allow.
 
@@ -61,7 +65,7 @@ To send follow-up messages for the same conversation, mention `@openhands` in a
 
 Conversation is started by mentioning `@openhands`.
 
-![slack-create-convo.png](/static/img/slack-create-convo.png)
+![slack-create-conversation.png](/static/img/slack-create-conversation.png)
 
 ### See agent response and send follow up messages
 

docs/usage/configuration-options.mdx

@@ -379,10 +379,10 @@ To use these with the docker command, pass in `-e SANDBOX_<option>`. Example: `-
   - Description: Environment variables to set at the launch of the runtime
 
 ### Evaluation
-- `browser_use_config`
+- `browsergym_eval_env`
   - Type: `str`
   - Default: `""`
-  - Description: Browser-Use configuration to use for evaluation
+  - Description: BrowserGym environment to use for evaluation
 
 ## Security Configuration
 

docs/usage/how-to/cli-mode.mdx

@@ -20,27 +20,42 @@ for scripting.
 
 ### Running with Python
 
-**Note** - OpenHands requires Python version 3.12 or higher (Python 3.14 is not currently supported)
+**Note** - OpenHands requires Python version 3.12 or higher (Python 3.14 is not currently supported) and `uv` for the default `fetch` MCP server (more details below).
 
-1. Install OpenHands using pip:
-```bash
-pip install openhands-ai
-```
+#### Recommended: Using uv
 
-  Or if you prefer not to manage your own Python environment, you can use `uvx`:
+We recommend using [uv](https://docs.astral.sh/uv/) for the best OpenHands experience. uv provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers.
 
+1. **Install uv** (if you haven't already):
+
+   See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+2. **Launch OpenHands CLI**:
 ```bash
 uvx --python 3.12 --from openhands-ai openhands
 ```
 
 <AccordionGroup>
 
+<Accordion title="Alternative: Traditional pip installation">
+
+If you prefer to use pip:
+
+```bash
+# Install OpenHands
+pip install openhands-ai
+```
+
+Note that you'll still need `uv` installed for the default MCP servers to work properly.
+
+</Accordion>
+
 <Accordion title="Create shell aliases for easy access across environments">
 
 Add the following to your shell configuration file (`.bashrc`, `.zshrc`, etc.):
 
 ```bash
-# Add OpenHands aliases
+# Add OpenHands aliases (recommended)
 alias openhands="uvx --python 3.12 --from openhands-ai openhands"
 alias oh="uvx --python 3.12 --from openhands-ai openhands"
 ```
@@ -72,18 +87,19 @@ source ~/.bashrc  # or source ~/.zshrc
 
 </AccordionGroup>
 
-2. Launch an interactive OpenHands conversation from the command line:
+3. Launch an interactive OpenHands conversation from the command line:
 ```bash
-openhands
+# If using uvx (recommended)
+uvx --python 3.12 --from openhands-ai openhands
 ```
 
 <Note>
   If you have cloned the repository, you can also run the CLI directly using Poetry:
 
-  poetry run python -m openhands.cli.main
+  poetry run openhands
 </Note>
 
-3. Set your model, API key, and other preferences using the UI (or alternatively environment variables, below).
+4. Set your model, API key, and other preferences using the UI (or alternatively environment variables, below).
 
 This command opens an interactive prompt where you can type tasks or commands and get responses from OpenHands.
 The first time you run the CLI, it will take you through configuring the required LLM
@@ -103,7 +119,7 @@ The conversation history will be saved in `~/.openhands/sessions`.
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e SANDBOX_VOLUMES=$SANDBOX_VOLUMES \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -112,8 +128,8 @@ docker run -it \
     -v ~/.openhands:/.openhands \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49 \
-    python -m openhands.cli.main --override-cli-mode true
+    docker.all-hands.dev/all-hands-ai/openhands:0.54 \
+    python -m openhands.cli.entry --override-cli-mode true
 ```
 
 <Note>
@@ -153,6 +169,7 @@ You can use the following commands whenever the prompt (`>`) is displayed:
 | `/new`       | Start a new conversation                                       |
 | `/settings`  | View and modify current LLM/agent settings                     |
 | `/resume`    | Resume the agent if paused                                     |
+| `/mcp`       | Manage MCP server configuration and view connection errors    |
 
 #### Settings and Configuration
 
@@ -162,7 +179,7 @@ follow the prompts:
 - **Basic settings**: Choose a model/provider and enter your API key.
 - **Advanced settings**: Set custom endpoints, enable or disable confirmation mode, and configure memory condensation.
 
-Settings can also be managed via the `config.toml` file.
+Settings can also be managed via the `config.toml` file in the current directory or `~/.openhands/config.toml`.
 
 #### Repository Initialization
 
@@ -174,6 +191,41 @@ project details and structure. Use this when onboarding the agent to a new codeb
 You can pause the agent while it is running by pressing `Ctrl-P`. To continue the conversation after pausing, simply
 type `/resume` at the prompt.
 
+#### MCP Server Management
+
+To configure Model Context Protocol (MCP) servers, you can refer to the documentation on [MCP servers](../mcp) and use the `/mcp` command in the CLI. This command provides an interactive interface for managing Model Context Protocol (MCP) servers:
+
+- **List configured servers**: View all currently configured MCP servers (SSE, Stdio, and SHTTP)
+- **Add new server**: Interactively add a new MCP server with guided prompts
+- **Remove server**: Remove an existing MCP server from your configuration
+- **View errors**: Display any connection errors that occurred during MCP server startup
+
+This command modifies your `~/.openhands/config.toml` file and will prompt you to restart OpenHands for changes to take effect.
+
+By default, the [Fetch MCP server](https://github.com/modelcontextprotocol/servers/tree/main/src/fetch) will be automatically configured for OpenHands. You can also [enable search engine](../search-engine-setup) via the [Tavily MCP server](https://github.com/tavily-ai/tavily-mcp) by setting the `search_api_key` under the `[core]` section in the `~/.openhands/config.toml` file.
+
+##### Example of the `config.toml` file with MCP server configuration:
+
+```toml
+[core]
+search_api_key = "tvly-your-api-key-here"
+
+[mcp]
+stdio_servers = [
+    {name="fetch", command="uvx", args=["mcp-server-fetch"]},
+]
+
+sse_servers = [
+    # Basic SSE server with just a URL
+    "http://example.com:8080/sse",
+]
+
+shttp_servers = [
+    # Streamable HTTP server with API key authentication
+    {url="https://secure-example.com/mcp", api_key="your-api-key"}
+]
+```
+
 ## Tips and Troubleshooting
 
 - Use `/help` at any time to see the list of available commands.

docs/usage/how-to/gui-mode.mdx

@@ -7,6 +7,67 @@ description: High level overview of the Graphical User Interface (GUI) in OpenHa
 
 - [OpenHands is running](/usage/local-setup)
 
+## Launching the GUI Server
+
+### Using the CLI Command
+
+You can launch the OpenHands GUI server directly from the command line using the `serve` command:
+
+<Callout type="info">
+**Prerequisites**: You need to have the [OpenHands CLI installed](/usage/how-to/cli-mode) first, OR have `uv` installed and run `uvx --python 3.12 --from openhands-ai openhands serve`. Otherwise, you'll need to use Docker directly (see the [Docker section](#using-docker-directly) below).
+</Callout>
+
+```bash
+openhands serve
+```
+
+This command will:
+- Check that Docker is installed and running
+- Pull the required Docker images
+- Launch the OpenHands GUI server at http://localhost:3000
+- Use the same configuration directory (`~/.openhands`) as the CLI mode
+
+#### Mounting Your Current Directory
+
+To mount your current working directory into the GUI server container, use the `--mount-cwd` flag:
+
+```bash
+openhands serve --mount-cwd
+```
+
+This is useful when you want to work on files in your current directory through the GUI. The directory will be mounted at `/workspace` inside the container.
+
+#### Using GPU Support
+
+If you have NVIDIA GPUs and want to make them available to the OpenHands container, use the `--gpu` flag:
+
+```bash
+openhands serve --gpu
+```
+
+This will enable GPU support via nvidia-docker, mounting all available GPUs into the container. You can combine this with other flags:
+
+```bash
+openhands serve --gpu --mount-cwd
+```
+
+**Prerequisites for GPU support:**
+- NVIDIA GPU drivers must be installed on your host system
+- [NVIDIA Container Toolkit (nvidia-docker2)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) must be installed and configured
+
+#### Requirements
+
+Before using the `openhands serve` command, ensure that:
+- Docker is installed and running on your system
+- You have internet access to pull the required Docker images
+- Port 3000 is available on your system
+
+The CLI will automatically check these requirements and provide helpful error messages if anything is missing.
+
+### Using Docker Directly
+
+Alternatively, you can run the GUI server using Docker directly. See the [local setup guide](/usage/local-setup) for detailed Docker instructions.
+
 ## Overview
 
 ### Initial Setup

docs/usage/how-to/headless-mode.mdx

@@ -61,7 +61,7 @@ export GITHUB_TOKEN="your-token"  # Required for repository operations
 # Run OpenHands
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e SANDBOX_VOLUMES=$SANDBOX_VOLUMES \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -73,7 +73,7 @@ docker run -it \
     -v ~/.openhands:/.openhands \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.54 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
 

docs/usage/llms/llms.mdx

@@ -18,7 +18,7 @@ Based on these findings and community feedback, these are the latest models that
 ### Cloud / API-Based Models
 
 - [anthropic/claude-sonnet-4-20250514](https://www.anthropic.com/api) (recommended)
-- [openai/o4-mini](https://openai.com/index/introducing-o3-and-o4-mini/)
+- [openai/gpt-5-2025-08-07](https://openai.com/api/) (recommended)
 - [gemini/gemini-2.5-pro](https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/)
 - [deepseek/deepseek-chat](https://api-docs.deepseek.com/)
 - [moonshot/kimi-k2-0711-preview](https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2)
@@ -39,6 +39,12 @@ limits and monitor usage.
 - [mistralai/devstral-small](https://www.all-hands.dev/blog/devstral-a-new-state-of-the-art-open-model-for-coding-agents) (20 May 2025) -- also available through [OpenRouter](https://openrouter.ai/mistralai/devstral-small:free)
 - [all-hands/openhands-lm-32b-v0.1](https://www.all-hands.dev/blog/introducing-openhands-lm-32b----a-strong-open-coding-agent-model) (31 March 2025) -- also available through [OpenRouter](https://openrouter.ai/all-hands/openhands-lm-32b-v0.1)
 
+### Known Issues
+
+<Warning>
+As of July 2025, there are known issues with Gemini 2.5 Pro conversations taking longer than normal with OpenHands. We are continuing to investigate.
+</Warning>
+
 <Note>
 Most current local and open source models are not as powerful. When using such models, you may see long
 wait times between messages, poor responses, or errors about malformed JSON. OpenHands can only be as powerful as the

docs/usage/llms/local-llms.mdx

@@ -68,23 +68,23 @@ Download and install the LM Studio desktop app from [lmstudio.ai](https://lmstud
 1. Check [the installation guide](/usage/local-setup) and ensure all prerequisites are met before running OpenHands, then run:
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49
+    docker.all-hands.dev/all-hands-ai/openhands:0.54
 ```
 
 2. Wait until the server is running (see log below):
 ```
 Digest: sha256:e72f9baecb458aedb9afc2cd5bc935118d1868719e55d50da73190d3a85c674f
-Status: Image is up to date for docker.all-hands.dev/all-hands-ai/openhands:0.49
+Status: Image is up to date for docker.all-hands.dev/all-hands-ai/openhands:0.54
 Starting OpenHands...
 Running OpenHands as root
 14:22:13 - openhands:INFO: server_config.py:50 - Using config class None

docs/usage/llms/openhands-llms.mdx

@@ -30,5 +30,6 @@ When running OpenHands, you'll need to set the following in the OpenHands UI thr
 
 ## Pricing
 
-Pricing follows official API provider rates.
-[You can view model prices here.](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
+Pricing follows official API provider rates. [You can view model prices here.](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
+
+For `qwen3-coder-480b`, we charge the cheapest FP8 rate available on openrouter: \$0.4 per million input tokens and \$1.6 per million output tokens.

docs/usage/local-setup.mdx

@@ -66,20 +66,64 @@ A system with a modern processor and a minimum of **4GB RAM** is recommended to
 
 ### Start the App
 
+#### Option 1: Using the CLI Launcher with uv (Recommended)
+
+We recommend using [uv](https://docs.astral.sh/uv/) for the best OpenHands experience. uv provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers (like the [fetch MCP server](https://github.com/modelcontextprotocol/servers/tree/main/src/fetch)).
+
+**Install uv** (if you haven't already):
+
+See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+**Launch OpenHands**:
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik
+# Launch the GUI server
+uvx --python 3.12 --from openhands-ai openhands serve
+
+# Or with GPU support (requires nvidia-docker)
+uvx --python 3.12 --from openhands-ai openhands serve --gpu
+
+# Or with current directory mounted
+uvx --python 3.12 --from openhands-ai openhands serve --mount-cwd
+```
+
+This will automatically handle Docker requirements checking, image pulling, and launching the GUI server. The `--gpu` flag enables GPU support via nvidia-docker, and `--mount-cwd` mounts your current directory into the container.
+
+<Accordion title="Alternative: Traditional pip installation">
+
+If you prefer to use pip and have Python 3.12+ installed:
+
+```bash
+# Install OpenHands
+pip install openhands-ai
+
+# Launch the GUI server
+openhands serve
+```
+
+Note that you'll still need `uv` installed for the default MCP servers to work properly.
+
+</Accordion>
+
+#### Option 2: Using Docker Directly
+
+<Accordion title="Docker Command (Click to expand)">
+
+```bash
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.49-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.54-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.49
+    docker.all-hands.dev/all-hands-ai/openhands:0.54
 ```
 
+</Accordion>
+
 > **Note**: If you used OpenHands before version 0.44, you may want to run `mv ~/.openhands-state ~/.openhands` to migrate your conversation history to the new location.
 
 You'll find OpenHands running at http://localhost:3000!
@@ -100,6 +144,16 @@ OpenHands requires an API key to access most language models. Here's how to get
 
 <AccordionGroup>
 
+<Accordion title="OpenHands (Recommended)">
+
+1. [Log in to OpenHands Cloud](https://app.all-hands.dev).
+2. Go to the Settings page and navigate to the `API Keys` tab.
+3. Copy your `LLM API Key`.
+
+OpenHands provides access to state-of-the-art agentic coding models with competitive pricing. [Learn more about OpenHands LLM provider](/usage/llms/openhands-llms).
+
+</Accordion>
+
 <Accordion title="Anthropic (Claude)">
 
 1. [Create an Anthropic account](https://console.anthropic.com/).

docs/usage/mcp.mdx

@@ -10,47 +10,83 @@ Model Context Protocol (MCP) is a mechanism that allows OpenHands to communicate
 servers can provide additional functionality to the agent, such as specialized data processing, external API access,
 or custom tools. MCP is based on the open standard defined at [modelcontextprotocol.io](https://modelcontextprotocol.io).
 
+
+<Note>
+MCP is currently not available on OpenHands Cloud. This feature is only available when running OpenHands locally.
+</Note>
+
+### How MCP Works
+
+When OpenHands starts, it:
+
+1. Reads the MCP configuration.
+2. Connects to any configured SSE and SHTTP servers.
+3. Starts any configured stdio servers.
+4. Registers the tools provided by these servers with the agent.
+
+The agent can then use these tools just like any built-in tool. When the agent calls an MCP tool:
+
+1. OpenHands routes the call to the appropriate MCP server.
+2. The server processes the request and returns a response.
+3. OpenHands converts the response to an observation and presents it to the agent.
+
 ## Configuration
 
 MCP configuration can be defined in:
 * The OpenHands UI through the Settings under the `MCP` tab.
 * The `config.toml` file under the `[mcp]` section if not using the UI.
 
-### Configuration Example via config.toml
+### Configuration Examples
+
+#### Recommended: Using Proxy Servers (SSE/HTTP)
+
+For stdio-based MCP servers, we recommend using MCP proxy tools like [`supergateway`](https://github.com/supercorp-ai/supergateway) instead of direct stdio connections.
+[SuperGateway](https://github.com/supercorp-ai/supergateway) is a popular MCP proxy that converts stdio MCP servers to HTTP/SSE endpoints:
+
+Start the proxy servers separately:
+```bash
+# Terminal 1: Filesystem server proxy
+supergateway --stdio "npx @modelcontextprotocol/server-filesystem /" --port 8080
+
+# Terminal 2: Fetch server proxy
+supergateway --stdio "uvx mcp-server-fetch" --port 8081
+```
+
+Then configure OpenHands to use the HTTP endpoint:
 
 ```toml
 [mcp]
-# SSE Servers - External servers that communicate via Server-Sent Events
+# SSE Servers - Recommended approach using proxy tools
 sse_servers = [
     # Basic SSE server with just a URL
     "http://example.com:8080/mcp",
 
-    # SSE server with API key authentication
-    {url="https://secure-example.com/mcp", api_key="your-api-key"}
+    # SuperGateway proxy for fetch server
+    "http://localhost:8081/sse",
+
+    # External MCP service with authentication
+    {url="https://api.example.com/mcp/sse", api_key="your-api-key"}
 ]
+```
 
-# SHTTP Servers - External servers that communicate via Streamable HTTP
-shttp_servers = [
-    # Basic SHTTP server with just a URL
-    "http://example.com:8080/mcp",
 
-    # SHTTP server with API key authentication
-    {url="https://secure-example.com/mcp", api_key="your-api-key"}
-]
 
-# Stdio Servers - Local processes that communicate via standard input/output
+#### Alternative: Direct Stdio Servers (Not Recommended for Production)
+
+```toml
+[mcp]
+# Direct stdio servers - use only for development/testing
 stdio_servers = [
     # Basic stdio server
     {name="fetch", command="uvx", args=["mcp-server-fetch"]},
 
     # Stdio server with environment variables
     {
-        name="data-processor",
-        command="python",
-        args=["-m", "my_mcp_server"],
+        name="filesystem",
+        command="npx",
+        args=["@modelcontextprotocol/server-filesystem", "/"],
         env={
-            "DEBUG": "true",
-            "PORT": "8080"
+            "DEBUG": "true"
         }
     }
 ]
@@ -84,6 +120,8 @@ SHTTP (Streamable HTTP) servers are configured using either a string URL or an o
 
 ### Stdio Servers
 
+**Note**: While stdio servers are supported, we recommend using MCP proxies (see above) for better reliability and performance.
+
 Stdio servers are configured using an object with the following properties:
 
 - `name` (required)
@@ -104,20 +142,38 @@ Stdio servers are configured using an object with the following properties:
   - Default: `{}`
   - Description: Environment variables to set for the server process
 
-## How MCP Works
 
-When OpenHands starts, it:
+#### When to Use Direct Stdio
 
-1. Reads the MCP configuration.
-2. Connects to any configured SSE and SHTTP servers.
-3. Starts any configured stdio servers.
-4. Registers the tools provided by these servers with the agent.
+Direct stdio connections may still be appropriate in these scenarios:
+- **Development and testing**: Quick prototyping of MCP servers
+- **Simple, single-use tools**: Tools that don't require high reliability or concurrent access
+- **Local-only environments**: When you don't want to manage additional proxy processes
 
-The agent can then use these tools just like any built-in tool. When the agent calls an MCP tool:
+For production use, we recommend using proxy tools like SuperGateway.
 
-1. OpenHands routes the call to the appropriate MCP server.
-2. The server processes the request and returns a response.
-3. OpenHands converts the response to an observation and presents it to the agent.
+### Other Proxy Tools
+
+Other options include:
+
+- **Custom FastAPI/Express servers**: Build your own HTTP wrapper around stdio MCP servers
+- **Docker-based proxies**: Containerized solutions for better isolation
+- **Cloud-hosted MCP services**: Third-party services that provide MCP endpoints
+
+### Troubleshooting MCP Connections
+
+#### Common Issues with Stdio Servers
+- **Process crashes**: Stdio processes may crash without proper error handling
+- **Deadlocks**: Stdio communication can deadlock under high load
+- **Resource leaks**: Zombie processes if not properly managed
+- **Debugging difficulty**: Hard to inspect stdio communication
+
+#### Benefits of Using Proxies
+- **HTTP status codes**: Clear error reporting via standard HTTP responses
+- **Request logging**: Easy to log and monitor HTTP requests
+- **Load balancing**: Can distribute requests across multiple server instances
+- **Health checks**: HTTP endpoints can provide health status
+- **CORS support**: Better integration with web-based tools
 
 ## Transport Protocols
 

docs/usage/runtimes/docker.mdx

@@ -130,3 +130,28 @@ docker run # ... \
 <Note>
 **Docker Desktop Required**: Network isolation features, including custom networks and `host.docker.internal` routing, require Docker Desktop. Docker Engine alone does not support these features on localhost across custom networks. If you're using Docker Engine without Docker Desktop, network isolation may not work as expected.
 </Note>
+
+### Sidecar Containers
+
+If you want to run sidecar containers to the sandbox 'runner' containers without exposing the sandbox containers to the host network, you can use the `SANDBOX_ADDITIONAL_NETWORKS` environment variable to specify additional Docker network names that should be added to the sandbox containers.
+
+```bash
+docker network create openhands-sccache
+
+docker run -d \
+  --hostname openhandsredis \
+  --network openhands-sccache \
+  redis
+
+docker run # ...
+    -e SANDBOX_ADDITIONAL_NETWORKS='["openhands-sccache"]' \
+    # ...
+```
+
+Then all sandbox instances will have to access a shared redis instance at `openhandsredis:6379`.
+
+#### Docker Compose gotcha
+
+Note that Docker Compose adds a prefix (a scope) by default to created networks, which is not taken into account by the additional networks config. Therefore when using docker compose you have to either:
+- specify a network name via the `name` field to remove the scoping (https://docs.docker.com/reference/compose-file/networks/#name) 
+- or provide the scope within the given config (e.g. `SANDBOX_ADDITIONAL_NETWORKS: '["myscope_openhands-sccache"]'` where `myscope` is the docker-compose assigned prefix). 

evaluation/benchmarks/EDA/run_infer.py

@@ -9,7 +9,8 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     compatibility_for_eval_history_pairs,
-    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -18,8 +19,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -60,18 +61,15 @@ AGENT_CLS_TO_INST_SUFFIX = {
 def get_config(
     metadata: EvalMetadata,
 ) -> OpenHandsConfig:
-    sandbox_config = get_default_sandbox_config_for_eval()
-    sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    # Create config with EDA-specific container image
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
+
+    # Override the container image for EDA
+    config.sandbox.base_container_image = 'python:3.12-bookworm'
+
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
     agent_config.enable_prompt_extensions = False
@@ -146,7 +144,7 @@ def process_instance(
 
     logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
     test_result = game.reward()
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
@@ -172,7 +170,7 @@ def process_instance(
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--answerer_model', '-a', default='gpt-3.5-turbo', help='answerer model'
     )

evaluation/benchmarks/agent_bench/run_infer.py

@@ -17,7 +17,8 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     compatibility_for_eval_history_pairs,
-    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -40,19 +41,12 @@ from openhands.utils.async_utils import call_async_from_sync
 def get_config(
     metadata: EvalMetadata,
 ) -> OpenHandsConfig:
-    sandbox_config = get_default_sandbox_config_for_eval()
-    sandbox_config.base_container_image = 'python:3.12-slim'
+    # Create config with agent_bench-specific container image
+    config = get_openhands_config_for_eval(metadata=metadata)
+
+    # Override the container image for agent_bench
+    config.sandbox.base_container_image = 'python:3.12-slim'
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
     agent_config.enable_prompt_extensions = False
@@ -273,7 +267,7 @@ def process_instance(
     # remove when it becomes unnecessary
     histories = compatibility_for_eval_history_pairs(state.history)
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/aider_bench/run_infer.py

@@ -17,6 +17,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -49,15 +51,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.11-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -246,7 +243,7 @@ def process_instance(
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
     histories = compatibility_for_eval_history_pairs(state.history)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/biocoder/run_infer.py

@@ -15,6 +15,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -60,15 +62,10 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = BIOCODER_BENCH_CONTAINER_IMAGE
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -294,7 +291,7 @@ def process_instance(
         raise ValueError('State should not be None.')
 
     test_result = complete_runtime(runtime, instance)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary

evaluation/benchmarks/bird/run_infer.py

@@ -18,6 +18,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -74,15 +76,10 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -422,7 +419,7 @@ def process_instance(
     # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
     if state is None:
         raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/browsing_delegation/run_infer.py

@@ -11,6 +11,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -39,14 +41,8 @@ def get_config(
     )
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        workspace_base=None,
-        workspace_mount_path=None,
+    config = get_openhands_config_for_eval(
+        metadata=metadata, runtime='docker', sandbox_config=sandbox_config
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -88,7 +84,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary

evaluation/benchmarks/commit0/run_infer.py

@@ -16,6 +16,8 @@ from evaluation.utils.shared import (
     assert_and_raise,
     codeact_user_response,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -26,8 +28,8 @@ from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -113,15 +115,11 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = base_container_image
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        enable_browser=RUN_WITH_BROWSING,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -479,7 +477,7 @@ def process_instance(
 
     # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
     histories = [event_to_dict(event) for event in state.history]
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(
@@ -505,7 +503,6 @@ def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
     Returns:
         Filtered dataset based on split type
     """
-
     filtered_dataset = pd.concat(
         [
             dataset[dataset['repo'].str.split('/').str[1] == repo]
@@ -524,7 +521,7 @@ def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--dataset',
         type=str,

evaluation/benchmarks/discoverybench/run_infer.py

@@ -17,6 +17,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -64,15 +66,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -89,8 +86,7 @@ def get_config(
 def get_dv_query_for_real(
     datasets, question, domain_knowledge=None, workflow_tags=None
 ):
-    """
-    Prepare a structured query for the agent to execute on the specified datasets.
+    """Prepare a structured query for the agent to execute on the specified datasets.
 
     This function constructs a query by compiling metadata from the provided datasets, along with any relevant domain knowledge and workflow tags.
 
@@ -104,7 +100,6 @@ def get_dv_query_for_real(
         query_to_dv: Query to be run on the dataset
         dataset_meta: Metadata of the dataset
     """
-
     dataset_meta = ''
     for dataset_metadata in datasets:
         dataset_meta += 'Dataset name: ' + dataset_metadata['name']
@@ -140,8 +135,7 @@ def get_dv_query_for_real(
 
 
 def initialize_runtime(runtime: Runtime, data_files: list[str]):
-    """
-    Initialize the runtime for the agent.
+    """Initialize the runtime for the agent.
 
     This function is called before the runtime is used to run the agent.
     """
@@ -231,8 +225,7 @@ def process_instance(
     metadata: EvalMetadata,
     reset_logger: bool = True,
 ):
-    """
-    Process and evaluate a single instance of the dataset.
+    """Process and evaluate a single instance of the dataset.
 
     This function executes the OpenHands agent
     for a specific instance of the dataset. It retrieves
@@ -247,7 +240,6 @@ def process_instance(
     Returns:
         output: EvalOutput object
     """
-
     config = get_config(metadata)
 
     # Setup the logger properly, so you can run
@@ -299,7 +291,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     test_result = complete_runtime(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
@@ -356,8 +348,7 @@ def list_csv_files(list_of_datasets):
 
 
 def create_dataset(repo_location: str, split: str = 'test'):
-    """
-    Create a dataset from the discoverybench repository
+    """Create a dataset from the discoverybench repository
     by walking through the repository and extracting metadata
     from the metadata_{}.json files
 
@@ -368,7 +359,6 @@ def create_dataset(repo_location: str, split: str = 'test'):
     Returns:
         df: DataFrame containing the dataset instances
     """
-
     data_dict = {}
 
     data_location = os.path.join(repo_location, 'discoverybench', 'real', split)

evaluation/benchmarks/gaia/run_infer.py

@@ -10,7 +10,6 @@ import huggingface_hub
 import pandas as pd
 from datasets import load_dataset
 from PIL import Image
-from pydantic import SecretStr
 
 from evaluation.benchmarks.gaia.scorer import question_scorer
 from evaluation.benchmarks.gaia.utils import (
@@ -23,6 +22,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -31,8 +32,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
     load_from_toml,
 )
 from openhands.core.config.utils import get_agent_config_arg
@@ -60,15 +61,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'nikolaik/python-nodejs:python3.12-nodejs22'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
     config.set_llm_config(metadata.llm_config)
     if metadata.agent_config:
@@ -80,8 +76,7 @@ def get_config(
 
     config_copy = copy.deepcopy(config)
     load_from_toml(config_copy)
-    if config_copy.search_api_key:
-        config.search_api_key = SecretStr(config_copy.search_api_key)
+    config.search_api_key = config_copy.search_api_key
     return config
 
 
@@ -271,7 +266,7 @@ Here is the task:
         'model_answer': model_answer,
         'ground_truth': instance['Final answer'],
     }
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
@@ -294,7 +289,7 @@ Here is the task:
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--level',
         type=str,

evaluation/benchmarks/gorilla/run_infer.py

@@ -12,6 +12,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -20,8 +22,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -42,15 +44,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -108,7 +105,7 @@ def process_instance(
     # attempt to parse model_answer
     ast_eval_fn = instance['ast_eval']
     correct, hallucination = ast_eval_fn(instance_id, model_answer_raw)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     logger.info(
         f'Final message: {model_answer_raw} | Correctness: {correct} | Hallucination: {hallucination}'
     )
@@ -134,7 +131,7 @@ def process_instance(
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--hubs',
         type=str,

evaluation/benchmarks/gpqa/run_infer.py

@@ -30,6 +30,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -38,8 +40,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -63,15 +65,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -292,7 +289,7 @@ Ok now its time to start solving the question. Good luck!
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(
@@ -312,7 +309,7 @@ Ok now its time to start solving the question. Good luck!
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     # data split must be one of 'gpqa_main', 'gqpa_diamond', 'gpqa_experts', 'gpqa_extended'
     parser.add_argument(
         '--data-split',

evaluation/benchmarks/humanevalfix/run_infer.py

@@ -23,6 +23,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -84,15 +86,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -248,7 +245,7 @@ def process_instance(
 
     if state is None:
         raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     test_result = complete_runtime(runtime, instance)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)

evaluation/benchmarks/lca_ci_build_repair/eval_infer.py

@@ -16,12 +16,13 @@ import ruamel.yaml
 from evaluation.utils.shared import (
     EvalMetadata,
     get_default_sandbox_config_for_eval,
+    get_openhands_config_for_eval,
     make_metadata,
 )
 from openhands.core.config import (
     LLMConfig,
     OpenHandsConfig,
-    get_parser,
+    get_evaluation_parser,
     load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
@@ -37,15 +38,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -167,7 +163,7 @@ def process_predictions(predictions_path: str):
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '-s',
         '--eval-split',

evaluation/benchmarks/lca_ci_build_repair/run_infer.py

@@ -22,6 +22,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -30,8 +32,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
     load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
@@ -47,15 +49,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -335,7 +332,7 @@ Be thorough in your exploration, testing, and reasoning. It's fine if your think
         )
     )
     assert state is not None
-    metrics = state.metrics.get() if state.metrics else {}
+    metrics = get_metrics(state)
 
     test_result = complete_runtime(runtime, instance)
 
@@ -358,7 +355,7 @@ Be thorough in your exploration, testing, and reasoning. It's fine if your think
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '-s',
         '--eval-split',

evaluation/benchmarks/logic_reasoning/run_infer.py

@@ -10,6 +10,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -18,8 +20,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -51,15 +53,10 @@ def get_config(
         '$OH_INTERPRETER_PATH -m pip install scitools-pyke'
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -247,7 +244,7 @@ def process_instance(
     )
     test_result['final_message'] = final_message
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
@@ -267,7 +264,7 @@ def process_instance(
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--dataset',
         type=str,

evaluation/benchmarks/miniwob/README.md

@@ -1,6 +1,6 @@
-# MiniWoB++ Evaluation
+# Mini-World of Bits Evaluation with OpenHands Browsing Agents
 
-This folder contains evaluation for [MiniWoB++](https://miniwob.farama.org/) benchmark, powered by [Browser-Use](https://github.com/browser-use/browser-use) for easy evaluation of how well an agent capable of browsing can perform on synthetic web browsing tasks.
+This folder contains evaluation for [MiniWoB++](https://miniwob.farama.org/) benchmark, powered by [BrowserGym](https://github.com/ServiceNow/BrowserGym) for easy evaluation of how well an agent capable of browsing can perform on synthetic web browsing tasks.
 
 ## Setup Environment and LLM Configuration
 

evaluation/benchmarks/miniwob/get_avg_reward.py

@@ -1,17 +1,33 @@
+import argparse
 import json
-import os
-import pandas as pd
-from openhands.core.logger import openhands_logger as logger
 
-# TODO: Update to work with Browser-Use evaluation environments
-# import browsergym.miniwob  # noqa F401 register miniwob tasks as gym environments
+import browsergym.miniwob  # noqa F401 register miniwob tasks as gym environments
+import gymnasium as gym
 
-def get_avg_reward(output_file: str) -> float:
-    """Get average reward from output file."""
-    if not os.path.exists(output_file):
-        logger.warning(f'Output file {output_file} does not exist')
-        return 0.0
+parser = argparse.ArgumentParser(description='Calculate average reward.')
+parser.add_argument('output_path', type=str, help='path to output.jsonl')
 
-    # TODO: Update environment ID filtering for Browser-Use
-    # For now, return 0.0 as we need to implement Browser-Use evaluation
-    return 0.0
+args = parser.parse_args()
+
+if __name__ == '__main__':
+    env_ids = [
+        id for id in gym.envs.registry.keys() if id.startswith('browsergym/miniwob')
+    ]
+    total_num = len(env_ids)
+    print('Total number of tasks: ', total_num)
+    total_reward = 0
+    total_cost = 0
+    actual_num = 0
+    with open(args.output_path, 'r') as f:
+        for line in f:
+            data = json.loads(line)
+            actual_num += 1
+            total_cost += data['metrics']['accumulated_cost']
+            total_reward += data['test_result']['reward']
+
+    avg_reward = total_reward / total_num
+    print('Avg Reward: ', avg_reward)
+
+    avg_cost = total_cost / actual_num
+    print('Avg Cost: ', avg_cost)
+    print('Actual number of tasks finished: ', actual_num)

evaluation/benchmarks/miniwob/run_infer.py

@@ -3,8 +3,7 @@ import json
 import os
 from typing import Any
 
-# TODO: Update to work with Browser-Use evaluation environments
-# import browsergym.miniwob  # noqa F401 register miniwob tasks as gym environments
+import browsergym.miniwob  # noqa F401 register miniwob tasks as gym environments
 import gymnasium as gym
 import pandas as pd
 
@@ -14,6 +13,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -58,15 +59,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'xingyaoww/od-eval-miniwob:v1.0'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -175,7 +171,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Instruction is the first message from the USER
     instruction = ''
@@ -214,11 +210,9 @@ if __name__ == '__main__':
     dataset = pd.DataFrame(
         {
             'instance_id': [
-                # TODO: Update to work with Browser-Use evaluation environments
-                # For now, return empty list as we need to implement Browser-Use evaluation
-                # id
-                # for id in gym.envs.registry.keys()
-                # if id.startswith('browsergym/miniwob')
+                id
+                for id in gym.envs.registry.keys()
+                if id.startswith('browsergym/miniwob')
             ]
         }
     )

evaluation/benchmarks/mint/run_infer.py

@@ -15,6 +15,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -23,8 +25,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -109,15 +111,10 @@ def get_config(
         f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}'
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -205,7 +202,7 @@ def process_instance(
         task_state = state.extra_data['task_state']
         logger.info('Task state: ' + str(task_state.to_dict()))
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
@@ -229,7 +226,7 @@ def process_instance(
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
 
     SUBSETS = [
         # Eurus subset: https://arxiv.org/abs/2404.02078

evaluation/benchmarks/ml_bench/run_analysis.py

@@ -4,7 +4,11 @@ import pprint
 
 import tqdm
 
-from openhands.core.config import get_llm_config_arg, get_parser, load_openhands_config
+from openhands.core.config import (
+    get_evaluation_parser,
+    get_llm_config_arg,
+    load_openhands_config,
+)
 from openhands.core.logger import openhands_logger as logger
 from openhands.llm.llm import LLM
 
@@ -111,7 +115,7 @@ def classify_error(llm: LLM, failed_case: dict) -> str:
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--json_file_path',
         type=str,

evaluation/benchmarks/ml_bench/run_infer.py

@@ -26,6 +26,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -34,8 +36,8 @@ from evaluation.utils.shared import (
 from openhands.controller.state.state import State
 from openhands.core.config import (
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
     load_openhands_config,
 )
 from openhands.core.logger import openhands_logger as logger
@@ -79,15 +81,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'public.ecr.aws/i5g0m1f6/ml-bench'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -250,7 +247,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
         )
     )
     assert state is not None
-    metrics = state.metrics.get() if state.metrics else {}
+    metrics = get_metrics(state)
 
     test_result = complete_runtime(runtime)
 
@@ -273,7 +270,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '-s',
         '--eval-split',

evaluation/benchmarks/multi_swe_bench/eval_infer.py

@@ -23,6 +23,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     get_default_sandbox_config_for_eval,
+    get_openhands_config_for_eval,
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
@@ -30,7 +31,7 @@ from evaluation.utils.shared import (
 from openhands.core.config import (
     LLMConfig,
     OpenHandsConfig,
-    get_parser,
+    get_evaluation_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime
@@ -87,13 +88,9 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
         dataset_name=metadata.dataset,
         instance_id=instance['instance_id'],
     )
-    config = OpenHandsConfig(
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     return config
 
@@ -105,8 +102,7 @@ def process_instance(
     log_dir: str | None = None,
     runtime_failure_count: int = 0,
 ) -> EvalOutput:
-    """
-    Evaluate agent performance on a SWE-bench problem instance.
+    """Evaluate agent performance on a SWE-bench problem instance.
 
     Note that this signature differs from the expected input to `run_evaluation`. Use
     `functools.partial` to provide optional arguments before passing to the evaluation harness.
@@ -323,7 +319,7 @@ def process_instance(
 
 
 if __name__ == '__main__':
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--input-file',
         type=str,

evaluation/benchmarks/multi_swe_bench/run_infer.py

@@ -21,6 +21,7 @@ from evaluation.utils.shared import (
     codeact_user_response,
     get_default_sandbox_config_for_eval,
     get_metrics,
+    get_openhands_config_for_eval,
     is_fatal_evaluation_error,
     make_metadata,
     prepare_dataset,
@@ -32,8 +33,8 @@ from openhands.controller.state.state import State
 from openhands.core.config import (
     AgentConfig,
     OpenHandsConfig,
+    get_evaluation_parser,
     get_llm_config_arg,
-    get_parser,
 )
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.main import create_runtime, run_controller
@@ -341,15 +342,11 @@ def get_config(
         instance_id=instance['instance_id'],
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        enable_browser=RUN_WITH_BROWSING,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -771,7 +768,7 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
 
 if __name__ == '__main__':
     # pdb.set_trace()
-    parser = get_parser()
+    parser = get_evaluation_parser()
     parser.add_argument(
         '--dataset',
         type=str,

evaluation/benchmarks/nocode_bench/README.md

@@ -0,0 +1,45 @@
+# Evaluate OpenHands on NoCode-bench
+
+## LLM Setup
+
+Please follow [here](../../README.md#setup).
+
+
+## Docker image download
+
+Evaluating OpenHands on NoCode-bench need instance-level docker image.
+Please follow the instructions of NoCode-bench image setup to build or download all instance-level dokcer [here](https://github.com/NoCode-bench/NoCode-bench).
+
+## Generate patch
+
+Please follow the instructions [here](../swe_bench/README.md#running-locally-with-docker)
+For example,
+```bash
+bash ./evaluation/benchmarks/nocode_bench/scripts/run_infer_nc.sh llm.claude HEAD CodeActAgent 114 100 10 NoCode-bench/NoCode-bench_Verified test
+```
+The results will be generated in evaluation/evaluation_outputs/outputs/XXX/CodeActAgent/YYY/output.jsonl.
+
+## Runing evaluation
+
+First, install [NoCode-bench](https://github.com/NoCode-bench/NoCode-bench).
+
+Second, convert the output.jsonl to patch.jsonl with [script](scripts/eval/convert.py).
+
+```bash
+python evaluation/benchmarks/multi_swe_bench/scripts/eval/convert.py
+```
+
+Finally, evaluate with NoCode-bench.
+
+```bash
+export PYTHONPATH=$PYTHONPATH:$(pwd)
+python ./evaluation/eval.py \
+    --predictions_path ./all_preds.jsonl \  # <path_to_your_predictions>
+    --log_dir ./evaluation/logs \ # <path_to_your_log_dir>
+    --bench_tasks NoCode-bench/NoCode-bench_Verified \ # <dataset_name>
+    --max_workers 110 \ # <number_of_workers>
+    --output_file eval_result.txt \ # <path_to_your_output_file>
+    --image_level repo \ # <cache_image_level>
+    --timeout 600 \ # <timeout_in_seconds>
+    --proxy None # <proxy_if_needed>
+```

evaluation/benchmarks/nocode_bench/binary_patch_utils.py

@@ -0,0 +1,52 @@
+"""
+Utilities for handling binary files and patch generation in SWE-bench evaluation.
+"""
+
+
+def remove_binary_diffs(patch_text):
+    """
+    Remove binary file diffs from a git patch.
+
+    Args:
+        patch_text (str): The git patch text
+
+    Returns:
+        str: The cleaned patch text with binary diffs removed
+    """
+    lines = patch_text.splitlines()
+    cleaned_lines = []
+    block = []
+    is_binary_block = False
+
+    for line in lines:
+        if line.startswith('diff --git '):
+            if block and not is_binary_block:
+                cleaned_lines.extend(block)
+            block = [line]
+            is_binary_block = False
+        elif 'Binary files' in line:
+            is_binary_block = True
+            block.append(line)
+        else:
+            block.append(line)
+
+    if block and not is_binary_block:
+        cleaned_lines.extend(block)
+    return '\n'.join(cleaned_lines)
+
+
+def remove_binary_files_from_git():
+    """
+    Generate a bash command to remove binary files from git staging.
+
+    Returns:
+        str: A bash command that removes binary files from git staging
+    """
+    return """
+    for file in $(git status --porcelain | grep -E "^(M| M|\\?\\?|A| A)" | cut -c4-); do
+        if [ -f "$file" ] && (file "$file" | grep -q "executable" || git check-attr binary "$file" | grep -q "binary: set"); then
+            git rm -f "$file" 2>/dev/null || rm -f "$file"
+            echo "Removed: $file"
+        fi
+    done
+    """.strip()

evaluation/benchmarks/nocode_bench/consistants.py

@@ -0,0 +1,545 @@
+DOCPATH_PATTERNS = [
+    r'docs/',
+    r'^CHANGES\.rst$',
+    r'doc/',
+    r'ChangeLog',
+    r'^changelog/',
+    r'^CHANGES$',
+]
+
+MATPLOTLIB_CONFIG = {
+    k: {
+        'python': '3.11',
+        'conda_env': 'matplotlib_35',
+        'install': 'python -m pip install -e .',
+        'test_cmd': 'pytest -rA --color=no',
+    }
+    for k in ['3.5', '3.6', '3.7', '3.8', '3.9']
+}
+MATPLOTLIB_CONFIG.update(
+    {
+        k: {
+            'python': '3.8',
+            'conda_env': 'matplotlib_31',
+            'install': 'python -m pip install -e .',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['3.1', '3.2', '3.3', '3.4']
+    }
+)
+MATPLOTLIB_CONFIG.update(
+    {
+        k: {
+            'python': '3.5',
+            'install': 'python setup.py build; python setup.py install',
+            'conda_env': 'matplotlib_11',
+            'nonroot': True,
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['2.0', '2.1', '2.2', '1.0', '1.1', '1.2', '1.3', '1.4', '1.5']
+    }
+)
+for k in ['3.8', '3.9']:
+    MATPLOTLIB_CONFIG[k]['install'] = (
+        'python -m pip install --no-build-isolation -e ".[dev]"'
+    )
+
+
+SYMPY_CONFIG = {}
+SYMPY_CONFIG.update(
+    {
+        '1.0': {
+            'conda_env': 'sympy_10',
+            'install': 'pip install -e .',
+            'test_cmd': 'bin/test -C -v',
+            # testfile -k testname
+        }
+    }
+)
+
+REQUESTS_CONFIG = {}
+REQUESTS_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'requests_227',
+            'install': 'pip install -r requirements-dev.txt',
+            'test_cmd': 'pytest -rA',
+        }
+        for k in ['2.27']
+    }
+)
+REQUESTS_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'requests_226',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest -rA',
+        }
+        for k in ['2.26']
+    }
+)
+
+PYTEST_CONFIG = {}
+PYTEST_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pytest_33',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest -v --color=no',
+        }
+        for k in ['4.4', '4.1', '3.7', '3.4', '3.3']
+    }
+)
+
+PYLINT_CONFIG = {}
+PYLINT_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pylint_210',
+            'install': 'pip install -r requirements_test.txt',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in [
+            '2.10',
+            '2.11',
+            '2.13',
+            '2.14',
+            '2.15',
+            '2.16',
+            '2.17',
+            '3.0',
+            '3.1',
+            '3.2',
+            '3.3',
+        ]
+    }
+)
+PYLINT_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pylint_210',
+            'pre_install': [
+                r"sed -i 's/setuptools==[0-9.]\+/setuptools==58.0.0/' requirements_test_min.txt"
+            ],
+            'install': 'pip install -r requirements_test.txt',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['3.0', '3.1', '3.2', '3.3']
+    }
+)
+
+ASTROPY_CONFIG = {}
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_11',
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.1', '1.2', '1.3', '2.0']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_30',
+            'pre_install': """echo '[pytest]
+filterwarnings =
+    ignore::DeprecationWarning' > pytest.ini""",
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['3.0', '3.1', '3.2']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_40',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml"""
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.0']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_41',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml""",
+                """sed -i 's/^qt_no_exception_capture = 1$/; qt_no_exception_capture = 1/' setup.cfg""",
+                r"""sed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.tomlsed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.toml""",
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.1']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_42',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml""",
+                r"""sed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.tomlsed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.toml""",
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.2', '4.3', '5.0', '5.1']
+    }
+)
+
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_52',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml"""
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['5.2', '5.3', '6.0', '6.1', '7.0']
+    }
+)
+
+DJANGO_CONFIG = {}
+DJANGO_CONFIG.update(
+    {
+        k: {
+            'install': 'pip install -e .',
+            'conda_env': 'django_22',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        }
+        for k in ['1.9', '2.2']
+    }
+)
+DJANGO_CONFIG.update(
+    {
+        '3.2': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_32',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+        '4.2': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_42',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+        '5.1': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_51',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+    }
+)
+SPHINX_CONFIG = {}
+SPHINX_CONFIG.update(
+    {  # 1.x 版本问题，实际无用
+        k: {
+            'conda_env': 'sphinx_20',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': ["sed -i 's/pytest/pytest -rA/' tox.ini"],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_20',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['2.0', '2.1', '2.2', '2.3', '2.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py",
+                "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['3.0', '3.1', '3.2', '3.3', '3.4', '3.5', '4.0']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                (
+                    "grep -q 'sphinxcontrib-htmlhelp>=2.0.0' setup.py && "
+                    "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py || "
+                    "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py"
+                ),
+                (
+                    "grep -q 'sphinxcontrib-serializinghtml>=1.1.5' setup.py && "
+                    "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py || "
+                    "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py"
+                ),
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.1']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py",
+                "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.2', '4.3', '4.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.5', '5.0', '5.1', '5.2']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_60',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy39 -v --',
+        }
+        for k in ['6.0', '6.2', '7.0', '7.1']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_72',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                'apt-get update && apt-get install -y graphviz',
+            ],
+            'test_cmd': 'tox --current-env -epy39 -v --',
+        }
+        for k in ['7.2', '7.3', '7.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_80',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy310 -v --',
+        }
+        for k in ['8.0', '8.1']
+    }
+)
+
+
+SKLEARN_CONFIG = {}
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_020',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.20', '0.21', '0.22']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_100',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.23', '0.24', '1.00', '1.01', '1.02']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_104',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.03', '1.04', '1.05']
+    }
+)
+
+SEABORN_CONFIG = {}
+SEABORN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'seaborn_010',
+            'install': 'pip install -e .[dev]',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.3', '0.4', '0.5', '0.6', '0.11', '0.12', '0.13', '0.14']
+    }
+)
+
+XARRAY_CONFIG = {}
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_0014',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0014', '0015', '0016']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_0017',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0017', '0018', '0019', '0020', '0021']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2203',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['2203', '2206', '2209', '2210', '2211', '2212']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2303',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in [
+            '2303',
+            '2304',
+            '2305',
+            '2306',
+            '2308',
+            '2309',
+            '2310',
+            '2311',
+            '2312',
+        ]
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2401',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['2401', '2402', '2403', '2405', '2407', '2409', '2410', '2411']
+    }
+)
+
+SKLEARN_CONFIG = {}
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_020',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.20', '0.21', '0.22']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_100',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.23', '0.24', '1.00', '1.01', '1.02']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_104',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.03', '1.04', '1.05', '1.06', '1.07']
+    }
+)
+
+
+MAP_REPO_TO_CONFIG = {
+    'pydata/xarray': XARRAY_CONFIG,
+    'mwaskom/seaborn': SEABORN_CONFIG,
+    'scikit-learn/scikit-learn': SKLEARN_CONFIG,
+    'sphinx-doc/sphinx': SPHINX_CONFIG,
+    'django/django': DJANGO_CONFIG,
+    'astropy/astropy': ASTROPY_CONFIG,
+    'pylint-dev/pylint': PYLINT_CONFIG,
+    'pytest-dev/pytest': PYTEST_CONFIG,
+    'psf/requests': REQUESTS_CONFIG,
+    'sympy/sympy': SYMPY_CONFIG,
+    'matplotlib/matplotlib': MATPLOTLIB_CONFIG,
+}

evaluation/benchmarks/nocode_bench/prompts/nc.j2

@@ -0,0 +1,65 @@
+<uploaded_files>
+/workspace/{{ workspace_dir_name }}
+</uploaded_files>
+
+I've uploaded a python code repository in the directory {{ workspace_dir_name }}. Consider the following issue description:
+
+<doc_change>
+{{ instance.problem_statement }}
+</doc_change>
+
+Can you help me add the new features to the repository based on the changes in the <doc_change>?
+I've already taken care of all changes to any of the test files described in the <doc_change>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages.
+Your task is to make the minimal changes to non-test files in the /workspace/{{ workspace_dir_name }} directory to implement the new features required by the documentation updates.
+
+Follow these phases to resolve the issue:
+
+Phase 1. READING: read the requirements and reword it in clearer terms
+   1.1 If there are code or config snippets. Express in words any best practices or conventions in them.
+   1.2 Hightlight method names, variables, file names, stack traces, and technical details, particularly those related to new features.
+   1.3 Explain the new feature requirements in clear terms.
+   1.4 Specify functional scope and expected behavior of new features.
+   1.5 Hightlight any best practices to take into account when developing and testing the new feature.
+
+Phase 2. RUNNING: install and run the functionality in the repository to validate the new features
+   2.1 Follow the readme.
+   2.2 Install the environment and anything needed.
+   2.2 Iterate and figure out how to validate the newly added features.
+
+Phase 3. EXPLORATION: find the files related to the new features and possible implementation solutions
+   3.1 Use `grep` to search for relevant methods, classes, keywords and feature requirements.
+   3.2 Identify all files related to the new features.
+   3.3 Propose the methods and files to implement the new features and explain why.
+   3.4 From the possible file locations, select the most likely location to implement the new features.
+
+Phase 4. TEST CREATION: before implementing any new features, create a script to validate the feature's correctness.
+   4.1 Look at existing test files in the repository to understand the test format/structure.
+   4.2 Create a minimal validation script to verify the newly added features.
+   4.3 Run the validation script to confirm the new features are successfully added and working as expected.
+   4.4 Adjust the validation script as necessary to ensure the new features fully meet the requirements.
+
+Phase 5. FEATURE ANALYSIS: state clearly the new feature and how to implement it
+   5.1 State clearly what the new feature is.
+   5.2 State clearly where the feature should be implemented.
+   5.3 State clearly how the test validates the new feature.
+   5.4 State clearly the best practices to take into account when implementing the new feature.
+   5.5 State clearly how to implement the new feature.
+
+Phase 6. FEATURE IMPLEMENTATION: edit the source code to implement your chosen solution for the new feature
+   6.1 Make minimal, focused changes to implement the new feature.
+
+Phase 7. VERIFICATION: Test your new feature thoroughly.
+   7.1 Run your validation script to verify the new feature works as expected.
+   7.2 Add edge cases to your test script to ensure comprehensive coverage of the new feature.
+   7.3 Run existing tests related to the modified code to ensure you haven't broken anything.
+
+Phase 8. FINAL REVIEW: Carefully re-read the feature requirements and compare your changes with the base commit {{ instance.base_commit }}
+   8.1 Ensure you've fully implemented all required features.
+   8.2 Run any tests in the repository related to:
+     8.2.1 The new features you are adding
+     8.2.2 The files you modified
+     8.2.3 The functions you changed
+   8.3 If any tests fail, revise your implementation until all tests pass and the new feature works as expected.
+
+Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity.

evaluation/benchmarks/nocode_bench/resource/mapping.py

@@ -0,0 +1,39 @@
+"""Mapping instance_id to resource_factor.
+
+Different instances may have different resource requirements.
+e.g., some instances may require more memory/CPU to run inference.
+This file tracks the resource requirements of different instances.
+"""
+
+import json
+import os
+
+from openhands.core.logger import openhands_logger as logger
+
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_RUNTIME_RESOURCE_FACTOR = int(
+    os.environ.get('DEFAULT_RUNTIME_RESOURCE_FACTOR', 1)
+)
+
+# dataset to resource mapping
+_global_resource_mapping: dict[str, dict[str, float]] = {}
+
+
+def get_resource_mapping(dataset_name: str) -> dict[str, float]:
+    if dataset_name not in _global_resource_mapping:
+        file_path = os.path.join(CUR_DIR, f'{dataset_name}.json')
+        if not os.path.exists(file_path):
+            logger.info(f'Resource mapping for {dataset_name} not found.')
+            return None
+
+        with open(file_path, 'r') as f:
+            _global_resource_mapping[dataset_name] = json.load(f)
+        logger.debug(f'Loaded resource mapping for {dataset_name}')
+    return _global_resource_mapping[dataset_name]
+
+
+def get_instance_resource_factor(dataset_name: str, instance_id: str) -> int:
+    resource_mapping = get_resource_mapping(dataset_name)
+    if resource_mapping is None:
+        return DEFAULT_RUNTIME_RESOURCE_FACTOR
+    return int(resource_mapping.get(instance_id, DEFAULT_RUNTIME_RESOURCE_FACTOR))

evaluation/benchmarks/nocode_bench/run_infer_nc.py

@@ -0,0 +1,905 @@
+import asyncio
+import copy
+import json
+import os
+import tempfile
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+import toml
+from datasets import load_dataset
+from jinja2 import Environment, FileSystemLoader
+
+import openhands.agenthub
+from evaluation.benchmarks.nocode_bench.binary_patch_utils import (
+    remove_binary_diffs,
+    remove_binary_files_from_git,
+)
+from evaluation.benchmarks.nocode_bench.consistants import MAP_REPO_TO_CONFIG
+from evaluation.benchmarks.nocode_bench.resource.mapping import (
+    get_instance_resource_factor,
+)
+from evaluation.benchmarks.nocode_bench.scripts.utils.evaluation_utils import (
+    run_evaluation_nocode_bench,
+)
+from evaluation.utils.shared import (
+    EvalException,
+    EvalMetadata,
+    EvalOutput,
+    assert_and_raise,
+    codeact_user_response,
+    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
+    is_fatal_evaluation_error,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    OpenHandsConfig,
+    get_evaluation_parser,
+    get_llm_config_arg,
+)
+from openhands.core.config.condenser_config import NoOpCondenserConfig
+from openhands.core.config.utils import get_condenser_config_arg
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.critic import AgentFinishedCritic
+from openhands.events.action import CmdRunAction, FileReadAction, MessageAction
+from openhands.events.observation import (
+    CmdOutputObservation,
+    ErrorObservation,
+    FileReadObservation,
+)
+from openhands.events.serialization.event import event_from_dict, event_to_dict
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+from openhands.utils.shutdown_listener import sleep_if_should_continue
+
+USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
+ENABLE_LLM_EDITOR = os.environ.get('ENABLE_LLM_EDITOR', 'false').lower() == 'true'
+BenchMode = Literal['swe', 'swt', 'swt-ci']
+
+# Global variable to track dataset type
+DATASET_TYPE = 'nc_bench'
+
+
+def set_dataset_type(dataset_name: str) -> str:
+    """Set dataset type based on dataset name."""
+    global DATASET_TYPE
+    DATASET_TYPE = 'nc_bench'
+
+    logger.info(f'Dataset type set to: {DATASET_TYPE}')
+
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
+
+
+def _get_swebench_workspace_dir_name(instance: pd.Series) -> str:
+    return f'{instance.repo.split("/")[-1]}'
+
+
+def get_instruction(instance: pd.Series, metadata: EvalMetadata) -> MessageAction:
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    metadata.details['mode']
+
+    # Determine the template file based on mode and LLM
+
+    template_name = 'nc.j2'
+
+    # Set up Jinja2 environment
+    # Assuming templates are in 'evaluation/benchmarks/swe_bench/prompts' relative to this script
+    prompts_dir = os.path.join(os.path.dirname(__file__), 'prompts')
+    env = Environment(loader=FileSystemLoader(prompts_dir))
+    template = env.get_template(template_name)
+
+    # Prepare context for rendering
+    context = {
+        'instance': instance,
+        'workspace_dir_name': workspace_dir_name,
+        'metadata': metadata,  # Pass metadata if needed in templates
+    }
+
+    context['test_instructions'] = ''  # Ensure it's defined for other modes
+
+    # Render the instruction
+    instruction = template.render(context)
+
+    if RUN_WITH_BROWSING:
+        instruction += (
+            '<IMPORTANT!>\nYou SHOULD NEVER attempt to browse the web. </IMPORTANT!>\n'
+        )
+
+    if 'image_assets' in instance:
+        assets = json.loads(instance['image_assets'])
+        assert 'problem_statement' in assets, (
+            'problem_statement is required in image_assets'
+        )
+        image_urls = assets['problem_statement']
+        return MessageAction(content=instruction, image_urls=image_urls)
+    return MessageAction(content=instruction)
+
+
+DEFAULT_DOCKER_IMAGE_PREFIX = os.environ.get(
+    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/'
+)
+logger.info(f'Default docker image prefix: {DEFAULT_DOCKER_IMAGE_PREFIX}')
+
+
+def get_instance_docker_image(
+    instance_id: str,
+    swebench_official_image: bool = False,
+) -> str:
+    if swebench_official_image:
+        # Official NoCode-Bench image
+        image_name = f'ncbench_{instance_id}:latest'.lower()
+        logger.debug(f'Using official NoCode-Bench image: {image_name}')
+        return image_name
+    else:
+        raise
+
+
+def get_config(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+) -> OpenHandsConfig:
+    # We use a different instance image for the each instance of NoCode-bench eval
+    use_swebench_official_image = True
+
+    base_container_image = get_instance_docker_image(
+        instance['instance_id'],
+        swebench_official_image=use_swebench_official_image,
+    )
+    logger.info(
+        f'Using instance container image: {base_container_image}. '
+        f'Please make sure this image exists. '
+        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+    )
+
+    sandbox_config = get_default_sandbox_config_for_eval()
+    sandbox_config.base_container_image = base_container_image
+    sandbox_config.enable_auto_lint = True
+    sandbox_config.use_host_network = False
+    # Add platform to the sandbox config to solve issue 4401
+    sandbox_config.platform = 'linux/amd64'
+    sandbox_config.remote_runtime_resource_factor = get_instance_resource_factor(
+        dataset_name=metadata.dataset,
+        instance_id=instance['instance_id'],
+    )
+
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        runtime=os.environ.get('RUNTIME', 'docker'),
+        sandbox_config=sandbox_config,
+    )
+
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
+        )
+    )
+    # get 'draft_editor' config if exists
+    config.set_llm_config(get_llm_config_arg('draft_editor'), 'draft_editor')
+
+    agent_config = AgentConfig(
+        enable_jupyter=False,
+        enable_browsing=RUN_WITH_BROWSING,
+        enable_llm_editor=ENABLE_LLM_EDITOR,
+        enable_mcp=False,
+        condenser=metadata.condenser_config,
+        enable_prompt_extensions=False,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def make_serializable(obj):
+    if isinstance(obj, pd.Series):
+        obj = obj.to_dict()
+    if isinstance(obj, dict):
+        return {k: make_serializable(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [make_serializable(v) for v in obj]
+    elif isinstance(obj, tuple):
+        return tuple(make_serializable(v) for v in obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, pd.Timestamp):
+        return str(obj)
+    else:
+        return obj
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+    metadata: EvalMetadata,
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Initialization Fn')
+    logger.info('-' * 30)
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    obs: CmdOutputObservation
+
+    # Set instance id and git configuration
+    action = CmdRunAction(
+        command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc && git config --global core.pager "" && git config --global diff.binary false"""
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to export SWE_INSTANCE_ID and configure git: {str(obs)}',
+    )
+
+    action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """)
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to export USER: {str(obs)}')
+
+    # inject the init script
+    script_dir = os.path.dirname(__file__)
+
+    # inject the instance info
+    action = CmdRunAction(command='mkdir -p /swe_util/eval_data/instances')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to create /swe_util/eval_data/instances: {str(obs)}',
+    )
+
+    swe_instance_json_name = 'swe-bench-instance.json'
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Construct the full path for the desired file name within the temporary directory
+        temp_file_path = os.path.join(temp_dir, swe_instance_json_name)
+        # Write to the file with the desired name within the temporary directory
+
+        with open(temp_file_path, 'w') as f:
+            if not isinstance(instance, dict):
+                instance_dict = make_serializable(instance)
+            else:
+                instance_dict = dict(instance)
+
+            if DATASET_TYPE == 'nc_bench':
+                config = MAP_REPO_TO_CONFIG.get(instance['repo'], {}).get(
+                    instance['version'], []
+                )
+                docker_conda_env_name = config['conda_env']
+                instance_dict['conda_env'] = docker_conda_env_name
+
+            json.dump([instance_dict], f)
+
+        # Copy the file to the desired location
+        runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/')
+
+        # inject the instance swe entry
+        entry_script_path = 'instance_nc_entry.sh'
+
+        runtime.copy_to(
+            str(os.path.join(script_dir, f'scripts/setup/{entry_script_path}')),
+            '/swe_util/',
+        )
+
+    action = CmdRunAction(command='cat ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to cat ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command='source ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    if isinstance(obs, ErrorObservation):
+        logger.error(f'Failed to source ~/.bashrc: {str(obs)}')
+    assert_and_raise(obs.exit_code == 0, f'Failed to source ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command=f'source /swe_util/{entry_script_path}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to source /swe_util/{entry_script_path}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git reset --hard')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to git reset --hard: {str(obs)}')
+
+    action = CmdRunAction(
+        command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done'
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to remove git remotes: {str(obs)}')
+
+    if DATASET_TYPE != 'Multimodal' and DATASET_TYPE != 'SWE-bench-Live':
+        # Only for non-multimodal datasets, we need to activate the testbed environment for Python
+        # SWE-Bench multimodal datasets and SWE-bench-Live are not using the testbed environment
+        action = CmdRunAction(command='which python')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        assert_and_raise(
+            obs.exit_code == 0,
+            f'Expected to find python interpreter, but got: {str(obs)}',
+        )
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Initialization Fn')
+    logger.info('-' * 30)
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Completion Fn')
+    logger.info('-' * 30)
+    obs: CmdOutputObservation
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    if obs.exit_code == -1:
+        # The previous command is still running
+        # We need to kill previous command
+        logger.info('The previous command is still running, trying to kill it...')
+        action = CmdRunAction(command='C-c')
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+        # Then run the command again
+        action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    if obs.exit_code == -1:
+        # The previous command is still running
+        # We need to kill previous command
+        logger.info('The previous command is still running, trying to ctrl+z it...')
+        action = CmdRunAction(command='C-z')
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+        # Then run the command again
+        action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git config --global core.pager ""')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git config --global core.pager "": {str(obs)}',
+    )
+
+    # First check for any git repositories in subdirectories
+    action = CmdRunAction(command='find . -type d -name .git -not -path "./.git"')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to find git repositories: {str(obs)}',
+    )
+
+    git_dirs = [p for p in obs.content.strip().split('\n') if p]
+    if git_dirs:
+        # Remove all .git directories in subdirectories
+        for git_dir in git_dirs:
+            action = CmdRunAction(command=f'rm -rf "{git_dir}"')
+            action.set_hard_timeout(600)
+            logger.info(action, extra={'msg_type': 'ACTION'})
+            obs = runtime.run_action(action)
+            logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+            assert_and_raise(
+                isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+                f'Failed to remove git directory {git_dir}: {str(obs)}',
+            )
+
+    # add all files
+    action = CmdRunAction(command='git add -A')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git add -A: {str(obs)}',
+    )
+
+    # Remove binary files from git staging
+    action = CmdRunAction(command=remove_binary_files_from_git())
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to remove binary files: {str(obs)}',
+    )
+
+    n_retries = 0
+    git_patch = None
+    while n_retries < 5:
+        action = CmdRunAction(
+            command=f'git diff --no-color --cached {instance["base_commit"]} > patch.diff'
+        )
+        action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        n_retries += 1
+        if isinstance(obs, CmdOutputObservation):
+            if obs.exit_code == 0:
+                # Read the patch file
+                action = FileReadAction(path='patch.diff')
+                action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+                logger.info(action, extra={'msg_type': 'ACTION'})
+                obs = runtime.run_action(action)
+                logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+                if isinstance(obs, FileReadObservation):
+                    git_patch = obs.content
+                    break
+                elif isinstance(obs, ErrorObservation):
+                    # Fall back to cat "patch.diff" to get the patch
+                    assert 'File could not be decoded as utf-8' in obs.content
+                    action = CmdRunAction(command='cat patch.diff')
+                    action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+                    logger.info(action, extra={'msg_type': 'ACTION'})
+                    obs = runtime.run_action(action)
+                    assert isinstance(obs, CmdOutputObservation) and obs.exit_code == 0
+                    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+                    git_patch = obs.content
+                    break
+                else:
+                    assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+            else:
+                logger.info('Failed to get git diff, retrying...')
+                sleep_if_should_continue(10)
+        elif isinstance(obs, ErrorObservation):
+            logger.error(f'Error occurred: {obs.content}. Retrying...')
+            sleep_if_should_continue(10)
+        else:
+            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+
+    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
+
+    # Remove binary diffs from the patch
+    git_patch = remove_binary_diffs(git_patch)
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Completion Fn')
+    logger.info('-' * 30)
+    return {'git_patch': git_patch}
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+    runtime_failure_count: int = 0,
+) -> EvalOutput:
+    config = get_config(instance, metadata)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    # Increase resource_factor with increasing attempt_id
+    if runtime_failure_count > 0:
+        config.sandbox.remote_runtime_resource_factor = min(
+            config.sandbox.remote_runtime_resource_factor * (2**runtime_failure_count),
+            8,
+        )
+        logger.warning(
+            f'This is the {runtime_failure_count + 1}th attempt for instance {instance.instance_id}, setting resource factor to {config.sandbox.remote_runtime_resource_factor}'
+        )
+
+    metadata = copy.deepcopy(metadata)
+    metadata.details['runtime_failure_count'] = runtime_failure_count
+    metadata.details['remote_runtime_resource_factor'] = (
+        config.sandbox.remote_runtime_resource_factor
+    )
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    try:
+        initialize_runtime(runtime, instance, metadata)
+
+        message_action = get_instruction(instance, metadata)
+
+        # Here's how you can run the agent (similar to the `main` function) and get the final task state
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=message_action,
+                runtime=runtime,
+                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                    metadata.agent_class
+                ],
+            )
+        )
+
+        # if fatal error, throw EvalError to trigger re-run
+        if is_fatal_evaluation_error(state.last_error):
+            raise EvalException('Fatal error detected: ' + state.last_error)
+
+        # ======= THIS IS SWE-Bench specific =======
+        # Get git patch
+        if DATASET_TYPE == 'SWE-bench-Live':
+            from evaluation.benchmarks.swe_bench.live_utils import (
+                complete_runtime as complete_runtime_fn,
+            )
+        else:
+            complete_runtime_fn = complete_runtime
+        return_val = complete_runtime_fn(runtime, instance)
+        git_patch = return_val['git_patch']
+        logger.info(
+            f'Got git diff for instance {instance.instance_id}:\n--------\n{git_patch}\n--------'
+        )
+    finally:
+        runtime.close()
+    # ==========================================
+
+    # ======= Attempt to evaluate the agent's edits =======
+    # we use eval_infer.sh to evaluate the agent's edits, not here
+    # because the agent may alter the environment / testcases
+    test_result = {
+        'git_patch': git_patch,
+    }
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
+    histories = [event_to_dict(event) for event in state.history]
+    metrics = get_metrics(state)
+
+    # Save the output
+    instruction = message_action.content
+    if message_action.image_urls:
+        instruction += (
+            '\n\n<image_urls>' + '\n'.join(message_action.image_urls) + '</image_urls>'
+        )
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        instance=instance.to_dict(),  # SWE Bench specific
+        test_result=test_result,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+    )
+    return output
+
+
+def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
+    file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.toml')
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as file:
+            data = toml.load(file)
+            if 'selected_ids' in data:
+                selected_ids = data['selected_ids']
+                logger.info(
+                    f'Filtering {len(selected_ids)} tasks from "selected_ids"...'
+                )
+                subset = dataset[dataset[filter_column].isin(selected_ids)]
+                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
+                return subset
+            if 'selected_repos' in data:
+                # repos for the swe-bench instances:
+                # ['astropy/astropy', 'django/django', 'matplotlib/matplotlib', 'mwaskom/seaborn', 'pallets/flask', 'psf/requests', 'pydata/xarray', 'pylint-dev/pylint', 'pytest-dev/pytest', 'scikit-learn/scikit-learn', 'sphinx-doc/sphinx', 'sympy/sympy']
+                selected_repos = data['selected_repos']
+                if isinstance(selected_repos, str):
+                    selected_repos = [selected_repos]
+                assert isinstance(selected_repos, list)
+                logger.info(
+                    f'Filtering {selected_repos} tasks from "selected_repos"...'
+                )
+                subset = dataset[dataset['repo'].isin(selected_repos)]
+                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
+                return subset
+
+    skip_ids = os.environ.get('SKIP_IDS', '').split(',')
+    if len(skip_ids) > 0:
+        logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
+        return dataset[~dataset[filter_column].isin(skip_ids)]
+    return dataset
+
+
+if __name__ == '__main__':
+    parser = get_evaluation_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='NoCode-bench/NoCode-bench_Verified',
+        help='data set to evaluate on, either full-test or lite-test',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+    parser.add_argument(
+        '--mode',
+        type=str,
+        default='swe',
+        choices=['swe', 'swt', 'swt-ci'],
+        help="mode to run the evaluation, either 'swe', 'swt', or 'swt-ci'",
+    )
+
+    args, _ = parser.parse_known_args()
+
+    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
+    # so we don't need to manage file uploading to OpenHands's repo
+
+    dataset = load_dataset(args.dataset, args.split)
+
+    # Set the global dataset type based on dataset name
+    set_dataset_type(args.dataset)
+
+    swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
+    logger.info(
+        f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
+    )
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        llm_config.log_completions = True
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    # Get condenser config from environment variable
+    condenser_name = os.environ.get('EVAL_CONDENSER')
+    if condenser_name:
+        condenser_config = get_condenser_config_arg(condenser_name)
+        if condenser_config is None:
+            raise ValueError(
+                f'Could not find Condenser config: EVAL_CONDENSER={condenser_name}'
+            )
+    else:
+        # If no specific condenser config is provided via env var, default to NoOpCondenser
+        condenser_config = NoOpCondenserConfig()
+        logger.debug(
+            'No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.'
+        )
+
+    details = {'mode': args.mode}
+    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
+
+    dataset_descrption = (
+        args.dataset.replace('/', '__') + '-' + args.split.replace('/', '__')
+    )
+    metadata = make_metadata(
+        llm_config,
+        dataset_descrption,
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        details=details,
+        condenser_config=condenser_config,
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    print(f'### OUTPUT FILE: {output_file} ###')
+
+    # Run evaluation in iterative mode:
+    # If a rollout fails to output AgentFinishAction, we will try again until it succeeds OR total 3 attempts have been made.
+    ITERATIVE_EVAL_MODE = (
+        os.environ.get('ITERATIVE_EVAL_MODE', 'false').lower() == 'true'
+    )
+    ITERATIVE_EVAL_MODE_MAX_ATTEMPTS = int(
+        os.environ.get('ITERATIVE_EVAL_MODE_MAX_ATTEMPTS', '3')
+    )
+
+    if not ITERATIVE_EVAL_MODE:
+        # load the dataset
+        instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
+        if len(instances) > 0 and not isinstance(
+            instances['PASS2PASS'][instances['PASS2PASS'].index[0]], str
+        ):
+            for col in ['PASS2PASS', 'FAIL2PASS']:
+                instances[col] = instances[col].apply(lambda x: str(x))
+
+        run_evaluation_nocode_bench(
+            instances,
+            metadata,
+            output_file,
+            args.eval_num_workers,
+            process_instance,
+            timeout_seconds=8
+            * 60
+            * 60,  # 8 hour PER instance should be more than enough
+            max_retries=5,
+        )
+    else:
+        critic = AgentFinishedCritic()
+
+        def get_cur_output_file_path(attempt: int) -> str:
+            return (
+                f'{output_file.removesuffix(".jsonl")}.critic_attempt_{attempt}.jsonl'
+            )
+
+        eval_ids = None
+        for attempt in range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1):
+            cur_output_file = get_cur_output_file_path(attempt)
+            logger.info(
+                f'Running evaluation with critic {critic.__class__.__name__} for attempt {attempt} of {ITERATIVE_EVAL_MODE_MAX_ATTEMPTS}.'
+            )
+
+            # For deterministic eval, we set temperature to 0.1 for (>1) attempt
+            # so hopefully we get slightly different results
+            if attempt > 1 and metadata.llm_config.temperature == 0:
+                logger.info(
+                    f'Detected temperature is 0 for (>1) attempt {attempt}. Setting temperature to 0.1...'
+                )
+                metadata.llm_config.temperature = 0.1
+
+            # Load instances - at first attempt, we evaluate all instances
+            # On subsequent attempts, we only evaluate the instances that failed the previous attempt determined by critic
+            instances = prepare_dataset(
+                swe_bench_tests, cur_output_file, args.eval_n_limit, eval_ids=eval_ids
+            )
+            if len(instances) > 0 and not isinstance(
+                instances['PASS2PASS'][instances['PASS2PASS'].index[0]], str
+            ):
+                for col in ['PASS2PASS', 'FAIL2PASS']:
+                    instances[col] = instances[col].apply(lambda x: str(x))
+
+            # Run evaluation - but save them to cur_output_file
+            logger.info(
+                f'Evaluating {len(instances)} instances for attempt {attempt}...'
+            )
+            run_evaluation_nocode_bench(
+                instances,
+                metadata,
+                cur_output_file,
+                args.eval_num_workers,
+                process_instance,
+                timeout_seconds=8
+                * 60
+                * 60,  # 8 hour PER instance should be more than enough
+                max_retries=5,
+            )
+
+            # When eval is done, we update eval_ids to the instances that failed the current attempt
+            instances_failed = []
+            logger.info(
+                f'Use critic {critic.__class__.__name__} to check {len(instances)} instances for attempt {attempt}...'
+            )
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    try:
+                        history = [
+                            event_from_dict(event) for event in instance['history']
+                        ]
+                        critic_result = critic.evaluate(
+                            history, instance['test_result'].get('git_patch', '')
+                        )
+                        if not critic_result.success:
+                            instances_failed.append(instance['instance_id'])
+                    except Exception as e:
+                        logger.error(
+                            f'Error loading history for instance {instance["instance_id"]}: {e}'
+                        )
+                        instances_failed.append(instance['instance_id'])
+            logger.info(
+                f'{len(instances_failed)} instances failed the current attempt {attempt}: {instances_failed}'
+            )
+            eval_ids = instances_failed
+
+            # If no instances failed, we break
+            if len(instances_failed) == 0:
+                break
+
+        # Then we should aggregate the results from all attempts into the original output file
+        # and remove the intermediate files
+        logger.info(
+            'Aggregating results from all attempts into the original output file...'
+        )
+        fout = open(output_file, 'w')
+        added_instance_ids = set()
+        for attempt in reversed(range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1)):
+            cur_output_file = get_cur_output_file_path(attempt)
+            if not os.path.exists(cur_output_file):
+                logger.warning(
+                    f'Intermediate output file {cur_output_file} does not exist. Skipping...'
+                )
+                continue
+
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    # Also make sure git_patch is not empty - otherwise we fall back to previous attempt (empty patch is worse than anything else)
+                    if (
+                        instance['instance_id'] not in added_instance_ids
+                        and instance['test_result'].get('git_patch', '').strip()
+                    ):
+                        fout.write(line)
+                        added_instance_ids.add(instance['instance_id'])
+            logger.info(
+                f'Aggregated instances from {cur_output_file}. Total instances added so far: {len(added_instance_ids)}'
+            )
+        fout.close()
+        logger.info(
+            f'Done! Total {len(added_instance_ids)} instances added to {output_file}'
+        )

evaluation/benchmarks/nocode_bench/scripts/eval/convert.py

@@ -0,0 +1,33 @@
+import argparse
+import json
+
+
+def main(output_jsonl: str):
+    with open(output_jsonl, 'r') as f:
+        for line in f:
+            try:
+                output = json.loads(line)
+                pred = {
+                    'instance_id': output['instance_id'],
+                    'model_name_or_path': output['metadata']['llm_config']['model'],
+                    'model_patch': output['test_result']['git_patch'],
+                }
+            except Exception as e:
+                print(
+                    f'Error while reading output of instance {output["instance_id"]}: {e}'
+                )
+
+            print(json.dumps(pred))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--output_jsonl',
+        type=str,
+        required=True,
+        help='Path to the prediction file (.../outputs.jsonl)',
+    )
+    args = parser.parse_args()
+
+    main(args.output_jsonl)

evaluation/benchmarks/nocode_bench/scripts/eval/verify_costs.py

@@ -0,0 +1,104 @@
+import argparse
+
+import pandas as pd
+
+from openhands.core.logger import openhands_logger as logger
+
+
+def verify_instance_costs(row: pd.Series) -> float:
+    """
+    Verifies that the accumulated_cost matches the sum of individual costs in metrics.
+    Also checks for duplicate consecutive costs which might indicate buggy counting.
+    If the consecutive costs are identical, the file is affected by this bug:
+    https://github.com/All-Hands-AI/OpenHands/issues/5383
+
+    Args:
+        row: DataFrame row containing instance data with metrics
+    Returns:
+        float: The verified total cost for this instance (corrected if needed)
+    """
+    try:
+        metrics = row.get('metrics')
+        if not metrics:
+            logger.warning(f'Instance {row["instance_id"]}: No metrics found')
+            return 0.0
+
+        accumulated = metrics.get('accumulated_cost')
+        costs = metrics.get('costs', [])
+
+        if accumulated is None:
+            logger.warning(
+                f'Instance {row["instance_id"]}: No accumulated_cost in metrics'
+            )
+            return 0.0
+
+        # Check for duplicate consecutive costs and systematic even-odd pairs
+        has_duplicate = False
+        all_pairs_match = True
+
+        # Check each even-odd pair (0-1, 2-3, etc.)
+        for i in range(0, len(costs) - 1, 2):
+            if abs(costs[i]['cost'] - costs[i + 1]['cost']) < 1e-6:
+                has_duplicate = True
+                logger.debug(
+                    f'Instance {row["instance_id"]}: Possible buggy double-counting detected! '
+                    f'Steps {i} and {i + 1} have identical costs: {costs[i]["cost"]:.2f}'
+                )
+            else:
+                all_pairs_match = False
+                break
+
+        # Calculate total cost, accounting for buggy double counting if detected
+        if len(costs) >= 2 and has_duplicate and all_pairs_match:
+            paired_steps_cost = sum(
+                cost_entry['cost']
+                for cost_entry in costs[: -1 if len(costs) % 2 else None]
+            )
+            real_paired_cost = paired_steps_cost / 2
+
+            unpaired_cost = costs[-1]['cost'] if len(costs) % 2 else 0
+            total_cost = real_paired_cost + unpaired_cost
+
+        else:
+            total_cost = sum(cost_entry['cost'] for cost_entry in costs)
+
+        if not abs(total_cost - accumulated) < 1e-6:
+            logger.warning(
+                f'Instance {row["instance_id"]}: Cost mismatch: '
+                f'accumulated: {accumulated:.2f}, sum of costs: {total_cost:.2f}, '
+            )
+
+        return total_cost
+
+    except Exception as e:
+        logger.error(
+            f'Error verifying costs for instance {row.get("instance_id", "UNKNOWN")}: {e}'
+        )
+        return 0.0
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Verify costs in SWE-bench output file'
+    )
+    parser.add_argument(
+        'input_filepath', type=str, help='Path to the output.jsonl file'
+    )
+    args = parser.parse_args()
+
+    try:
+        # Load and verify the JSONL file
+        df = pd.read_json(args.input_filepath, lines=True)
+        logger.info(f'Loaded {len(df)} instances from {args.input_filepath}')
+
+        # Verify costs for each instance and sum up total
+        total_cost = df.apply(verify_instance_costs, axis=1).sum()
+        logger.info(f'Total verified cost across all instances: ${total_cost:.2f}')
+
+    except Exception as e:
+        logger.error(f'Failed to process file: {e}')
+        raise
+
+
+if __name__ == '__main__':
+    main()

evaluation/benchmarks/nocode_bench/scripts/run_infer_nc.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+MAX_ITER=$5
+NUM_WORKERS=$6
+DATASET=$7
+SPLIT=$8
+N_RUNS=$9
+MODE=${10}
+
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$MAX_ITER" ]; then
+  echo "MAX_ITER not specified, use default 100"
+  MAX_ITER=100
+fi
+
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
+
+if [ -z "$DATASET" ]; then
+  echo "DATASET not specified, use default princeton-nlp/SWE-bench_Lite"
+  DATASET="princeton-nlp/SWE-bench_Lite"
+fi
+
+if [ -z "$SPLIT" ]; then
+  echo "SPLIT not specified, use default test"
+  SPLIT="test"
+fi
+
+if [ -z "$MODE" ]; then
+  MODE="swe"
+  echo "MODE not specified, use default $MODE"
+fi
+
+if [ -n "$EVAL_CONDENSER" ]; then
+  echo "Using Condenser Config: $EVAL_CONDENSER"
+else
+  echo "No Condenser Config provided via EVAL_CONDENSER, use default (NoOpCondenser)."
+fi
+
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+echo "SPLIT: $SPLIT"
+echo "MAX_ITER: $MAX_ITER"
+echo "NUM_WORKERS: $NUM_WORKERS"
+echo "COMMIT_HASH: $COMMIT_HASH"
+echo "MODE: $MODE"
+echo "EVAL_CONDENSER: $EVAL_CONDENSER"
+
+# Default to NOT use Hint
+if [ -z "$USE_HINT_TEXT" ]; then
+  export USE_HINT_TEXT=false
+fi
+echo "USE_HINT_TEXT: $USE_HINT_TEXT"
+EVAL_NOTE="$OPENHANDS_VERSION"
+# if not using Hint, add -no-hint to the eval note
+if [ "$USE_HINT_TEXT" = false ]; then
+  EVAL_NOTE="$EVAL_NOTE-no-hint"
+fi
+
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
+if [ -n "$EXP_NAME" ]; then
+  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+# if mode != swe, add mode to the eval note
+if [ "$MODE" != "swe" ]; then
+  EVAL_NOTE="${EVAL_NOTE}-${MODE}"
+fi
+# Add condenser config to eval note if provided
+if [ -n "$EVAL_CONDENSER" ]; then
+  EVAL_NOTE="${EVAL_NOTE}-${EVAL_CONDENSER}"
+fi
+
+function run_eval() {
+  local eval_note="${1}"
+  COMMAND="poetry run python evaluation/benchmarks/nocode_bench/run_infer_nc.py \
+    --agent-cls $AGENT \
+    --llm-config $MODEL_CONFIG \
+    --max-iterations $MAX_ITER \
+    --eval-num-workers $NUM_WORKERS \
+    --eval-note $eval_note \
+    --dataset $DATASET \
+    --split $SPLIT \
+    --mode $MODE"
+
+
+
+  if [ -n "$EVAL_LIMIT" ]; then
+    echo "EVAL_LIMIT: $EVAL_LIMIT"
+    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+  fi
+
+  # Run the command
+  eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+if [ -z "$N_RUNS" ]; then
+  N_RUNS=1
+  echo "N_RUNS not specified, use default $N_RUNS"
+fi
+
+# Skip runs if the run number is in the SKIP_RUNS list
+# read from env variable SKIP_RUNS as a comma separated list of run numbers
+SKIP_RUNS=(${SKIP_RUNS//,/ })
+for i in $(seq 1 $N_RUNS); do
+  if [[ " ${SKIP_RUNS[@]} " =~ " $i " ]]; then
+    echo "Skipping run $i"
+    continue
+  fi
+  current_eval_note="$EVAL_NOTE-run_$i"
+  echo "EVAL_NOTE: $current_eval_note"
+  run_eval $current_eval_note
+done
+
+checkout_original_branch

evaluation/benchmarks/nocode_bench/scripts/setup/compare_patch_filename.py

@@ -0,0 +1,54 @@
+"""This script compares gold patches with OpenHands-generated patches and check whether
+OpenHands found the right (set of) files to modify.
+"""
+
+import argparse
+import json
+import re
+
+
+def extract_modified_files(patch):
+    modified_files = set()
+    file_pattern = re.compile(r'^diff --git a/(.*?) b/')
+
+    for line in patch.split('\n'):
+        match = file_pattern.match(line)
+        if match:
+            modified_files.add(match.group(1))
+
+    return modified_files
+
+
+def process_report(oh_output_file):
+    succ = 0
+    fail = 0
+    for line in open(oh_output_file):
+        line = json.loads(line)
+        instance_id = line['instance_id']
+        gold_patch = line['swe_instance']['patch']
+        generated_patch = line['git_patch']
+        gold_modified_files = extract_modified_files(gold_patch)
+        # swe-bench lite only: a gold patch always contains exactly one file
+        assert len(gold_modified_files) == 1
+        generated_modified_files = extract_modified_files(generated_patch)
+
+        # Check if all files in gold_patch are also in generated_patch
+        all_files_in_generated = gold_modified_files.issubset(generated_modified_files)
+        if all_files_in_generated:
+            succ += 1
+        else:
+            fail += 1
+            print(
+                f'{instance_id}: file mismatch, gold = {gold_modified_files}, generated = {generated_modified_files}'
+            )
+    print(
+        f'\nSUMMARY: {succ} out of {succ + fail} instances found correct files to edit, success rate = {succ / float(succ + fail)}'
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--oh_output_file', help='Path to the OH output file')
+    args = parser.parse_args()
+
+    process_report(args.oh_output_file)

evaluation/benchmarks/nocode_bench/scripts/setup/instance_nc_entry.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+source ~/.bashrc
+SWEUTIL_DIR=/swe_util
+
+if [ -z "$SWE_INSTANCE_ID" ]; then
+    echo "Error: SWE_INSTANCE_ID is not set." >&2
+    exit 1
+fi
+
+
+item=$(jq --arg INSTANCE_ID "$SWE_INSTANCE_ID" '.[] | select(.instance_id == $INSTANCE_ID)' $SWEUTIL_DIR/eval_data/instances/swe-bench-instance.json)
+
+
+if [[ -z "$item" ]]; then
+  echo "No item found for the provided instance ID."
+  exit 1
+fi
+
+REPO_NAME=$(echo "$item" | jq -r '.repo | split("/")[-1]')
+WORKSPACE_NAME="$REPO_NAME"
+
+
+echo "WORKSPACE_NAME: $WORKSPACE_NAME"
+
+
+# Clear the workspace
+if [ -d /workspace ]; then
+    rm -rf /workspace/*
+else
+    mkdir /workspace
+fi
+# Copy repo to workspace
+if [ -d /workspace/$WORKSPACE_NAME ]; then
+    rm -rf /workspace/$WORKSPACE_NAME
+fi
+mkdir -p /workspace
+
+SRC_DIR="/root/$REPO_NAME"
+DEST_DIR="/workspace/$WORKSPACE_NAME"
+
+cp -r "$SRC_DIR" "$DEST_DIR"
+
+
+
+echo ">> Extracting conda environment name..."
+CONDA_ENV_NAME=$(echo "$item" | jq -r '.conda_env // empty')
+
+# Activate instance-specific environment
+if [ -d /opt/miniconda3 ]; then
+    . /opt/miniconda3/etc/profile.d/conda.sh
+    conda activate $CONDA_ENV_NAME
+fi