Integrate AuthSystem docs into documentation navigation

- Add AuthSystem design documents to docs.json navigation under Architecture section
- Convert .md files to .mdx format for proper documentation system integration
- Add frontmatter with appropriate titles for both documents
- Documents now accessible in deployed documentation under OpenHands Developers > Architecture

.github/CODEOWNERS

@@ -2,7 +2,7 @@
 # See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
 # Frontend code owners
-/frontend/ @rbren @amanape
+/frontend/ @amanape
 /openhands-ui/ @amanape
 
 # Evaluation code owners

.github/workflows/e2e-tests.yml

@@ -0,0 +1,228 @@
+name: End-to-End Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    branches:
+      - main
+      - develop
+  workflow_dispatch:
+
+jobs:
+  e2e-tests:
+    if: contains(github.event.pull_request.labels.*.name, 'end-to-end') || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    env:
+      GITHUB_REPO_NAME: ${{ github.repository }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install poetry via pipx
+        uses: abatilo/actions-poetry@v4
+        with:
+          poetry-version: 2.1.3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'poetry'
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 xauth xvfb libgbm1 libasound2t64 netcat-openbsd
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+          cache-dependency-path: 'frontend/package-lock.json'
+
+      - name: Setup environment for end-to-end tests
+        run: |
+          # Create test results directory
+          mkdir -p test-results
+
+          # Create downloads directory for OpenHands (use a directory in the home folder)
+          mkdir -p $HOME/downloads
+          sudo chown -R $USER:$USER $HOME/downloads
+          sudo chmod -R 755 $HOME/downloads
+
+      - name: Build OpenHands
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          INSTALL_DOCKER: 1
+          RUNTIME: docker
+          FRONTEND_PORT: 12000
+          FRONTEND_HOST: 0.0.0.0
+          BACKEND_HOST: 0.0.0.0
+          BACKEND_PORT: 3000
+          ENABLE_BROWSER: true
+          INSTALL_PLAYWRIGHT: 1
+        run: |
+          # Fix poetry.lock file if needed
+          echo "Fixing poetry.lock file if needed..."
+          poetry lock
+
+          # Build OpenHands using make build
+          echo "Running make build..."
+          make build
+
+          # Install Chromium Headless Shell for Playwright (needed for pytest-playwright)
+          echo "Installing Chromium Headless Shell for Playwright..."
+          poetry run playwright install chromium-headless-shell
+
+          # Verify Playwright browsers are installed (for e2e tests only)
+          echo "Verifying Playwright browsers installation for e2e tests..."
+          BROWSER_CHECK=$(poetry run python tests/e2e/check_playwright.py 2>/dev/null)
+
+          if [ "$BROWSER_CHECK" != "chromium_found" ]; then
+            echo "ERROR: Chromium browser not found or not working for e2e tests"
+            echo "$BROWSER_CHECK"
+            exit 1
+          else
+            echo "Playwright browsers are properly installed for e2e tests."
+          fi
+
+          # Docker runtime will handle workspace directory creation
+
+          # Start the application using make run with custom parameters and reduced logging
+          echo "Starting OpenHands using make run..."
+          # Set environment variables to reduce logging verbosity
+          export PYTHONUNBUFFERED=1
+          export LOG_LEVEL=WARNING
+          export UVICORN_LOG_LEVEL=warning
+          export OPENHANDS_LOG_LEVEL=WARNING
+          FRONTEND_PORT=12000 FRONTEND_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0 make run > /tmp/openhands-e2e-test.log 2>&1 &
+
+          # Store the PID of the make run process
+          MAKE_PID=$!
+          echo "OpenHands started with PID: $MAKE_PID"
+
+          # Wait for the application to start
+          echo "Waiting for OpenHands to start..."
+          max_attempts=15
+          attempt=1
+
+          while [ $attempt -le $max_attempts ]; do
+            echo "Checking if OpenHands is running (attempt $attempt of $max_attempts)..."
+
+            # Check if the process is still running
+            if ! ps -p $MAKE_PID > /dev/null; then
+              echo "ERROR: OpenHands process has terminated unexpectedly"
+              echo "Last 50 lines of the log:"
+              tail -n 50 /tmp/openhands-e2e-test.log
+              exit 1
+            fi
+
+            # Check if frontend port is open
+            if nc -z localhost 12000; then
+              # Verify we can get HTML content
+              if curl -s http://localhost:12000 | grep -q "<html"; then
+                echo "SUCCESS: OpenHands is running and serving HTML content on port 12000"
+                break
+              else
+                echo "Port 12000 is open but not serving HTML content yet"
+              fi
+            else
+              echo "Frontend port 12000 is not open yet"
+            fi
+
+            # Show log output on each attempt
+            echo "Recent log output:"
+            tail -n 20 /tmp/openhands-e2e-test.log
+
+            # Wait before next attempt
+            echo "Waiting 10 seconds before next check..."
+            sleep 10
+            attempt=$((attempt + 1))
+
+            # Exit if we've reached the maximum number of attempts
+            if [ $attempt -gt $max_attempts ]; then
+              echo "ERROR: OpenHands failed to start after $max_attempts attempts"
+              echo "Last 50 lines of the log:"
+              tail -n 50 /tmp/openhands-e2e-test.log
+              exit 1
+            fi
+          done
+
+          # Final verification that the app is running
+          if ! nc -z localhost 12000 || ! curl -s http://localhost:12000 | grep -q "<html"; then
+            echo "ERROR: OpenHands is not running properly on port 12000"
+            echo "Last 50 lines of the log:"
+            tail -n 50 /tmp/openhands-e2e-test.log
+            exit 1
+          fi
+
+          # Print success message
+          echo "OpenHands is running successfully on port 12000"
+
+      - name: Run end-to-end tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+        run: |
+          # Check if the application is running
+          if ! nc -z localhost 12000; then
+            echo "ERROR: OpenHands is not running on port 12000"
+            echo "Last 50 lines of the log:"
+            tail -n 50 /tmp/openhands-e2e-test.log
+            exit 1
+          fi
+
+          # Run the tests with detailed output
+          cd tests/e2e
+          poetry run python -m pytest \
+            test_settings.py::test_github_token_configuration \
+            test_conversation.py::test_conversation_start \
+            test_browsing_catchphrase.py::test_browsing_catchphrase \
+            test_multi_conversation_resume.py::test_multi_conversation_resume \
+            -v --no-header --capture=no --timeout=900
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: tests/e2e/test-results/
+          retention-days: 30
+
+      - name: Upload OpenHands logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: openhands-logs
+          path: |
+            /tmp/openhands-e2e-test.log
+            /tmp/openhands-e2e-build.log
+            /tmp/openhands-backend.log
+            /tmp/openhands-frontend.log
+            /tmp/backend-health-check.log
+            /tmp/frontend-check.log
+            /tmp/vite-config.log
+            /tmp/makefile-contents.log
+          retention-days: 30
+
+      - name: Cleanup
+        if: always()
+        run: |
+          # Stop OpenHands processes
+          echo "Stopping OpenHands processes..."
+          pkill -f "python -m openhands.server" || true
+          pkill -f "npm run dev" || true
+          pkill -f "make run" || true
+
+          # Print process status for debugging
+          echo "Checking if any OpenHands processes are still running:"
+          ps aux | grep -E "openhands|npm run dev" || true

.github/workflows/ghcr-build.yml

@@ -225,7 +225,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
+          poetry run pytest -n 0 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
         env:
           DEBUG: "1"
 
@@ -284,7 +284,7 @@ jobs:
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 7 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
+          poetry run pytest -n 0 -raRs --reruns 2 --reruns-delay 5 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
         env:
           DEBUG: "1"
 

.github/workflows/lint-fix.yml

@@ -29,6 +29,12 @@ jobs:
         run: |
           cd frontend
           npm install --frozen-lockfile
+      - name: Generate i18n and route types
+        run: |
+          cd frontend
+          npm run make-i18n
+          npx react-router typegen || true
+
       - name: Fix frontend lint issues
         run: |
           cd frontend
@@ -45,7 +51,7 @@ jobs:
           git config --local user.email "openhands@all-hands.dev"
           git config --local user.name "OpenHands Bot"
           git add -A
-          git commit -m "🤖 Auto-fix frontend linting issues"
+          git commit -m "🤖 Auto-fix frontend linting issues" --no-verify
           git push
 
   # Python lint fixes
@@ -87,5 +93,5 @@ jobs:
           git config --local user.email "openhands@all-hands.dev"
           git config --local user.name "OpenHands Bot"
           git add -A
-          git commit -m "🤖 Auto-fix Python linting issues"
+          git commit -m "🤖 Auto-fix Python linting issues" --no-verify
           git push

.github/workflows/py-tests.yml

@@ -51,8 +51,6 @@ jobs:
         run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest --forked -n auto -svv ./tests/unit
       - name: Run Runtime Tests with CLIRuntime
         run: PYTHONPATH=".:$PYTHONPATH" TEST_RUNTIME=cli poetry run pytest -svv tests/runtime/test_bash.py
-      - name: Run E2E Tests
-        run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest -svv tests/e2e
 
   # Run specific Windows python tests
   test-on-windows:
@@ -75,7 +73,7 @@ jobs:
       - name: Install Python dependencies using Poetry
         run: poetry install --with dev,test,runtime
       - name: Run Windows unit tests
-        run: poetry run pytest -svv tests/unit/test_windows_bash.py
+        run: poetry run pytest -svv tests/unit/runtime/utils/test_windows_bash.py
         env:
           PYTHONPATH: ".;$env:PYTHONPATH"
           DEBUG: "1"

.github/workflows/stale.yml

@@ -15,7 +15,7 @@ jobs:
           stale-issue-message: 'This issue is stale because it has been open for 40 days with no activity. Remove the stale label or leave a comment, otherwise it will be closed in 10 days.'
           stale-pr-message: 'This PR is stale because it has been open for 40 days with no activity. Remove the stale label or leave a comment, otherwise it will be closed in 10 days.'
           days-before-stale: 40
-          exempt-issue-labels: 'roadmap'
+          exempt-issue-labels: roadmap,backlog
           close-issue-message: 'This issue was automatically closed due to 50 days of inactivity. We do this to help keep the issues somewhat manageable and focus on active issues.'
           close-pr-message: 'This PR was closed because it had no activity for 50 days. If you feel this was closed in error, and you would like to continue the PR, please resubmit or let us know.'
           days-before-close: 10

.github/workflows/welcome-good-first-issue.yml

@@ -0,0 +1,51 @@
+name: Welcome Good First Issue
+
+on:
+  issues:
+    types: [labeled]
+
+permissions:
+  issues: write
+
+jobs:
+  comment-on-good-first-issue:
+    if: github.event.label.name == 'good first issue'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if welcome comment already exists
+        id: check_comment
+        uses: actions/github-script@v7
+        with:
+          result-encoding: string
+          script: |
+            const issueNumber = context.issue.number;
+            const comments = await github.rest.issues.listComments({
+              ...context.repo,
+              issue_number: issueNumber
+            });
+
+            const alreadyCommented = comments.data.some(
+              (comment) =>
+                comment.body.includes('<!-- auto-comment:good-first-issue -->')
+            );
+
+            return alreadyCommented ? 'true' : 'false';
+
+      - name: Leave welcome comment
+        if: steps.check_comment.outputs.result == 'false'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const repoUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}`;
+
+            await github.rest.issues.createComment({
+              ...context.repo,
+              issue_number: context.issue.number,
+              body: "🙌 **Hey there, future contributor!** 🙌\n\n" +
+                    "This issue has been labeled as **good first issue**, which means it's a great place to get started with the OpenHands project.\n\n" +
+                    "If you're interested in working on it, feel free to! No need to ask for permission.\n\n" +
+                    "Be sure to check out our [development setup guide](" + repoUrl + "/blob/main/Development.md) to get your environment set up, and follow our [contribution guidelines](" + repoUrl + "/blob/main/CONTRIBUTING.md) when you're ready to submit a fix.\n\n" +
+                    "Feel free to join our developer community on [Slack](dub.sh/openhands). You can ask for [help](https://openhands-ai.slack.com/archives/C078L0FUGUX), [feedback](https://openhands-ai.slack.com/archives/C086ARSNMGA), and even ask for a [PR review](https://openhands-ai.slack.com/archives/C08D8FJ5771).\n\n" +
+                    "🙌 Happy hacking! 🙌\n\n" +
+                    "<!-- auto-comment:good-first-issue -->"
+            });

.gitignore

@@ -254,3 +254,8 @@ containers/runtime/Dockerfile
 containers/runtime/project.tar.gz
 containers/runtime/code
 **/node_modules/
+
+# test results
+test-results
+.sessions
+.eval_sessions

.openhands/microagents/repo.md

@@ -87,6 +87,8 @@ VSCode Extension:
 
 If you are starting a pull request (PR), please follow the template in `.github/pull_request_template.md`.
 
+If you need to add labels when opening a PR, check the existing labels defined on that repository and select from existing ones. Do not invent your own labels.
+
 ## Implementation Details
 
 These details may or may not be useful for your current task.
@@ -142,6 +144,35 @@ Your specialized knowledge and instructions here...
      - Add the setting to the `Settings` model in `openhands/storage/data_models/settings.py`
      - Update any relevant backend code to apply the setting (e.g., in session creation)
 
+#### Settings UI Patterns:
+
+There are two main patterns for saving settings in the OpenHands frontend:
+
+**Pattern 1: Entity-based Resources (Immediate Save)**
+- Used for: API Keys, Secrets, MCP Servers
+- Behavior: Changes are saved immediately when user performs actions (add/edit/delete)
+- Implementation:
+  - No "Save Changes" button
+  - No local state management or `isDirty` tracking
+  - Uses dedicated mutation hooks for each operation (e.g., `use-add-mcp-server.ts`, `use-delete-mcp-server.ts`)
+  - Each mutation triggers immediate API call with query invalidation for UI updates
+  - Example: MCP settings, API Keys & Secrets tabs
+- Benefits: Simpler UX, no risk of losing changes, consistent with modern web app patterns
+
+**Pattern 2: Form-based Settings (Manual Save)**
+- Used for: Application settings, LLM configuration
+- Behavior: Changes are accumulated locally and saved when user clicks "Save Changes"
+- Implementation:
+  - Has "Save Changes" button that becomes enabled when changes are detected
+  - Uses local state management with `isDirty` tracking
+  - Uses `useSaveSettings` hook to save all changes at once
+  - Example: LLM tab, Application tab
+- Benefits: Allows bulk changes, explicit save action, can validate all fields before saving
+
+**When to use each pattern:**
+- Use Pattern 1 (Immediate Save) for entity management where each item is independent
+- Use Pattern 2 (Manual Save) for configuration forms where settings are interdependent or need validation
+
 ### Adding New LLM Models
 
 To add a new LLM model to OpenHands, you need to update multiple files across both frontend and backend:

Development.md

@@ -159,7 +159,7 @@ poetry run pytest ./tests/unit/test_*.py
 To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker
 container image by setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
 
-Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.51-nikolaik`
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.55-nikolaik`
 
 ## Develop inside Docker container
 

LICENSE

@@ -1,7 +1,12 @@
-The MIT License (MIT)
+Portions of this software are licensed as follows:
+* All content that resides under the enterprise/ directory is licensed under the license defined in "enterprise/LICENSE".
+* Content outside of the above mentioned directories or restrictions above is available under the MIT license as defined below.
+
 =====================
 
-Copyright © 2023
+The MIT License (MIT)
+
+Copyright © 2025
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.md

@@ -52,37 +52,63 @@ which comes with $20 in free credits for new users.
 
 ## 💻 Running OpenHands Locally
 
-OpenHands can also run on your local system using Docker.
-See the [Running OpenHands](https://docs.all-hands.dev/usage/installation) guide for
-system requirements and more information.
+### Option 1: CLI Launcher (Recommended)
 
-> [!WARNING]
-> On a public network? See our [Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)
-> to secure your deployment by restricting network binding and implementing additional security measures.
+The easiest way to run OpenHands locally is using the CLI launcher with [uv](https://docs.astral.sh/uv/). This provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers.
 
+**Install uv** (if you haven't already):
+
+See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+**Launch OpenHands**:
+```bash
+# Launch the GUI server
+uvx --python 3.12 --from openhands-ai openhands serve
+
+# Or launch the CLI
+uvx --python 3.12 --from openhands-ai openhands
+```
+
+You'll find OpenHands running at [http://localhost:3000](http://localhost:3000) (for GUI mode)!
+
+### Option 2: Docker
+
+<details>
+<summary>Click to expand Docker command</summary>
+
+You can also run OpenHands directly with Docker:
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51
+    docker.all-hands.dev/all-hands-ai/openhands:0.55
 ```
 
+</details>
+
 > **Note**: If you used OpenHands before version 0.44, you may want to run `mv ~/.openhands-state ~/.openhands` to migrate your conversation history to the new location.
 
-You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
+> [!WARNING]
+> On a public network? See our [Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)
+> to secure your deployment by restricting network binding and implementing additional security measures.
+
+### Getting Started
 
 When you open the application, you'll be asked to choose an LLM provider and add an API key.
 [Anthropic's Claude Sonnet 4](https://www.anthropic.com/api) (`anthropic/claude-sonnet-4-20250514`)
 works best, but you have [many options](https://docs.all-hands.dev/usage/llms).
 
+See the [Running OpenHands](https://docs.all-hands.dev/usage/installation) guide for
+system requirements and more information.
+
 ## 💡 Other ways to run OpenHands
 
 > [!WARNING]
@@ -93,8 +119,8 @@ works best, but you have [many options](https://docs.all-hands.dev/usage/llms).
 > [OpenHands Cloud Helm Chart](https://github.com/all-Hands-AI/OpenHands-cloud)
 
 You can [connect OpenHands to your local filesystem](https://docs.all-hands.dev/usage/runtimes/docker#connecting-to-your-filesystem),
-run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/usage/how-to/headless-mode),
 interact with it via a [friendly CLI](https://docs.all-hands.dev/usage/how-to/cli-mode),
+run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/usage/how-to/headless-mode),
 or run it on tagged issues with [a github action](https://docs.all-hands.dev/usage/how-to/github-action).
 
 Visit [Running OpenHands](https://docs.all-hands.dev/usage/installation) for more information and setup instructions.
@@ -104,7 +130,6 @@ If you want to modify the OpenHands source code, check out [Development.md](http
 Having issues? The [Troubleshooting Guide](https://docs.all-hands.dev/usage/troubleshooting) can help.
 
 ## 📖 Documentation
-  <a href="https://deepwiki.com/All-Hands-AI/OpenHands"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki" title="Autogenerated Documentation by DeepWiki"></a>
 
 To learn more about the project, and for tips on using OpenHands,
 check out our [documentation](https://docs.all-hands.dev/usage/getting-started).

README_CN.md

@@ -51,17 +51,17 @@ OpenHands也可以使用Docker在本地系统上运行。
 
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51
+    docker.all-hands.dev/all-hands-ai/openhands:0.55
 ```
 
 > **注意**: 如果您在0.44版本之前使用过OpenHands，您可能需要运行 `mv ~/.openhands-state ~/.openhands` 来将对话历史迁移到新位置。

README_JA.md

@@ -42,17 +42,17 @@ OpenHandsはDockerを利用してローカル環境でも実行できます。
 > 公共ネットワークで実行していますか？[Hardened Docker Installation Guide](https://docs.all-hands.dev/usage/runtimes/docker#hardened-docker-installation)を参照して、ネットワークバインディングの制限や追加のセキュリティ対策を実施してください。
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51
+    docker.all-hands.dev/all-hands-ai/openhands:0.55
 ```
 
 **注**: バージョン0.44以前のOpenHandsを使用していた場合は、会話履歴を移行するために `mv ~/.openhands-state ~/.openhands` を実行してください。

build_vscode.py

@@ -93,8 +93,7 @@ def build_vscode_extension():
 
 
 def build(setup_kwargs):
-    """
-    This function is called by Poetry during the build process.
+    """This function is called by Poetry during the build process.
     `setup_kwargs` is a dictionary that will be passed to `setuptools.setup()`.
     """
     print('--- Running custom Poetry build script (build_vscode.py) ---')

config.template.toml

@@ -363,10 +363,11 @@ classpath = "my_package.my_module.MyCustomAgent"
 #confirmation_mode = false
 
 # The security analyzer to use (For Headless / CLI only -  In Web this is overridden by Session Init)
-#security_analyzer = ""
+# Available options: 'llm' (default), 'invariant'
+#security_analyzer = "llm"
 
 # Whether to enable security analyzer
-#enable_security_analyzer = false
+#enable_security_analyzer = true
 
 #################################### Condenser #################################
 # Condensers control how conversation history is managed and compressed when

containers/app/Dockerfile

@@ -21,7 +21,7 @@ ENV POETRY_NO_INTERACTION=1 \
     POETRY_CACHE_DIR=/tmp/poetry_cache
 
 RUN apt-get update -y \
-    && apt-get install -y curl make git build-essential \
+    && apt-get install -y curl make git build-essential jq gettext \
     && python3 -m pip install poetry --break-system-packages
 
 COPY pyproject.toml poetry.lock ./
@@ -58,34 +58,34 @@ RUN sed -i 's/^UID_MIN.*/UID_MIN 499/' /etc/login.defs
 # Default is 60000, but we've seen up to 200000
 RUN sed -i 's/^UID_MAX.*/UID_MAX 1000000/' /etc/login.defs
 
-RUN groupadd --gid $OPENHANDS_USER_ID app
+RUN groupadd --gid $OPENHANDS_USER_ID openhands
 RUN useradd -l -m -u $OPENHANDS_USER_ID --gid $OPENHANDS_USER_ID -s /bin/bash openhands && \
-    usermod -aG app openhands && \
+    usermod -aG openhands openhands && \
     usermod -aG sudo openhands && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-RUN chown -R openhands:app /app && chmod -R 770 /app
-RUN sudo chown -R openhands:app $WORKSPACE_BASE && sudo chmod -R 770 $WORKSPACE_BASE
+RUN chown -R openhands:openhands /app && chmod -R 770 /app
+RUN sudo chown -R openhands:openhands $WORKSPACE_BASE && sudo chmod -R 770 $WORKSPACE_BASE
 USER openhands
 
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH" \
     PYTHONPATH='/app'
 
-COPY --chown=openhands:app --chmod=770 --from=backend-builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY --chown=openhands:openhands --chmod=770 --from=backend-builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-COPY --chown=openhands:app --chmod=770 ./microagents ./microagents
-COPY --chown=openhands:app --chmod=770 ./openhands ./openhands
-COPY --chown=openhands:app --chmod=777 ./openhands/runtime/plugins ./openhands/runtime/plugins
-COPY --chown=openhands:app pyproject.toml poetry.lock README.md MANIFEST.in LICENSE ./
+COPY --chown=openhands:openhands --chmod=770 ./microagents ./microagents
+COPY --chown=openhands:openhands --chmod=770 ./openhands ./openhands
+COPY --chown=openhands:openhands --chmod=777 ./openhands/runtime/plugins ./openhands/runtime/plugins
+COPY --chown=openhands:openhands pyproject.toml poetry.lock README.md MANIFEST.in LICENSE ./
 
 # This is run as "openhands" user, and will create __pycache__ with openhands:openhands ownership
 RUN python openhands/core/download.py # No-op to download assets
 # Add this line to set group ownership of all files/directories not already in "app" group
-# openhands:openhands -> openhands:app
-RUN find /app \! -group app -exec chgrp app {} +
+# openhands:openhands -> openhands:openhands
+RUN find /app \! -group openhands -exec chgrp openhands {} +
 
-COPY --chown=openhands:app --chmod=770 --from=frontend-builder /app/build ./frontend/build
-COPY --chown=openhands:app --chmod=770 ./containers/app/entrypoint.sh /app/entrypoint.sh
+COPY --chown=openhands:openhands --chmod=770 --from=frontend-builder /app/build ./frontend/build
+COPY --chown=openhands:openhands --chmod=770 ./containers/app/entrypoint.sh /app/entrypoint.sh
 
 USER root
 

containers/app/entrypoint.sh

@@ -54,7 +54,7 @@ else
       fi
     fi
   fi
-  usermod -aG app enduser
+  usermod -aG openhands enduser
   # get the user group of /var/run/docker.sock and set openhands to that group
   DOCKER_SOCKET_GID=$(stat -c '%g' /var/run/docker.sock)
   echo "Docker socket group id: $DOCKER_SOCKET_GID"

containers/dev/compose.yml

@@ -12,7 +12,7 @@ services:
       - SANDBOX_API_HOSTNAME=host.docker.internal
       - DOCKER_HOST_ADDR=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.51-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.55-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

dev_config/python/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies:
-          [types-requests, types-setuptools, types-pyyaml, types-toml, types-docker, pydantic, lxml]
+          [types-requests, types-setuptools, types-pyyaml, types-toml, types-docker, types-Markdown, pydantic, lxml]
         # To see gaps add `--html-report mypy-report/`
         entry: mypy --config-file dev_config/python/mypy.ini openhands/
         always_run: true

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik}
       #- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of ~/.openhands for this user
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/docs.json

@@ -153,7 +153,9 @@
                 "group": "Architecture",
                 "pages": [
                   "usage/architecture/backend",
-                  "usage/architecture/runtime"
+                  "usage/architecture/runtime",
+                  "usage/architecture/auth-system-summary",
+                  "usage/architecture/auth-system-design"
                 ]
               },
               "usage/how-to/debugging",

docs/usage/architecture/auth-system-design.md

@@ -0,0 +1,856 @@
+# OpenHands AuthSystem Design
+
+## Executive Summary
+
+This document proposes a comprehensive AuthSystem design for OpenHands that supports three authentication strategies: **None** (current behavior), **Single User (SU)** with GitHub OAuth, and **Multi User (MU)** (for custom builds). The design introduces clean abstraction boundaries, eliminates scattered `user_id` threading, and provides a foundation for future authentication enhancements.
+
+## Problem Statement
+
+### Current Issues
+
+1. **No Auth Strategy Abstraction**: OpenHands currently has a monolithic `DefaultUserAuth` that always returns `None` for `user_id`, with no clear path to support different authentication modes.
+
+2. **Scattered user_id Threading**: 339+ occurrences of `user_id` across 68 files, with complex threading through:
+   - Storage partitioning (`users/{user_id}/` paths)
+   - Conversation/session scoping
+   - API route dependencies
+   - Provider token resolution
+   - Data model fields
+
+3. **Provider Token Pollution**: Routes accept `provider_tokens` parameters and thread them through `ProviderHandler`, creating security risks and complex signatures.
+
+4. **No Single User Support**: No way to enable GitHub OAuth for personal/single-user deployments while maintaining the simplicity of the current "None" mode.
+
+5. **Boundary Violations**: Auth concerns are mixed with business logic throughout the codebase, making it difficult to switch between authentication modes.
+
+### Requirements from GitHub Issues
+
+From **Issue #10751** (user_id audit):
+- Support None, SU, and MU modes
+- Introduce `UserContext` and `StorageNamespace` abstractions
+- Remove redundant `if user_id` guards (7 identified)
+- Clean up storage path helpers
+
+From **Issue #10730** (token provider):
+- Remove `provider_tokens` dependency injection
+- Introduce `TokenProvider` boundary abstraction
+- Support backend-only credential resolution
+- Enable custom builds with token refresh/rotation patterns
+
+## Solution Architecture
+
+### Core Components
+
+#### 1. AuthStrategy Interface
+
+```python
+# openhands/auth/strategies/base.py
+from abc import ABC, abstractmethod
+from typing import Optional
+from fastapi import Request
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+
+class AuthStrategy(ABC):
+    """Base class for authentication strategies"""
+    
+    @abstractmethod
+    def get_name(self) -> str:
+        """Return strategy name for logging/debugging"""
+    
+    @abstractmethod
+    def requires_auth(self) -> bool:
+        """Whether this strategy requires user authentication"""
+    
+    @abstractmethod
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        """Authenticate request and return UserContext or None"""
+    
+    @abstractmethod
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        """Get token provider for this request"""
+    
+    @abstractmethod
+    def get_login_url(self) -> Optional[str]:
+        """Get login URL for frontend, None if no auth required"""
+```
+
+#### 2. UserContext
+
+```python
+# openhands/auth/user_context.py
+from dataclasses import dataclass
+from typing import Optional
+from datetime import datetime
+
+@dataclass(frozen=True)
+class UserContext:
+    """Immutable user context for authenticated requests"""
+    
+    user_id: str
+    email: Optional[str] = None
+    username: Optional[str] = None
+    github_id: Optional[int] = None
+    github_username: Optional[str] = None
+    is_admin: bool = False
+    created_at: Optional[datetime] = None
+    last_login: Optional[datetime] = None
+    
+    @property
+    def storage_namespace(self) -> str:
+        """Get storage namespace for this user"""
+        return self.user_id
+```
+
+#### 3. TokenProvider Interface
+
+```python
+# openhands/auth/token_provider.py
+from abc import ABC, abstractmethod
+from typing import Optional, Mapping
+from openhands.integrations.service_types import ProviderType
+from openhands.integrations.provider import ProviderToken
+
+class TokenProvider(ABC):
+    """Abstract token provider for git integrations"""
+    
+    @abstractmethod
+    async def get_token(self, provider: ProviderType) -> Optional[ProviderToken]:
+        """Get token for specific provider"""
+    
+    @abstractmethod
+    async def get_all_tokens(self) -> Mapping[ProviderType, ProviderToken]:
+        """Get all available provider tokens"""
+```
+
+#### 4. StorageNamespace
+
+```python
+# openhands/auth/storage_namespace.py
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass(frozen=True)
+class StorageNamespace:
+    """Encapsulates storage path logic for user data"""
+    
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+    
+    def get_conversation_events_dir(self, sid: str) -> str:
+        return f'{self.get_conversation_dir(sid)}events/'
+    
+    def get_conversation_metadata_filename(self, sid: str) -> str:
+        return f'{self.get_conversation_dir(sid)}metadata.json'
+    
+    # ... other path methods
+```
+
+### Authentication Strategies
+
+#### 1. None Strategy (Current Behavior)
+
+```python
+# openhands/auth/strategies/none_strategy.py
+from typing import Optional
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider, DefaultTokenProvider
+
+class NoneStrategy(AuthStrategy):
+    """No authentication - current OpenHands behavior"""
+    
+    def get_name(self) -> str:
+        return "none"
+    
+    def requires_auth(self) -> bool:
+        return False
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        return None  # No user context
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        return DefaultTokenProvider()  # Uses secrets.json
+    
+    def get_login_url(self) -> Optional[str]:
+        return None
+```
+
+#### 2. Single User Strategy
+
+```python
+# openhands/auth/strategies/single_user_strategy.py
+from typing import Optional
+from fastapi import Request, HTTPException
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider, SingleUserTokenProvider
+from openhands.server.shared import server_config
+
+class SingleUserStrategy(AuthStrategy):
+    """Single user with GitHub OAuth"""
+    
+    def get_name(self) -> str:
+        return "single_user"
+    
+    def requires_auth(self) -> bool:
+        return server_config.enable_su_auth
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        if not self.requires_auth():
+            # SU mode without auth - create virtual user
+            return UserContext(
+                user_id="local",
+                username="local_user",
+                is_admin=True
+            )
+        
+        # Extract JWT token from cookie/header
+        token = self._extract_token(request)
+        if not token:
+            return None
+            
+        # Validate JWT and extract user info
+        user_data = self._validate_jwt(token)
+        if not user_data:
+            return None
+            
+        # Verify user is allowed (if configured)
+        if (server_config.su_github_username and 
+            user_data.get('github_username') != server_config.su_github_username):
+            raise HTTPException(403, "Access denied")
+            
+        return UserContext(
+            user_id=user_data['github_username'],
+            email=user_data.get('email'),
+            username=user_data['github_username'],
+            github_id=user_data.get('github_id'),
+            github_username=user_data['github_username'],
+            is_admin=True  # SU user is always admin
+        )
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        user_context = await self.authenticate(request)
+        return SingleUserTokenProvider(user_context)
+    
+    def get_login_url(self) -> Optional[str]:
+        if not self.requires_auth():
+            return None
+        return f"/api/auth/github/login"
+```
+
+#### 3. Multi User Strategy (Custom Build Extension Point)
+
+```python
+# openhands/auth/strategies/multi_user_strategy.py
+from typing import Optional
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+
+class MultiUserStrategy(AuthStrategy):
+    """Multi-user strategy - extension point for custom builds"""
+    
+    def get_name(self) -> str:
+        return "multi_user"
+    
+    def requires_auth(self) -> bool:
+        return True
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        # This would be implemented by custom builds/applications built on OH
+        raise NotImplementedError("Multi-user strategy not available in base OpenHands")
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        raise NotImplementedError("Multi-user strategy not available in base OpenHands")
+    
+    def get_login_url(self) -> Optional[str]:
+        return "/api/auth/login"
+```
+
+### Integration Points
+
+#### 1. Updated UserAuth
+
+```python
+# openhands/server/user_auth/strategy_user_auth.py
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.storage_namespace import StorageNamespace
+from openhands.server.user_auth.user_auth import UserAuth
+from openhands.storage.settings.settings_store import SettingsStore
+from openhands.storage.secrets.secrets_store import SecretsStore
+
+class StrategyUserAuth(UserAuth):
+    """UserAuth implementation using AuthStrategy pattern"""
+    
+    def __init__(self, strategy: AuthStrategy, user_context: Optional[UserContext]):
+        self.strategy = strategy
+        self.user_context = user_context
+        self._storage_namespace = StorageNamespace(
+            user_context.storage_namespace if user_context else None
+        )
+    
+    async def get_user_id(self) -> str | None:
+        return self.user_context.user_id if self.user_context else None
+    
+    async def get_user_email(self) -> str | None:
+        return self.user_context.email if self.user_context else None
+    
+    # ... other methods using storage_namespace
+```
+
+#### 2. FastAPI Dependencies
+
+```python
+# openhands/server/dependencies/auth.py
+from fastapi import Depends, Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+from openhands.server.shared import get_auth_strategy
+
+async def get_current_user(
+    request: Request,
+    strategy: AuthStrategy = Depends(get_auth_strategy)
+) -> Optional[UserContext]:
+    """Get current user context"""
+    return await strategy.authenticate(request)
+
+async def get_token_provider(
+    request: Request,
+    strategy: AuthStrategy = Depends(get_auth_strategy)
+) -> TokenProvider:
+    """Get token provider for current request"""
+    return await strategy.get_token_provider(request)
+
+async def require_auth(
+    user: Optional[UserContext] = Depends(get_current_user)
+) -> UserContext:
+    """Require authentication"""
+    if not user:
+        raise HTTPException(401, "Authentication required")
+    return user
+```
+
+#### 3. Updated Routes
+
+```python
+# openhands/server/routes/git.py (AFTER)
+from fastapi import APIRouter, Depends
+from openhands.auth.token_provider import TokenProvider
+from openhands.auth.user_context import UserContext
+from openhands.server.dependencies.auth import get_token_provider, get_current_user
+from openhands.integrations.provider import ProviderHandler
+
+app = APIRouter(prefix='/api/user')
+
+@app.get('/repositories')
+async def get_user_repositories(
+    sort: str = "pushed",
+    selected_provider: ProviderType | None = None,
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user)
+):
+    """Get user repositories - no provider_tokens parameter!"""
+    client = ProviderHandler(token_provider=token_provider)
+    return await client.get_repositories(sort, selected_provider)
+```
+
+## Before/After Code Comparison
+
+### Before: Current Implementation
+
+```python
+# BEFORE: openhands/server/routes/git.py
+@app.get('/repositories', response_model=list[Repository])
+async def get_user_repositories(
+    sort: str = Query(default='pushed'),
+    selected_provider: ProviderType | None = Query(default=None),
+    page: int | None = Query(default=None),
+    per_page: int | None = Query(default=None),
+    installation_id: str | None = Query(default=None),
+    provider_tokens: PROVIDER_TOKEN_TYPE | None = Depends(get_provider_tokens),
+    access_token: SecretStr | None = Depends(get_access_token),
+    user_id: str | None = Depends(get_user_id),
+):
+    if provider_tokens:
+        client = ProviderHandler(
+            provider_tokens=provider_tokens,
+            external_auth_token=access_token,
+            external_auth_id=user_id,
+        )
+        # ... complex logic
+```
+
+```python
+# BEFORE: openhands/storage/locations.py
+def get_conversation_dir(sid: str, user_id: str | None = None) -> str:
+    if user_id:
+        return f'users/{user_id}/conversations/{sid}/'
+    else:
+        return f'sessions/{sid}/'
+```
+
+```python
+# BEFORE: openhands/server/user_auth/default_user_auth.py
+class DefaultUserAuth(UserAuth):
+    async def get_user_id(self) -> str | None:
+        return None  # Always None - no multi-tenancy support
+    
+    async def get_provider_tokens(self) -> PROVIDER_TOKEN_TYPE | None:
+        user_secrets = await self.get_user_secrets()
+        if user_secrets is None:
+            return None
+        return user_secrets.provider_tokens
+```
+
+### After: Proposed Implementation
+
+```python
+# AFTER: openhands/server/routes/git.py
+@app.get('/repositories', response_model=list[Repository])
+async def get_user_repositories(
+    sort: str = Query(default='pushed'),
+    selected_provider: ProviderType | None = Query(default=None),
+    page: int | None = Query(default=None),
+    per_page: int | None = Query(default=None),
+    installation_id: str | None = Query(default=None),
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user),
+):
+    client = ProviderHandler(token_provider=token_provider)
+    return await client.get_repositories(
+        sort, server_config.app_mode, selected_provider, page, per_page, installation_id
+    )
+```
+
+```python
+# AFTER: openhands/auth/storage_namespace.py
+@dataclass(frozen=True)
+class StorageNamespace:
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+```
+
+```python
+# AFTER: openhands/server/user_auth/strategy_user_auth.py
+class StrategyUserAuth(UserAuth):
+    def __init__(self, strategy: AuthStrategy, user_context: Optional[UserContext]):
+        self.strategy = strategy
+        self.user_context = user_context
+        self.storage_namespace = StorageNamespace(
+            user_context.storage_namespace if user_context else None
+        )
+    
+    async def get_user_id(self) -> str | None:
+        return self.user_context.user_id if self.user_context else None
+```
+
+## Configuration
+
+### Environment Variables
+
+```bash
+# Authentication Strategy
+OH_AUTH_STRATEGY=none  # Options: none, single_user, multi_user
+
+# Single User Mode Settings
+OH_ENABLE_SU_AUTH=false  # Enable GitHub OAuth in SU mode
+OH_SU_GITHUB_USERNAME=your_username  # Restrict access to specific user
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+
+# Multi User Mode (custom build extension point)
+OH_MU_ADMIN_USERNAME=admin_user
+```
+
+### Configuration Modes
+
+#### 1. None Mode (Current Default)
+```bash
+OH_AUTH_STRATEGY=none
+# No additional config needed
+```
+
+#### 2. Single User - No Auth
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+```
+
+#### 3. Single User - GitHub Auth
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=your_username
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+```
+
+## Implementation Benefits
+
+### 1. Clean Separation of Concerns
+- Auth logic isolated in strategy classes
+- Business logic doesn't need to know about user_id
+- Clear boundaries between auth and core functionality
+
+### 2. Reduced Complexity
+- Eliminates 7 redundant `if user_id` guards
+- Removes provider_tokens dependency injection
+- Simplifies method signatures throughout codebase
+
+### 3. Forward Compatibility
+- custom builds can extend with custom strategies
+- Token refresh/rotation support built-in
+- Multi-tenancy ready without core changes
+
+### 4. Security Improvements
+- Tokens never exposed in route parameters
+- Centralized token management
+- Immutable user context prevents tampering
+
+### 5. Developer Experience
+- Clear configuration options
+- Easy mode switching
+- Consistent patterns across codebase
+
+## Migration Strategy
+
+### Phase 1: Foundation
+1. Introduce auth strategy interfaces
+2. Add UserContext and StorageNamespace
+3. Create TokenProvider abstraction
+4. Update core dependencies
+
+### Phase 2: Strategy Implementation
+1. Implement NoneStrategy (backward compatible)
+2. Implement SingleUserStrategy
+3. Add configuration support
+4. Update UserAuth integration
+
+### Phase 3: Route Migration
+1. Update FastAPI dependencies
+2. Remove provider_tokens dependency injection
+3. Update ProviderHandler integration
+4. Clean up redundant if-guards
+
+### Phase 4: Storage Migration
+1. Replace storage path helpers
+2. Update conversation managers
+3. Migrate event stores
+4. Clean up legacy code
+
+## Testing Strategy
+
+### Unit Tests
+- Strategy implementations
+- UserContext immutability
+- StorageNamespace path generation
+- TokenProvider implementations
+
+### Integration Tests
+- End-to-end auth flows
+- Route authentication
+- Storage partitioning
+- Configuration switching
+
+### Migration Tests
+- Backward compatibility
+- Data migration paths
+- Configuration validation
+
+## Future Extensions
+
+### custom builds Integration Points
+```python
+# custom builds can provide their own strategies
+class custom buildsMultiUserStrategy(AuthStrategy):
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        # Custom custom builds authentication logic
+        pass
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        # custom builds token refresh/rotation
+        return CustomBuildTokenProvider(request)
+```
+
+### Additional Auth Methods
+- SAML/OIDC strategies
+- API key authentication
+- Custom JWT providers
+- Enterprise SSO integration
+
+## Architecture Diagrams
+
+### 1. Overall Auth System Architecture
+
+```mermaid
+graph TB
+    subgraph "FastAPI Application"
+        Routes[API Routes]
+        Deps[FastAPI Dependencies]
+    end
+    
+    subgraph "Auth Layer"
+        AuthStrategy[AuthStrategy Interface]
+        NoneStrategy[NoneStrategy]
+        SUStrategy[SingleUserStrategy]
+        MUStrategy[MultiUserStrategy]
+    end
+    
+    subgraph "Core Abstractions"
+        UserContext[UserContext]
+        TokenProvider[TokenProvider Interface]
+        StorageNamespace[StorageNamespace]
+    end
+    
+    subgraph "Token Providers"
+        DefaultTP[DefaultTokenProvider]
+        SingleUserTP[SingleUserTokenProvider]
+        custom buildsTP[CustomBuildTokenProvider]
+    end
+    
+    subgraph "Storage Layer"
+        SecretsStore[SecretsStore]
+        SettingsStore[SettingsStore]
+        ConversationStore[ConversationStore]
+    end
+    
+    Routes --> Deps
+    Deps --> AuthStrategy
+    AuthStrategy --> UserContext
+    AuthStrategy --> TokenProvider
+    UserContext --> StorageNamespace
+    
+    NoneStrategy --> DefaultTP
+    SUStrategy --> SingleUserTP
+    MUStrategy --> custom buildsTP
+    
+    TokenProvider --> SecretsStore
+    StorageNamespace --> ConversationStore
+    StorageNamespace --> SettingsStore
+```
+
+### 2. Authentication Flow - None Strategy
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant NoneStrategy
+    participant DefaultTP
+    participant SecretsStore
+    
+    Client->>Route: API Request
+    Route->>NoneStrategy: authenticate(request)
+    NoneStrategy->>Route: None (no user context)
+    Route->>NoneStrategy: get_token_provider(request)
+    NoneStrategy->>DefaultTP: create()
+    DefaultTP->>SecretsStore: load secrets.json
+    SecretsStore->>DefaultTP: provider tokens
+    DefaultTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 3. Authentication Flow - Single User Strategy (No Auth)
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant SUStrategy
+    participant UserContext
+    participant SingleUserTP
+    participant SecretsStore
+    
+    Client->>Route: API Request
+    Route->>SUStrategy: authenticate(request)
+    SUStrategy->>UserContext: create virtual user (local)
+    UserContext->>SUStrategy: user context
+    SUStrategy->>Route: UserContext(user_id="local")
+    Route->>SUStrategy: get_token_provider(request)
+    SUStrategy->>SingleUserTP: create(user_context)
+    SingleUserTP->>SecretsStore: load user secrets
+    SecretsStore->>SingleUserTP: provider tokens
+    SingleUserTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 4. Authentication Flow - Single User Strategy (GitHub Auth)
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant SUStrategy
+    participant GitHub
+    participant UserContext
+    participant SingleUserTP
+    participant Database
+    
+    Client->>Route: API Request with JWT cookie
+    Route->>SUStrategy: authenticate(request)
+    SUStrategy->>SUStrategy: extract JWT token
+    SUStrategy->>SUStrategy: validate JWT
+    SUStrategy->>SUStrategy: check allowed user
+    SUStrategy->>UserContext: create from JWT data
+    UserContext->>SUStrategy: user context
+    SUStrategy->>Route: UserContext
+    Route->>SUStrategy: get_token_provider(request)
+    SUStrategy->>SingleUserTP: create(user_context)
+    SingleUserTP->>Database: load encrypted tokens
+    Database->>SingleUserTP: encrypted provider tokens
+    SingleUserTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 5. GitHub OAuth Flow - Single User Strategy
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant AuthRoute
+    participant GitHub
+    participant SUStrategy
+    participant Database
+    participant UserContext
+    
+    Client->>AuthRoute: GET /auth/login
+    AuthRoute->>Client: GitHub OAuth URL
+    Client->>GitHub: OAuth authorization
+    GitHub->>AuthRoute: GET /auth/callback?code=xxx
+    AuthRoute->>GitHub: exchange code for token
+    GitHub->>AuthRoute: access token + user info
+    AuthRoute->>SUStrategy: validate user allowed
+    SUStrategy->>AuthRoute: user authorized
+    AuthRoute->>Database: create/update user record
+    Database->>AuthRoute: user saved
+    AuthRoute->>AuthRoute: create JWT token
+    AuthRoute->>Client: Set JWT cookie + redirect
+```
+
+### 6. Storage Namespace Architecture
+
+```mermaid
+graph TB
+    subgraph "User Context"
+        UC[UserContext]
+        UC --> SN[StorageNamespace]
+    end
+    
+    subgraph "Storage Paths"
+        SN --> ConvDir[get_conversation_dir]
+        SN --> EventsDir[get_events_dir]
+        SN --> MetaFile[get_metadata_file]
+        SN --> StateFile[get_state_file]
+    end
+    
+    subgraph "Path Examples"
+        ConvDir --> NonePath[sessions/sid/]
+        ConvDir --> UserPath[users/user_id/conversations/sid/]
+        
+        EventsDir --> NoneEvents[sessions/sid/events/]
+        EventsDir --> UserEvents[users/user_id/conversations/sid/events/]
+    end
+    
+    subgraph "Strategy Impact"
+        NoneStrategy2[NoneStrategy] --> NonePath
+        SUStrategy2[SingleUserStrategy] --> UserPath
+        MUStrategy2[MultiUserStrategy] --> UserPath
+    end
+```
+
+### 7. Token Provider Architecture
+
+```mermaid
+graph TB
+    subgraph "Token Provider Interface"
+        TP[TokenProvider]
+        TP --> GetToken[get_token(provider)]
+        TP --> GetAllTokens[get_all_tokens()]
+    end
+    
+    subgraph "Implementations"
+        DefaultTP2[DefaultTokenProvider]
+        SingleUserTP2[SingleUserTokenProvider]
+        custom buildsTP2[CustomBuildTokenProvider]
+    end
+    
+    subgraph "Token Sources"
+        SecretsJSON[secrets.json]
+        UserDB[User Database]
+        Custom Build API[custom builds Token API]
+    end
+    
+    subgraph "Provider Integration"
+        ProviderHandler[ProviderHandler]
+        GitHubService[GitHubService]
+        GitLabService[GitLabService]
+        BitBucketService[BitBucketService]
+    end
+    
+    DefaultTP2 --> SecretsJSON
+    SingleUserTP2 --> UserDB
+    custom buildsTP2 --> Custom Build API
+    
+    TP --> ProviderHandler
+    ProviderHandler --> GitHubService
+    ProviderHandler --> GitLabService
+    ProviderHandler --> BitBucketService
+```
+
+### 8. Configuration-Driven Strategy Selection
+
+```mermaid
+graph TB
+    subgraph "Configuration"
+        Config[OH_AUTH_STRATEGY]
+        Config --> None[none]
+        Config --> SU[single_user]
+        Config --> MU[multi_user]
+    end
+    
+    subgraph "Strategy Factory"
+        Factory[AuthStrategyFactory]
+        Factory --> CreateNone[create NoneStrategy]
+        Factory --> CreateSU[create SingleUserStrategy]
+        Factory --> CreateMU[create MultiUserStrategy]
+    end
+    
+    subgraph "Additional Config"
+        SUConfig[OH_ENABLE_SU_AUTH<br/>OH_SU_GITHUB_USERNAME<br/>OH_GITHUB_CLIENT_ID]
+        MUConfig[OH_MU_ADMIN_USERNAME<br/>Database Config]
+    end
+    
+    None --> CreateNone
+    SU --> CreateSU
+    MU --> CreateMU
+    
+    CreateSU --> SUConfig
+    CreateMU --> MUConfig
+```
+
+## Conclusion
+
+This AuthSystem design provides OpenHands with a robust, extensible authentication foundation that:
+
+1. **Maintains backward compatibility** with the current "None" mode
+2. **Enables Single User mode** with optional GitHub OAuth
+3. **Provides extension points** for custom builds with multi-user implementations
+4. **Cleans up the codebase** by removing scattered user_id threading
+5. **Improves security** by centralizing token management
+6. **Simplifies development** with clear abstractions and patterns
+
+The design is ready for implementation and will significantly improve OpenHands' authentication capabilities while maintaining its current simplicity for users who don't need authentication.

docs/usage/architecture/auth-system-design.mdx

@@ -0,0 +1,860 @@
+---
+title: AuthSystem Design - Complete Specification
+---
+
+# OpenHands AuthSystem Design
+
+## Executive Summary
+
+This document proposes a comprehensive AuthSystem design for OpenHands that supports three authentication strategies: **None** (current behavior), **Single User (SU)** with GitHub OAuth, and **Multi User (MU)** (for custom builds). The design introduces clean abstraction boundaries, eliminates scattered `user_id` threading, and provides a foundation for future authentication enhancements.
+
+## Problem Statement
+
+### Current Issues
+
+1. **No Auth Strategy Abstraction**: OpenHands currently has a monolithic `DefaultUserAuth` that always returns `None` for `user_id`, with no clear path to support different authentication modes.
+
+2. **Scattered user_id Threading**: 339+ occurrences of `user_id` across 68 files, with complex threading through:
+   - Storage partitioning (`users/{user_id}/` paths)
+   - Conversation/session scoping
+   - API route dependencies
+   - Provider token resolution
+   - Data model fields
+
+3. **Provider Token Pollution**: Routes accept `provider_tokens` parameters and thread them through `ProviderHandler`, creating security risks and complex signatures.
+
+4. **No Single User Support**: No way to enable GitHub OAuth for personal/single-user deployments while maintaining the simplicity of the current "None" mode.
+
+5. **Boundary Violations**: Auth concerns are mixed with business logic throughout the codebase, making it difficult to switch between authentication modes.
+
+### Requirements from GitHub Issues
+
+From **Issue #10751** (user_id audit):
+- Support None, SU, and MU modes
+- Introduce `UserContext` and `StorageNamespace` abstractions
+- Remove redundant `if user_id` guards (7 identified)
+- Clean up storage path helpers
+
+From **Issue #10730** (token provider):
+- Remove `provider_tokens` dependency injection
+- Introduce `TokenProvider` boundary abstraction
+- Support backend-only credential resolution
+- Enable custom builds with token refresh/rotation patterns
+
+## Solution Architecture
+
+### Core Components
+
+#### 1. AuthStrategy Interface
+
+```python
+# openhands/auth/strategies/base.py
+from abc import ABC, abstractmethod
+from typing import Optional
+from fastapi import Request
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+
+class AuthStrategy(ABC):
+    """Base class for authentication strategies"""
+    
+    @abstractmethod
+    def get_name(self) -> str:
+        """Return strategy name for logging/debugging"""
+    
+    @abstractmethod
+    def requires_auth(self) -> bool:
+        """Whether this strategy requires user authentication"""
+    
+    @abstractmethod
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        """Authenticate request and return UserContext or None"""
+    
+    @abstractmethod
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        """Get token provider for this request"""
+    
+    @abstractmethod
+    def get_login_url(self) -> Optional[str]:
+        """Get login URL for frontend, None if no auth required"""
+```
+
+#### 2. UserContext
+
+```python
+# openhands/auth/user_context.py
+from dataclasses import dataclass
+from typing import Optional
+from datetime import datetime
+
+@dataclass(frozen=True)
+class UserContext:
+    """Immutable user context for authenticated requests"""
+    
+    user_id: str
+    email: Optional[str] = None
+    username: Optional[str] = None
+    github_id: Optional[int] = None
+    github_username: Optional[str] = None
+    is_admin: bool = False
+    created_at: Optional[datetime] = None
+    last_login: Optional[datetime] = None
+    
+    @property
+    def storage_namespace(self) -> str:
+        """Get storage namespace for this user"""
+        return self.user_id
+```
+
+#### 3. TokenProvider Interface
+
+```python
+# openhands/auth/token_provider.py
+from abc import ABC, abstractmethod
+from typing import Optional, Mapping
+from openhands.integrations.service_types import ProviderType
+from openhands.integrations.provider import ProviderToken
+
+class TokenProvider(ABC):
+    """Abstract token provider for git integrations"""
+    
+    @abstractmethod
+    async def get_token(self, provider: ProviderType) -> Optional[ProviderToken]:
+        """Get token for specific provider"""
+    
+    @abstractmethod
+    async def get_all_tokens(self) -> Mapping[ProviderType, ProviderToken]:
+        """Get all available provider tokens"""
+```
+
+#### 4. StorageNamespace
+
+```python
+# openhands/auth/storage_namespace.py
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass(frozen=True)
+class StorageNamespace:
+    """Encapsulates storage path logic for user data"""
+    
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+    
+    def get_conversation_events_dir(self, sid: str) -> str:
+        return f'{self.get_conversation_dir(sid)}events/'
+    
+    def get_conversation_metadata_filename(self, sid: str) -> str:
+        return f'{self.get_conversation_dir(sid)}metadata.json'
+    
+    # ... other path methods
+```
+
+### Authentication Strategies
+
+#### 1. None Strategy (Current Behavior)
+
+```python
+# openhands/auth/strategies/none_strategy.py
+from typing import Optional
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider, DefaultTokenProvider
+
+class NoneStrategy(AuthStrategy):
+    """No authentication - current OpenHands behavior"""
+    
+    def get_name(self) -> str:
+        return "none"
+    
+    def requires_auth(self) -> bool:
+        return False
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        return None  # No user context
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        return DefaultTokenProvider()  # Uses secrets.json
+    
+    def get_login_url(self) -> Optional[str]:
+        return None
+```
+
+#### 2. Single User Strategy
+
+```python
+# openhands/auth/strategies/single_user_strategy.py
+from typing import Optional
+from fastapi import Request, HTTPException
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider, SingleUserTokenProvider
+from openhands.server.shared import server_config
+
+class SingleUserStrategy(AuthStrategy):
+    """Single user with GitHub OAuth"""
+    
+    def get_name(self) -> str:
+        return "single_user"
+    
+    def requires_auth(self) -> bool:
+        return server_config.enable_su_auth
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        if not self.requires_auth():
+            # SU mode without auth - create virtual user
+            return UserContext(
+                user_id="local",
+                username="local_user",
+                is_admin=True
+            )
+        
+        # Extract JWT token from cookie/header
+        token = self._extract_token(request)
+        if not token:
+            return None
+            
+        # Validate JWT and extract user info
+        user_data = self._validate_jwt(token)
+        if not user_data:
+            return None
+            
+        # Verify user is allowed (if configured)
+        if (server_config.su_github_username and 
+            user_data.get('github_username') != server_config.su_github_username):
+            raise HTTPException(403, "Access denied")
+            
+        return UserContext(
+            user_id=user_data['github_username'],
+            email=user_data.get('email'),
+            username=user_data['github_username'],
+            github_id=user_data.get('github_id'),
+            github_username=user_data['github_username'],
+            is_admin=True  # SU user is always admin
+        )
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        user_context = await self.authenticate(request)
+        return SingleUserTokenProvider(user_context)
+    
+    def get_login_url(self) -> Optional[str]:
+        if not self.requires_auth():
+            return None
+        return f"/api/auth/github/login"
+```
+
+#### 3. Multi User Strategy (Custom Build Extension Point)
+
+```python
+# openhands/auth/strategies/multi_user_strategy.py
+from typing import Optional
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+
+class MultiUserStrategy(AuthStrategy):
+    """Multi-user strategy - extension point for custom builds"""
+    
+    def get_name(self) -> str:
+        return "multi_user"
+    
+    def requires_auth(self) -> bool:
+        return True
+    
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        # This would be implemented by custom builds/applications built on OH
+        raise NotImplementedError("Multi-user strategy not available in base OpenHands")
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        raise NotImplementedError("Multi-user strategy not available in base OpenHands")
+    
+    def get_login_url(self) -> Optional[str]:
+        return "/api/auth/login"
+```
+
+### Integration Points
+
+#### 1. Updated UserAuth
+
+```python
+# openhands/server/user_auth/strategy_user_auth.py
+from fastapi import Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.storage_namespace import StorageNamespace
+from openhands.server.user_auth.user_auth import UserAuth
+from openhands.storage.settings.settings_store import SettingsStore
+from openhands.storage.secrets.secrets_store import SecretsStore
+
+class StrategyUserAuth(UserAuth):
+    """UserAuth implementation using AuthStrategy pattern"""
+    
+    def __init__(self, strategy: AuthStrategy, user_context: Optional[UserContext]):
+        self.strategy = strategy
+        self.user_context = user_context
+        self._storage_namespace = StorageNamespace(
+            user_context.storage_namespace if user_context else None
+        )
+    
+    async def get_user_id(self) -> str | None:
+        return self.user_context.user_id if self.user_context else None
+    
+    async def get_user_email(self) -> str | None:
+        return self.user_context.email if self.user_context else None
+    
+    # ... other methods using storage_namespace
+```
+
+#### 2. FastAPI Dependencies
+
+```python
+# openhands/server/dependencies/auth.py
+from fastapi import Depends, Request
+from openhands.auth.strategies.base import AuthStrategy
+from openhands.auth.user_context import UserContext
+from openhands.auth.token_provider import TokenProvider
+from openhands.server.shared import get_auth_strategy
+
+async def get_current_user(
+    request: Request,
+    strategy: AuthStrategy = Depends(get_auth_strategy)
+) -> Optional[UserContext]:
+    """Get current user context"""
+    return await strategy.authenticate(request)
+
+async def get_token_provider(
+    request: Request,
+    strategy: AuthStrategy = Depends(get_auth_strategy)
+) -> TokenProvider:
+    """Get token provider for current request"""
+    return await strategy.get_token_provider(request)
+
+async def require_auth(
+    user: Optional[UserContext] = Depends(get_current_user)
+) -> UserContext:
+    """Require authentication"""
+    if not user:
+        raise HTTPException(401, "Authentication required")
+    return user
+```
+
+#### 3. Updated Routes
+
+```python
+# openhands/server/routes/git.py (AFTER)
+from fastapi import APIRouter, Depends
+from openhands.auth.token_provider import TokenProvider
+from openhands.auth.user_context import UserContext
+from openhands.server.dependencies.auth import get_token_provider, get_current_user
+from openhands.integrations.provider import ProviderHandler
+
+app = APIRouter(prefix='/api/user')
+
+@app.get('/repositories')
+async def get_user_repositories(
+    sort: str = "pushed",
+    selected_provider: ProviderType | None = None,
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user)
+):
+    """Get user repositories - no provider_tokens parameter!"""
+    client = ProviderHandler(token_provider=token_provider)
+    return await client.get_repositories(sort, selected_provider)
+```
+
+## Before/After Code Comparison
+
+### Before: Current Implementation
+
+```python
+# BEFORE: openhands/server/routes/git.py
+@app.get('/repositories', response_model=list[Repository])
+async def get_user_repositories(
+    sort: str = Query(default='pushed'),
+    selected_provider: ProviderType | None = Query(default=None),
+    page: int | None = Query(default=None),
+    per_page: int | None = Query(default=None),
+    installation_id: str | None = Query(default=None),
+    provider_tokens: PROVIDER_TOKEN_TYPE | None = Depends(get_provider_tokens),
+    access_token: SecretStr | None = Depends(get_access_token),
+    user_id: str | None = Depends(get_user_id),
+):
+    if provider_tokens:
+        client = ProviderHandler(
+            provider_tokens=provider_tokens,
+            external_auth_token=access_token,
+            external_auth_id=user_id,
+        )
+        # ... complex logic
+```
+
+```python
+# BEFORE: openhands/storage/locations.py
+def get_conversation_dir(sid: str, user_id: str | None = None) -> str:
+    if user_id:
+        return f'users/{user_id}/conversations/{sid}/'
+    else:
+        return f'sessions/{sid}/'
+```
+
+```python
+# BEFORE: openhands/server/user_auth/default_user_auth.py
+class DefaultUserAuth(UserAuth):
+    async def get_user_id(self) -> str | None:
+        return None  # Always None - no multi-tenancy support
+    
+    async def get_provider_tokens(self) -> PROVIDER_TOKEN_TYPE | None:
+        user_secrets = await self.get_user_secrets()
+        if user_secrets is None:
+            return None
+        return user_secrets.provider_tokens
+```
+
+### After: Proposed Implementation
+
+```python
+# AFTER: openhands/server/routes/git.py
+@app.get('/repositories', response_model=list[Repository])
+async def get_user_repositories(
+    sort: str = Query(default='pushed'),
+    selected_provider: ProviderType | None = Query(default=None),
+    page: int | None = Query(default=None),
+    per_page: int | None = Query(default=None),
+    installation_id: str | None = Query(default=None),
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user),
+):
+    client = ProviderHandler(token_provider=token_provider)
+    return await client.get_repositories(
+        sort, server_config.app_mode, selected_provider, page, per_page, installation_id
+    )
+```
+
+```python
+# AFTER: openhands/auth/storage_namespace.py
+@dataclass(frozen=True)
+class StorageNamespace:
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+```
+
+```python
+# AFTER: openhands/server/user_auth/strategy_user_auth.py
+class StrategyUserAuth(UserAuth):
+    def __init__(self, strategy: AuthStrategy, user_context: Optional[UserContext]):
+        self.strategy = strategy
+        self.user_context = user_context
+        self.storage_namespace = StorageNamespace(
+            user_context.storage_namespace if user_context else None
+        )
+    
+    async def get_user_id(self) -> str | None:
+        return self.user_context.user_id if self.user_context else None
+```
+
+## Configuration
+
+### Environment Variables
+
+```bash
+# Authentication Strategy
+OH_AUTH_STRATEGY=none  # Options: none, single_user, multi_user
+
+# Single User Mode Settings
+OH_ENABLE_SU_AUTH=false  # Enable GitHub OAuth in SU mode
+OH_SU_GITHUB_USERNAME=your_username  # Restrict access to specific user
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+
+# Multi User Mode (custom build extension point)
+OH_MU_ADMIN_USERNAME=admin_user
+```
+
+### Configuration Modes
+
+#### 1. None Mode (Current Default)
+```bash
+OH_AUTH_STRATEGY=none
+# No additional config needed
+```
+
+#### 2. Single User - No Auth
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+```
+
+#### 3. Single User - GitHub Auth
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=your_username
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+```
+
+## Implementation Benefits
+
+### 1. Clean Separation of Concerns
+- Auth logic isolated in strategy classes
+- Business logic doesn't need to know about user_id
+- Clear boundaries between auth and core functionality
+
+### 2. Reduced Complexity
+- Eliminates 7 redundant `if user_id` guards
+- Removes provider_tokens dependency injection
+- Simplifies method signatures throughout codebase
+
+### 3. Forward Compatibility
+- custom builds can extend with custom strategies
+- Token refresh/rotation support built-in
+- Multi-tenancy ready without core changes
+
+### 4. Security Improvements
+- Tokens never exposed in route parameters
+- Centralized token management
+- Immutable user context prevents tampering
+
+### 5. Developer Experience
+- Clear configuration options
+- Easy mode switching
+- Consistent patterns across codebase
+
+## Migration Strategy
+
+### Phase 1: Foundation
+1. Introduce auth strategy interfaces
+2. Add UserContext and StorageNamespace
+3. Create TokenProvider abstraction
+4. Update core dependencies
+
+### Phase 2: Strategy Implementation
+1. Implement NoneStrategy (backward compatible)
+2. Implement SingleUserStrategy
+3. Add configuration support
+4. Update UserAuth integration
+
+### Phase 3: Route Migration
+1. Update FastAPI dependencies
+2. Remove provider_tokens dependency injection
+3. Update ProviderHandler integration
+4. Clean up redundant if-guards
+
+### Phase 4: Storage Migration
+1. Replace storage path helpers
+2. Update conversation managers
+3. Migrate event stores
+4. Clean up legacy code
+
+## Testing Strategy
+
+### Unit Tests
+- Strategy implementations
+- UserContext immutability
+- StorageNamespace path generation
+- TokenProvider implementations
+
+### Integration Tests
+- End-to-end auth flows
+- Route authentication
+- Storage partitioning
+- Configuration switching
+
+### Migration Tests
+- Backward compatibility
+- Data migration paths
+- Configuration validation
+
+## Future Extensions
+
+### custom builds Integration Points
+```python
+# custom builds can provide their own strategies
+class custom buildsMultiUserStrategy(AuthStrategy):
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        # Custom custom builds authentication logic
+        pass
+    
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        # custom builds token refresh/rotation
+        return CustomBuildTokenProvider(request)
+```
+
+### Additional Auth Methods
+- SAML/OIDC strategies
+- API key authentication
+- Custom JWT providers
+- Enterprise SSO integration
+
+## Architecture Diagrams
+
+### 1. Overall Auth System Architecture
+
+```mermaid
+graph TB
+    subgraph "FastAPI Application"
+        Routes[API Routes]
+        Deps[FastAPI Dependencies]
+    end
+    
+    subgraph "Auth Layer"
+        AuthStrategy[AuthStrategy Interface]
+        NoneStrategy[NoneStrategy]
+        SUStrategy[SingleUserStrategy]
+        MUStrategy[MultiUserStrategy]
+    end
+    
+    subgraph "Core Abstractions"
+        UserContext[UserContext]
+        TokenProvider[TokenProvider Interface]
+        StorageNamespace[StorageNamespace]
+    end
+    
+    subgraph "Token Providers"
+        DefaultTP[DefaultTokenProvider]
+        SingleUserTP[SingleUserTokenProvider]
+        custom buildsTP[CustomBuildTokenProvider]
+    end
+    
+    subgraph "Storage Layer"
+        SecretsStore[SecretsStore]
+        SettingsStore[SettingsStore]
+        ConversationStore[ConversationStore]
+    end
+    
+    Routes --> Deps
+    Deps --> AuthStrategy
+    AuthStrategy --> UserContext
+    AuthStrategy --> TokenProvider
+    UserContext --> StorageNamespace
+    
+    NoneStrategy --> DefaultTP
+    SUStrategy --> SingleUserTP
+    MUStrategy --> custom buildsTP
+    
+    TokenProvider --> SecretsStore
+    StorageNamespace --> ConversationStore
+    StorageNamespace --> SettingsStore
+```
+
+### 2. Authentication Flow - None Strategy
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant NoneStrategy
+    participant DefaultTP
+    participant SecretsStore
+    
+    Client->>Route: API Request
+    Route->>NoneStrategy: authenticate(request)
+    NoneStrategy->>Route: None (no user context)
+    Route->>NoneStrategy: get_token_provider(request)
+    NoneStrategy->>DefaultTP: create()
+    DefaultTP->>SecretsStore: load secrets.json
+    SecretsStore->>DefaultTP: provider tokens
+    DefaultTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 3. Authentication Flow - Single User Strategy (No Auth)
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant SUStrategy
+    participant UserContext
+    participant SingleUserTP
+    participant SecretsStore
+    
+    Client->>Route: API Request
+    Route->>SUStrategy: authenticate(request)
+    SUStrategy->>UserContext: create virtual user (local)
+    UserContext->>SUStrategy: user context
+    SUStrategy->>Route: UserContext(user_id="local")
+    Route->>SUStrategy: get_token_provider(request)
+    SUStrategy->>SingleUserTP: create(user_context)
+    SingleUserTP->>SecretsStore: load user secrets
+    SecretsStore->>SingleUserTP: provider tokens
+    SingleUserTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 4. Authentication Flow - Single User Strategy (GitHub Auth)
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Route
+    participant SUStrategy
+    participant GitHub
+    participant UserContext
+    participant SingleUserTP
+    participant Database
+    
+    Client->>Route: API Request with JWT cookie
+    Route->>SUStrategy: authenticate(request)
+    SUStrategy->>SUStrategy: extract JWT token
+    SUStrategy->>SUStrategy: validate JWT
+    SUStrategy->>SUStrategy: check allowed user
+    SUStrategy->>UserContext: create from JWT data
+    UserContext->>SUStrategy: user context
+    SUStrategy->>Route: UserContext
+    Route->>SUStrategy: get_token_provider(request)
+    SUStrategy->>SingleUserTP: create(user_context)
+    SingleUserTP->>Database: load encrypted tokens
+    Database->>SingleUserTP: encrypted provider tokens
+    SingleUserTP->>Route: token provider
+    Route->>Client: API Response
+```
+
+### 5. GitHub OAuth Flow - Single User Strategy
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant AuthRoute
+    participant GitHub
+    participant SUStrategy
+    participant Database
+    participant UserContext
+    
+    Client->>AuthRoute: GET /auth/login
+    AuthRoute->>Client: GitHub OAuth URL
+    Client->>GitHub: OAuth authorization
+    GitHub->>AuthRoute: GET /auth/callback?code=xxx
+    AuthRoute->>GitHub: exchange code for token
+    GitHub->>AuthRoute: access token + user info
+    AuthRoute->>SUStrategy: validate user allowed
+    SUStrategy->>AuthRoute: user authorized
+    AuthRoute->>Database: create/update user record
+    Database->>AuthRoute: user saved
+    AuthRoute->>AuthRoute: create JWT token
+    AuthRoute->>Client: Set JWT cookie + redirect
+```
+
+### 6. Storage Namespace Architecture
+
+```mermaid
+graph TB
+    subgraph "User Context"
+        UC[UserContext]
+        UC --> SN[StorageNamespace]
+    end
+    
+    subgraph "Storage Paths"
+        SN --> ConvDir[get_conversation_dir]
+        SN --> EventsDir[get_events_dir]
+        SN --> MetaFile[get_metadata_file]
+        SN --> StateFile[get_state_file]
+    end
+    
+    subgraph "Path Examples"
+        ConvDir --> NonePath[sessions/sid/]
+        ConvDir --> UserPath[users/user_id/conversations/sid/]
+        
+        EventsDir --> NoneEvents[sessions/sid/events/]
+        EventsDir --> UserEvents[users/user_id/conversations/sid/events/]
+    end
+    
+    subgraph "Strategy Impact"
+        NoneStrategy2[NoneStrategy] --> NonePath
+        SUStrategy2[SingleUserStrategy] --> UserPath
+        MUStrategy2[MultiUserStrategy] --> UserPath
+    end
+```
+
+### 7. Token Provider Architecture
+
+```mermaid
+graph TB
+    subgraph "Token Provider Interface"
+        TP[TokenProvider]
+        TP --> GetToken[get_token(provider)]
+        TP --> GetAllTokens[get_all_tokens()]
+    end
+    
+    subgraph "Implementations"
+        DefaultTP2[DefaultTokenProvider]
+        SingleUserTP2[SingleUserTokenProvider]
+        custom buildsTP2[CustomBuildTokenProvider]
+    end
+    
+    subgraph "Token Sources"
+        SecretsJSON[secrets.json]
+        UserDB[User Database]
+        Custom Build API[custom builds Token API]
+    end
+    
+    subgraph "Provider Integration"
+        ProviderHandler[ProviderHandler]
+        GitHubService[GitHubService]
+        GitLabService[GitLabService]
+        BitBucketService[BitBucketService]
+    end
+    
+    DefaultTP2 --> SecretsJSON
+    SingleUserTP2 --> UserDB
+    custom buildsTP2 --> Custom Build API
+    
+    TP --> ProviderHandler
+    ProviderHandler --> GitHubService
+    ProviderHandler --> GitLabService
+    ProviderHandler --> BitBucketService
+```
+
+### 8. Configuration-Driven Strategy Selection
+
+```mermaid
+graph TB
+    subgraph "Configuration"
+        Config[OH_AUTH_STRATEGY]
+        Config --> None[none]
+        Config --> SU[single_user]
+        Config --> MU[multi_user]
+    end
+    
+    subgraph "Strategy Factory"
+        Factory[AuthStrategyFactory]
+        Factory --> CreateNone[create NoneStrategy]
+        Factory --> CreateSU[create SingleUserStrategy]
+        Factory --> CreateMU[create MultiUserStrategy]
+    end
+    
+    subgraph "Additional Config"
+        SUConfig[OH_ENABLE_SU_AUTH<br/>OH_SU_GITHUB_USERNAME<br/>OH_GITHUB_CLIENT_ID]
+        MUConfig[OH_MU_ADMIN_USERNAME<br/>Database Config]
+    end
+    
+    None --> CreateNone
+    SU --> CreateSU
+    MU --> CreateMU
+    
+    CreateSU --> SUConfig
+    CreateMU --> MUConfig
+```
+
+## Conclusion
+
+This AuthSystem design provides OpenHands with a robust, extensible authentication foundation that:
+
+1. **Maintains backward compatibility** with the current "None" mode
+2. **Enables Single User mode** with optional GitHub OAuth
+3. **Provides extension points** for custom builds with multi-user implementations
+4. **Cleans up the codebase** by removing scattered user_id threading
+5. **Improves security** by centralizing token management
+6. **Simplifies development** with clear abstractions and patterns
+
+The design is ready for implementation and will significantly improve OpenHands' authentication capabilities while maintaining its current simplicity for users who don't need authentication.

docs/usage/architecture/auth-system-summary.md

@@ -0,0 +1,210 @@
+# OpenHands AuthSystem Design - Executive Summary
+
+## Goal
+Design a flexible authentication system for OpenHands that supports three strategies:
+- **None**: Current behavior (no auth, optional GitHub token)
+- **SU (Single User)**: GitHub OAuth for personal use
+- **MU (Multi User)**: Extension point for custom builds (not in base OH)
+
+## Current Problems
+- 339+ `user_id` occurrences scattered across 68 files
+- No auth strategy abstraction
+- `provider_tokens` dependency injection complexity
+- No single-user GitHub OAuth support
+- Mixed auth/business logic concerns
+
+## Solution Architecture
+
+### Core Components
+1. **AuthStrategy Interface** - Pluggable auth strategies
+2. **UserContext** - Immutable user data container
+3. **TokenProvider** - Centralized token management
+4. **StorageNamespace** - Clean storage path abstraction
+
+### Auth Strategies
+```python
+# None Strategy (current behavior)
+OH_AUTH_STRATEGY=none
+
+# Single User - No Auth (virtual user)
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+
+# Single User - GitHub OAuth
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=your_username
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+```
+
+## 🔄 Key Changes
+
+### Before (Current)
+```python
+# Route with complex dependencies
+async def get_repositories(
+    provider_tokens: PROVIDER_TOKEN_TYPE = Depends(get_provider_tokens),
+    user_id: str | None = Depends(get_user_id),
+):
+    client = ProviderHandler(
+        provider_tokens=provider_tokens,
+        external_auth_id=user_id,
+    )
+
+# Scattered path logic
+def get_conversation_dir(sid: str, user_id: str | None = None) -> str:
+    if user_id:
+        return f'users/{user_id}/conversations/{sid}/'
+    else:
+        return f'sessions/{sid}/'
+```
+
+### After (Proposed)
+```python
+# Clean route signature
+async def get_repositories(
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user),
+):
+    client = ProviderHandler(token_provider=token_provider)
+
+# Encapsulated storage logic
+@dataclass(frozen=True)
+class StorageNamespace:
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+```
+
+## Architectural Benefits
+
+### Codebase Cleanup
+- Removes 7 redundant `if user_id` guards across the codebase
+- Eliminates `provider_tokens` dependency injection complexity
+- Reduces method signature complexity throughout the system
+- Centralizes storage path logic in dedicated abstractions
+
+### Extensibility
+- Strategy pattern enables custom build extension points
+- Token refresh/rotation patterns built-in
+- Multi-tenancy ready without core changes
+- Additional auth methods can be added without refactoring
+
+### Code Organization
+- Clear separation of auth and business logic
+- Consistent patterns across all authentication modes
+- Centralized token and credential management
+- Immutable user context prevents state corruption
+
+## Implementation Plan
+
+### Phase 1: Foundation
+- [ ] Auth strategy interfaces
+- [ ] UserContext & StorageNamespace
+- [ ] TokenProvider abstraction
+- [ ] Core dependencies
+
+### Phase 2: Strategies
+- [ ] NoneStrategy (backward compatible)
+- [ ] SingleUserStrategy
+- [ ] Configuration support
+- [ ] UserAuth integration
+
+### Phase 3: Routes
+- [ ] Update FastAPI dependencies
+- [ ] Remove provider_tokens
+- [ ] Update ProviderHandler
+- [ ] Clean redundant guards
+
+### Phase 4: Storage
+- [ ] Replace path helpers
+- [ ] Update conversation managers
+- [ ] Migrate event stores
+- [ ] Legacy cleanup
+
+## Architecture Highlights
+
+### Strategy Pattern
+```python
+class AuthStrategy(ABC):
+    @abstractmethod
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        pass
+    
+    @abstractmethod
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        pass
+```
+
+### Immutable User Context
+```python
+@dataclass(frozen=True)
+class UserContext:
+    user_id: str
+    email: Optional[str] = None
+    username: Optional[str] = None
+    is_admin: bool = False
+```
+
+### Token Provider Interface
+```python
+class TokenProvider(ABC):
+    @abstractmethod
+    async def get_token(self, provider: ProviderType) -> Optional[ProviderToken]:
+        pass
+```
+
+## 🔧 Configuration Examples
+
+### Current Default (None)
+```bash
+# No configuration needed - maintains current behavior
+```
+
+### Personal Use (SU without auth)
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+# Creates virtual "local" user, uses secrets.json
+```
+
+### Personal Use (SU with GitHub)
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=myusername
+OH_GITHUB_CLIENT_ID=abc123
+OH_GITHUB_CLIENT_SECRET=secret456
+# Requires GitHub OAuth, restricts to specific user
+```
+
+## Implementation Readiness
+
+### Backward Compatibility
+- None strategy maintains exact current behavior
+- No breaking changes for existing users
+- Gradual migration path available
+
+### Code Quality Improvements
+- Reduces complexity from 339 to ~50 user_id references
+- Introduces clear abstractions and boundaries
+- Enables better testing and maintainability
+
+### Extensibility Foundation
+- Custom builds can add authentication strategies
+- Token refresh/rotation patterns built-in
+- Multi-tenancy foundation without core changes
+
+## Summary
+
+This design provides a clean authentication architecture for OpenHands with three key outcomes:
+
+1. **Maintains simplicity** - Current users see no changes
+2. **Enables extension** - Custom builds can add authentication features
+3. **Improves codebase** - Reduces scattered auth logic and complexity
+
+The architecture is well-defined with a clear migration path.

docs/usage/architecture/auth-system-summary.mdx

@@ -0,0 +1,214 @@
+---
+title: AuthSystem Design - Executive Summary
+---
+
+# OpenHands AuthSystem Design - Executive Summary
+
+## Goal
+Design a flexible authentication system for OpenHands that supports three strategies:
+- **None**: Current behavior (no auth, optional GitHub token)
+- **SU (Single User)**: GitHub OAuth for personal use
+- **MU (Multi User)**: Extension point for custom builds (not in base OH)
+
+## Current Problems
+- 339+ `user_id` occurrences scattered across 68 files
+- No auth strategy abstraction
+- `provider_tokens` dependency injection complexity
+- No single-user GitHub OAuth support
+- Mixed auth/business logic concerns
+
+## Solution Architecture
+
+### Core Components
+1. **AuthStrategy Interface** - Pluggable auth strategies
+2. **UserContext** - Immutable user data container
+3. **TokenProvider** - Centralized token management
+4. **StorageNamespace** - Clean storage path abstraction
+
+### Auth Strategies
+```python
+# None Strategy (current behavior)
+OH_AUTH_STRATEGY=none
+
+# Single User - No Auth (virtual user)
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+
+# Single User - GitHub OAuth
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=your_username
+OH_GITHUB_CLIENT_ID=your_client_id
+OH_GITHUB_CLIENT_SECRET=your_client_secret
+```
+
+## 🔄 Key Changes
+
+### Before (Current)
+```python
+# Route with complex dependencies
+async def get_repositories(
+    provider_tokens: PROVIDER_TOKEN_TYPE = Depends(get_provider_tokens),
+    user_id: str | None = Depends(get_user_id),
+):
+    client = ProviderHandler(
+        provider_tokens=provider_tokens,
+        external_auth_id=user_id,
+    )
+
+# Scattered path logic
+def get_conversation_dir(sid: str, user_id: str | None = None) -> str:
+    if user_id:
+        return f'users/{user_id}/conversations/{sid}/'
+    else:
+        return f'sessions/{sid}/'
+```
+
+### After (Proposed)
+```python
+# Clean route signature
+async def get_repositories(
+    token_provider: TokenProvider = Depends(get_token_provider),
+    user: Optional[UserContext] = Depends(get_current_user),
+):
+    client = ProviderHandler(token_provider=token_provider)
+
+# Encapsulated storage logic
+@dataclass(frozen=True)
+class StorageNamespace:
+    namespace: Optional[str]
+    
+    def get_conversation_dir(self, sid: str) -> str:
+        if self.namespace:
+            return f'users/{self.namespace}/conversations/{sid}/'
+        return f'sessions/{sid}/'
+```
+
+## Architectural Benefits
+
+### Codebase Cleanup
+- Removes 7 redundant `if user_id` guards across the codebase
+- Eliminates `provider_tokens` dependency injection complexity
+- Reduces method signature complexity throughout the system
+- Centralizes storage path logic in dedicated abstractions
+
+### Extensibility
+- Strategy pattern enables custom build extension points
+- Token refresh/rotation patterns built-in
+- Multi-tenancy ready without core changes
+- Additional auth methods can be added without refactoring
+
+### Code Organization
+- Clear separation of auth and business logic
+- Consistent patterns across all authentication modes
+- Centralized token and credential management
+- Immutable user context prevents state corruption
+
+## Implementation Plan
+
+### Phase 1: Foundation
+- [ ] Auth strategy interfaces
+- [ ] UserContext & StorageNamespace
+- [ ] TokenProvider abstraction
+- [ ] Core dependencies
+
+### Phase 2: Strategies
+- [ ] NoneStrategy (backward compatible)
+- [ ] SingleUserStrategy
+- [ ] Configuration support
+- [ ] UserAuth integration
+
+### Phase 3: Routes
+- [ ] Update FastAPI dependencies
+- [ ] Remove provider_tokens
+- [ ] Update ProviderHandler
+- [ ] Clean redundant guards
+
+### Phase 4: Storage
+- [ ] Replace path helpers
+- [ ] Update conversation managers
+- [ ] Migrate event stores
+- [ ] Legacy cleanup
+
+## Architecture Highlights
+
+### Strategy Pattern
+```python
+class AuthStrategy(ABC):
+    @abstractmethod
+    async def authenticate(self, request: Request) -> Optional[UserContext]:
+        pass
+    
+    @abstractmethod
+    async def get_token_provider(self, request: Request) -> TokenProvider:
+        pass
+```
+
+### Immutable User Context
+```python
+@dataclass(frozen=True)
+class UserContext:
+    user_id: str
+    email: Optional[str] = None
+    username: Optional[str] = None
+    is_admin: bool = False
+```
+
+### Token Provider Interface
+```python
+class TokenProvider(ABC):
+    @abstractmethod
+    async def get_token(self, provider: ProviderType) -> Optional[ProviderToken]:
+        pass
+```
+
+## 🔧 Configuration Examples
+
+### Current Default (None)
+```bash
+# No configuration needed - maintains current behavior
+```
+
+### Personal Use (SU without auth)
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=false
+# Creates virtual "local" user, uses secrets.json
+```
+
+### Personal Use (SU with GitHub)
+```bash
+OH_AUTH_STRATEGY=single_user
+OH_ENABLE_SU_AUTH=true
+OH_SU_GITHUB_USERNAME=myusername
+OH_GITHUB_CLIENT_ID=abc123
+OH_GITHUB_CLIENT_SECRET=secret456
+# Requires GitHub OAuth, restricts to specific user
+```
+
+## Implementation Readiness
+
+### Backward Compatibility
+- None strategy maintains exact current behavior
+- No breaking changes for existing users
+- Gradual migration path available
+
+### Code Quality Improvements
+- Reduces complexity from 339 to ~50 user_id references
+- Introduces clear abstractions and boundaries
+- Enables better testing and maintainability
+
+### Extensibility Foundation
+- Custom builds can add authentication strategies
+- Token refresh/rotation patterns built-in
+- Multi-tenancy foundation without core changes
+
+## Summary
+
+This design provides a clean authentication architecture for OpenHands with three key outcomes:
+
+1. **Maintains simplicity** - Current users see no changes
+2. **Enables extension** - Custom builds can add authentication features
+3. **Improves codebase** - Reduces scattered auth logic and complexity
+
+The architecture is well-defined with a clear migration path.

docs/usage/architecture/backend.mdx

@@ -2,55 +2,102 @@
 title: Backend Architecture
 ---
 
-<div style={{ textAlign: 'center' }}>
-  <img src="https://github.com/All-Hands-AI/OpenHands/assets/16201837/97d747e3-29d8-4ccb-8d34-6ad1adb17f38" alt="OpenHands System Architecture Diagram Jul 4 2024" />
-  <p><em>OpenHands System Architecture Diagram (July 4, 2024)</em></p>
-</div>
 
 This is a high-level overview of the system architecture. The system is divided into two main components: the frontend and the backend. The frontend is responsible for handling user interactions and displaying the results. The backend is responsible for handling the business logic and executing the agents.
 
-# Frontend architecture
+# System overview
 
-![system_architecture.svg](/static/img/system_architecture.svg)
+```mermaid
+flowchart LR
+  U["User"] --> FE["Frontend (SPA)"]
+  FE -- "HTTP/WS" --> BE["OpenHands Backend"]
+  BE --> ES["EventStream"]
+  BE --> ST["Storage"]
+  BE --> RT["Runtime Interface"]
+  BE --> LLM["LLM Providers"]
+
+  subgraph Runtime
+    direction TB
+    RT --> DRT["Docker Runtime"]
+    RT --> LRT["Local Runtime"]
+    RT --> RRT["Remote Runtime"]
+    DRT --> AES["Action Execution Server"]
+    LRT --> AES
+    RRT --> AES
+    AES --> Bash["Bash Session"]
+    AES --> Jupyter["Jupyter Plugin"]
+    AES --> Browser["BrowserEnv"]
+  end
+```
 
 This Overview is simplified to show the main components and their interactions. For a more detailed view of the backend architecture, see the Backend Architecture section below.
 
 # Backend Architecture
 
-_**Disclaimer**: The backend architecture is a work in progress and is subject to change. The following diagram shows the current architecture of the backend based on the commit that is shown in the footer of the diagram._
 
-![backend_architecture.svg](/static/img/backend_architecture.svg)
+```mermaid
+classDiagram
+  class Agent {
+    <<abstract>>
+    +sandbox_plugins: list[PluginRequirement]
+  }
+  class CodeActAgent {
+    +tools
+  }
+  Agent <|-- CodeActAgent
+
+  class EventStream
+  class Observation
+  class Action
+  Action --> Observation
+  Agent --> EventStream
+
+  class Runtime {
+    +connect()
+    +send_action_for_execution()
+  }
+  class ActionExecutionClient {
+    +_send_action_server_request()
+  }
+  class DockerRuntime
+  class LocalRuntime
+  class RemoteRuntime
+  Runtime <|-- ActionExecutionClient
+  ActionExecutionClient <|-- DockerRuntime
+  ActionExecutionClient <|-- LocalRuntime
+  ActionExecutionClient <|-- RemoteRuntime
+
+  class ActionExecutionServer {
+    +/execute_action
+    +/alive
+  }
+  class BashSession
+  class JupyterPlugin
+  class BrowserEnv
+  ActionExecutionServer --> BashSession
+  ActionExecutionServer --> JupyterPlugin
+  ActionExecutionServer --> BrowserEnv
+
+  Agent --> Runtime
+  Runtime ..> ActionExecutionServer : REST
+```
 
 <details>
   <summary>Updating this Diagram</summary>
   <div>
-    The generation of the backend architecture diagram is partially automated.
-    The diagram is generated from the type hints in the code using the py2puml
-    tool. The diagram is then manually reviewed, adjusted and exported to PNG
-    and SVG.
+    We maintain architecture diagrams inline with Mermaid in this MDX.
 
-    ## Prerequisites
-
-    - Running python environment in which openhands is executable
-    (according to the instructions in the README.md file in the root of the repository)
-    - [py2puml](https://github.com/lucsorel/py2puml) installed
-
-## Steps
-
-1.  Autogenerate the diagram by running the following command from the root of the repository:
-    `py2puml openhands openhands > docs/architecture/backend_architecture.puml`
-
-2.  Open the generated file in a PlantUML editor, e.g. Visual Studio Code with the PlantUML extension or [PlantText](https://www.planttext.com/)
-
-3.  Review the generated PUML and make all necessary adjustments to the diagram (add missing parts, fix mistakes, improve positioning).
-    _py2puml creates the diagram based on the type hints in the code, so missing or incorrect type hints may result in an incomplete or incorrect diagram._
-
-4.  Review the diff between the new and the previous diagram and manually check if the changes are correct.
-    _Make sure not to remove parts that were manually added to the diagram in the past and are still relevant._
-
-5.  Add the commit hash of the commit that was used to generate the diagram to the diagram footer.
-
-6.  Export the diagram as PNG and SVG files and replace the existing diagrams in the `docs/architecture` directory. This can be done with (e.g. [PlantText](https://www.planttext.com/))
+    Guidance:
+    - Edit the Mermaid blocks directly (flowchart/classDiagram).
+    - Quote labels and edge text for GitHub preview compatibility.
+    - Keep relationships concise and reflect stable abstractions (agents, runtime client/server, plugins).
+    - Verify accuracy against code:
+      - openhands/runtime/impl/action_execution/action_execution_client.py
+      - openhands/runtime/impl/docker/docker_runtime.py
+      - openhands/runtime/impl/local/local_runtime.py
+      - openhands/runtime/action_execution_server.py
+      - openhands/runtime/plugins/*
+    - Build docs locally or view on GitHub to confirm diagrams render.
 
   </div>
 </details>

docs/usage/architecture/runtime.mdx

@@ -52,7 +52,7 @@ graph TD
 2. Image Building: OpenHands builds a new Docker image (the "OH runtime image") based on the user-provided image. This new image includes OpenHands-specific code, primarily the "runtime client"
 3. Container Launch: When OpenHands starts, it launches a Docker container using the OH runtime image
 4. Action Execution Server Initialization: The action execution server initializes an `ActionExecutor` inside the container, setting up necessary components like a bash shell and loading any specified plugins
-5. Communication: The OpenHands backend (`openhands/runtime/impl/eventstream/eventstream_runtime.py`) communicates with the action execution server over RESTful API, sending actions and receiving observations
+5. Communication: The OpenHands backend (client: `openhands/runtime/impl/action_execution/action_execution_client.py`; runtimes: `openhands/runtime/impl/docker/docker_runtime.py`, `openhands/runtime/impl/local/local_runtime.py`) communicates with the action execution server over RESTful API, sending actions and receiving observations
 6. Action Execution: The runtime client receives actions from the backend, executes them in the sandboxed environment, and sends back observations
 7. Observation Return: The action execution server sends execution results back to the OpenHands backend as observations
 
@@ -72,7 +72,7 @@ Check out the [relevant code](https://github.com/All-Hands-AI/OpenHands/blob/mai
 ### Image Tagging System
 
 OpenHands uses a three-tag system for its runtime images to balance reproducibility with flexibility.
-Tags may be in one of 2 formats:
+The tags are:
 
 - **Versioned Tag**: `oh_v{openhands_version}_{base_image}` (e.g.: `oh_v0.9.9_nikolaik_s_python-nodejs_t_python3.12-nodejs22`)
 - **Lock Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`)
@@ -119,18 +119,52 @@ This tagging approach allows OpenHands to efficiently manage both development an
 2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images)
 3. The **lock** tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image, dependency, and OpenHands version combination
 
+## Volume mounts: named volumes and overlay
+
+OpenHands supports both bind mounts and Docker named volumes in SandboxConfig.volumes:
+
+- Bind mount: "/abs/host/path:/container/path[:mode]"
+- Named volume: "volume:<name>:/container/path[:mode]" or any non-absolute host spec treated as a named volume
+
+Overlay mode (copy-on-write layer) is supported for bind mounts by appending ":overlay" to the mode (e.g., ":ro,overlay").
+To enable overlay COW, set SANDBOX_VOLUME_OVERLAYS to a writable host directory; per-container upper/work dirs are created under it. If SANDBOX_VOLUME_OVERLAYS is unset, overlay mounts are skipped.
+
+Implementation references:
+- openhands/runtime/impl/docker/docker_runtime.py (named volumes in _build_docker_run_args; overlay mounts in _process_overlay_mounts)
+- openhands/core/config/sandbox_config.py (volumes field)
+
+
 ## Runtime Plugin System
 
-The OpenHands Runtime supports a plugin system that allows for extending functionality and customizing the runtime environment. Plugins are initialized when the runtime client starts up.
+The OpenHands Runtime supports a plugin system that allows for extending functionality and customizing the runtime environment. Plugins are initialized when the action execution server starts up inside the runtime.
 
-Check [an example of Jupyter plugin here](https://github.com/All-Hands-AI/OpenHands/blob/ecf4aed28b0cf7c18d4d8ff554883ba182fc6bdd/openhands/runtime/plugins/jupyter/__init__.py#L21-L55) if you want to implement your own plugin.
+## Ports and URLs
 
-*More details about the Plugin system are still under construction - contributions are welcomed!*
+- Host port allocation uses file-locked ranges for stability and concurrency:
+  - Main runtime port: find_available_port_with_lock on configured range
+  - VSCode port: SandboxConfig.sandbox.vscode_port if provided, else find_available_port_with_lock in VSCODE_PORT_RANGE
+  - App ports: two additional ranges for plugin/web apps
+- DOCKER_HOST_ADDR (if set) adjusts how URLs are formed for LocalRuntime/Docker environments.
+- VSCode URL is exposed with a connection token from the action execution server endpoint /vscode/connection_token and rendered as:
+  - Docker/Local: http://localhost:{port}/?tkn={token}&folder={workspace_mount_path_in_sandbox}
+  - RemoteRuntime: scheme://vscode-{host}/?tkn={token}&folder={workspace_mount_path_in_sandbox}
+
+References:
+- openhands/runtime/impl/docker/docker_runtime.py (port ranges, locking, DOCKER_HOST_ADDR, vscode_url)
+- openhands/runtime/impl/local/local_runtime.py (vscode_url factory)
+- openhands/runtime/impl/remote/remote_runtime.py (vscode_url mapping)
+- openhands/runtime/action_execution_server.py (/vscode/connection_token)
+
+
+Examples:
+- Jupyter: openhands/runtime/plugins/jupyter/__init__.py (JupyterPlugin, Kernel Gateway)
+- VS Code: openhands/runtime/plugins/vscode/* (VSCodePlugin, exposes tokenized URL)
+- Agent Skills: openhands/runtime/plugins/agent_skills/*
 
 Key aspects of the plugin system:
 
 1. Plugin Definition: Plugins are defined as Python classes that inherit from a base `Plugin` class
-2. Plugin Registration: Available plugins are registered in an `ALL_PLUGINS` dictionary
+2. Plugin Registration: Available plugins are registered in `openhands/runtime/plugins/__init__.py` via `ALL_PLUGINS`
 3. Plugin Specification: Plugins are associated with `Agent.sandbox_plugins: list[PluginRequirement]`. Users can specify which plugins to load when initializing the runtime
-4. Initialization: Plugins are initialized asynchronously when the runtime client starts
-5. Usage: The runtime client can use initialized plugins to extend its capabilities (e.g., the JupyterPlugin for running IPython cells)
+4. Initialization: Plugins are initialized asynchronously when the runtime starts and are accessible to actions
+5. Usage: Plugins extend capabilities (e.g., Jupyter for IPython cells); the server exposes any web endpoints (ports) via host port mapping

docs/usage/cloud/project-management/jira-dc-integration.mdx

@@ -1,5 +1,5 @@
 ---
-title: Jira Data Center Integration (Beta)
+title: Jira Data Center Integration (Coming soon...)
 description: Complete guide for setting up Jira Data Center integration with OpenHands Cloud, including service account creation, personal access token generation, webhook configuration, and workspace integration setup.
 ---
 
@@ -78,6 +78,14 @@ description: Complete guide for setting up Jira Data Center integration with Ope
          - **Service Account API Key**: The personal access token from Step 2 above
          - Ensure **Active** toggle is enabled
 
+<Note>
+Workspace name is the host name of your Jira Data Center instance.
+
+Eg: http://jira.all-hands.dev/projects/OH/issues/OH-77
+
+Here the workspace name is **jira.all-hands.dev**.
+</Note>
+
 3. **Complete OAuth Flow**
    - You'll be redirected to Jira Data Center to complete OAuth verification
    - Grant the necessary permissions to verify your workspace access. If you have access to multiple workspaces, select the correct one that you initially provided
@@ -101,18 +109,18 @@ description: Complete guide for setting up Jira Data Center integration with Ope
 
 <AccordionGroup>
 <Accordion title="Workspace link flow">
-![workspace-link.png](/static/img/workspace-link.png)
+![workspace-link.png](/static/img/jira-dc-user-link.png)
 </Accordion>
 
 <Accordion title="Workspace Configure flow">
-![workspace-link.png](/static/img/workspace-configure.png)
+![workspace-link.png](/static/img/jira-dc-admin-configure.png)
 </Accordion>
 
 <Accordion title="Edit view as a user">
-![workspace-link.png](/static/img/workspace-user-edit.png)
+![workspace-link.png](/static/img/jira-dc-user-unlink.png)
 </Accordion>
 
 <Accordion title="Edit view as the workspace creator">
-![workspace-link.png](/static/img/workspace-admin-edit.png)
+![workspace-link.png](/static/img/jira-dc-admin-edit.png)
 </Accordion>
 </AccordionGroup>

docs/usage/cloud/project-management/jira-integration.mdx

@@ -1,5 +1,5 @@
 ---
-title: Jira Cloud Integration
+title: Jira Cloud Integration (Coming soon...)
 description: Complete guide for setting up Jira Cloud integration with OpenHands Cloud, including service account creation, API token generation, webhook configuration, and workspace integration setup.
 ---
 
@@ -15,28 +15,27 @@ description: Complete guide for setting up Jira Cloud integration with OpenHands
    - Go to **Directory** > **Users**
 
 2. **Create OpenHands Service Account**
-   - Click **Add user**
-   - Email: `openhands@yourcompany.com` (replace with your preferred service account email)
-   - Display name: `OpenHands Agent`
-   - Send invitation: **No** (you'll set password manually)
-   - Click **Add user**
-
-3. **Configure Account**
-   - Locate the created user and click on it
-   - Set a secure password
-   - Add to relevant Jira projects with appropriate permissions
+   - Click **Service accounts**
+   - Click **Create a service account**
+   - Name: `OpenHands Agent`
+   - Click **Next**
+   - Select **User** role for Jira app
+   - Click **Create**
 
 ### Step 2: Generate API Token
 
-1. **Access API Token Management**
-   - Log in as the OpenHands service account
-   - Go to [API Tokens](https://id.atlassian.com/manage-profile/security/api-tokens)
-
-2. **Create API Token**
+1. **Access Service Account Configuration**
+   - Locate the created service account from above step and click on it
    - Click **Create API token**
-   - Label: `OpenHands Cloud Integration`
-   - Expiry: Set appropriate expiration (recommend 1 year)
-   - Click **Create**
+   - Set the expiry to 365 days (maximum allowed value)
+   - Click **Next**
+   - In **Select token scopes** screen, filter by following values
+      - App: Jira
+      - Scope type: Classic
+      - Scope actions: Write, Read
+   - Select `read:jira-work` and `write:jira-work` scopes
+   - Click **Next**
+   - Review and create API token
    - **Important**: Copy and securely store the token immediately
 
 ### Step 3: Configure Webhook
@@ -83,6 +82,14 @@ description: Complete guide for setting up Jira Cloud integration with OpenHands
          - **Service Account API Key**: The API token from Step 2 above
          - Ensure **Active** toggle is enabled
 
+<Note>
+Workspace name is the host name when accessing a resource in Jira Cloud.
+
+Eg: https://all-hands.atlassian.net/browse/OH-55
+
+Here the workspace name is **all-hands**.
+</Note>
+
 3. **Complete OAuth Flow**
    - You'll be redirected to Jira Cloud to complete OAuth verification
    - Grant the necessary permissions to verify your workspace access.
@@ -106,18 +113,18 @@ description: Complete guide for setting up Jira Cloud integration with OpenHands
 
 <AccordionGroup>
 <Accordion title="Workspace link flow">
-![workspace-link.png](/static/img/workspace-link.png)
+![workspace-link.png](/static/img/jira-user-link.png)
 </Accordion>
 
 <Accordion title="Workspace Configure flow">
-![workspace-link.png](/static/img/workspace-configure.png)
+![workspace-link.png](/static/img/jira-admin-configure.png)
 </Accordion>
 
 <Accordion title="Edit view as a user">
-![workspace-link.png](/static/img/workspace-user-edit.png)
+![workspace-link.png](/static/img/jira-user-unlink.png)
 </Accordion>
 
 <Accordion title="Edit view as the workspace creator">
-![workspace-link.png](/static/img/workspace-admin-edit.png)
+![workspace-link.png](/static/img/jira-admin-edit.png)
 </Accordion>
 </AccordionGroup>

docs/usage/cloud/project-management/linear-integration.mdx

@@ -1,5 +1,5 @@
 ---
-title: Linear Integration
+title: Linear Integration (Coming soon...)
 description: Complete guide for setting up Linear integration with OpenHands Cloud, including service account creation, API key generation, webhook configuration, and workspace integration setup.
 ---
 
@@ -28,7 +28,7 @@ description: Complete guide for setting up Linear integration with OpenHands Clo
 
 1. **Access API Settings**
    - Log in as the service account
-   - Go to **Settings** > **API**
+   - Go to **Settings** > **Security & access**
 
 2. **Create Personal API Key**
    - Click **Create new key**
@@ -82,6 +82,14 @@ description: Complete guide for setting up Linear integration with OpenHands Clo
          - **Service Account API Key**: The API key from Step 2 above
          - Ensure **Active** toggle is enabled
 
+<Note>
+Workspace name is the identifier after the host name when accessing a resource in Linear.
+
+Eg: https://linear.app/allhands/issue/OH-37
+
+Here the workspace name is **allhands**.
+</Note>
+
 3. **Complete OAuth Flow**
    - You'll be redirected to Linear to complete OAuth verification
    - Grant the necessary permissions to verify your workspace access. If you have access to multiple workspaces, select the correct one that you initially provided
@@ -105,15 +113,15 @@ description: Complete guide for setting up Linear integration with OpenHands Clo
 
 <AccordionGroup>
 <Accordion title="Workspace link flow">
-![workspace-link.png](/static/img/workspace-link.png)
+![workspace-link.png](/static/img/linear-user-link.png)
 </Accordion>
 
 <Accordion title="Workspace Configure flow">
-![workspace-link.png](/static/img/workspace-configure.png)
+![workspace-link.png](/static/img/linear-admin-configure.png)
 </Accordion>
 
 <Accordion title="Edit view as a user">
-![workspace-link.png](/static/img/workspace-user-edit.png)
+![workspace-link.png](/static/img/linear-admin-edit.png)
 </Accordion>
 
 <Accordion title="Edit view as the workspace creator">

docs/usage/cloud/project-management/overview.mdx

@@ -1,5 +1,5 @@
 ---
-title: Project Management Tool Integrations
+title: Project Management Tool Integrations (Coming soon...)
 description: Overview of OpenHands Cloud integrations with project management platforms including Jira Cloud, Jira Data Center, and Linear. Learn about setup requirements, usage methods, and troubleshooting.
 ---
 
@@ -18,9 +18,9 @@ Integration requires two levels of setup:
 2. **Workspace Integration** - Self-service configuration through the OpenHands Cloud UI to link your OpenHands account to the target workspace
 
 ### Platform-Specific Setup Guides:
-- [Jira Cloud Integration](./jira-integration.md)
-- [Jira Data Center Integration](./jira-dc-integration.md)
-- [Linear Integration](./linear-integration.md)
+- [Jira Cloud Integration (Coming soon...)](./jira-integration.md)
+- [Jira Data Center Integration (Coming soon...)](./jira-dc-integration.md)
+- [Linear Integration (Coming soon...)](./linear-integration.md)
 
 ## Usage
 
@@ -58,17 +58,18 @@ The OpenHands agent needs to identify which Git repository to work with when pro
 
 ### Platform Configuration Issues
 - **Webhook not triggering**: Verify the webhook URL is correct and the proper event types are selected (Comment, Issue updated)
-- **API authentication failing**: Check API key/token validity and ensure required scopes are granted
+- **API authentication failing**: Check API key/token validity and ensure required scopes are granted. If your current API token is expired, make sure to update it in the respective integration settings
 - **Permission errors**: Ensure the service account has access to relevant projects/teams and appropriate permissions
 
 ### Workspace Integration Issues
 - **Workspace linking requests credentials**: If there are no active workspace integrations for the workspace you specified, you need to configure it first. Contact your platform administrator that you want to integrate with (eg: Jira, Linear)
-- **OAuth flow fails**: Ensure you're signing in with the same Git provider account that contains the repositories you want OpenHands to work on
 - **Integration not found**: Verify the workspace name matches exactly and that platform configuration was completed first
+- **OAuth flow fails**: Make sure that you're authorizing with the correct account with proper workspace access
 
 ### General Issues
 - **Agent not responding**: Check webhook logs in your platform settings and verify service account status
 - **Authentication errors**: Verify Git provider permissions and OpenHands Cloud access
+- **Agent fails to identify git repo**: Ensure you're signing in with the same Git provider account that contains the repositories you want OpenHands to work on
 - **Partial functionality**: Ensure both platform configuration and workspace integration are properly completed
 
 ### Getting Help

docs/usage/cloud/slack-installation.mdx

@@ -65,7 +65,7 @@ To send follow-up messages for the same conversation, mention `@openhands` in a
 
 Conversation is started by mentioning `@openhands`.
 
-![slack-create-convo.png](/static/img/slack-create-convo.png)
+![slack-create-conversation.png](/static/img/slack-create-conversation.png)
 
 ### See agent response and send follow up messages
 

docs/usage/confirmation-mode.mdx

@@ -0,0 +1,52 @@
+# Confirmation Mode and Security Analyzers
+
+OpenHands provides a security framework to help protect users from potentially risky actions through **Confirmation Mode** and **Security Analyzers**. This system analyzes agent actions and prompts users for confirmation when high-risk operations are detected.
+
+## Overview
+
+The security system consists of two main components:
+
+1. **Confirmation Mode**: When enabled, the agent will pause and ask for user confirmation before executing actions that are flagged as high-risk by the security analyzer.
+
+2. **Security Analyzers**: These are modules that evaluate the risk level of agent actions and determine whether user confirmation is required.
+
+## Configuration
+
+### CLI
+In CLI mode, confirmation is enabled by default. You will have an option to uses the LLM Analyzer and will automatically confirm LOW and MEDIUM risk actions, only prompting for HIGH risk actions.
+
+## Security Analyzers
+
+OpenHands includes multiple analyzers:
+
+- **No Analyzer**: Do not use any security analyzer. The agent will prompt you to confirm *EVERY* action.
+- **LLM Risk Analyzer** (default): Uses the same LLM as the agent to assess action risk levels
+- **Invariant Analyzer**: Uses Invariant Labs' policy engine to evaluate action traces against security policies
+
+### LLM Risk Analyzer
+The default analyzer that leverages the agent's LLM to evaluate the security risk of each action. It considers the action type, parameters, and context to assign risk levels.
+
+### Invariant Analyzer
+An advanced analyzer that:
+- Collects conversation events and parses them into a trace
+- Checks the trace against an Invariant policy to classify risk (low, medium, high)
+- Manages an Invariant server container automatically if needed
+- Supports optional browsing-alignment and harmful-content checks
+
+## How It Works
+
+1. **Action Analysis**: When the agent wants to perform an action, the selected security analyzer evaluates its risk level.
+
+2. **Risk Assessment**: The analyzer returns one of three risk levels:
+   - **LOW**: Action proceeds without confirmation
+   - **MEDIUM**: Action proceeds without confirmation (may be configurable in future)
+   - **HIGH**: Action is paused, and user confirmation is requested
+
+3. **User Confirmation**: For high-risk actions, a confirmation dialog appears with:
+   - Description of the action
+   - Risk assessment explanation
+   - Options to approve or deny action
+
+4. **Action Execution**: Based on user response:
+   - **Approve**: Action proceeds as planned
+   - **Deny**: Action is cancelled

docs/usage/how-to/cli-mode.mdx

@@ -20,27 +20,42 @@ for scripting.
 
 ### Running with Python
 
-**Note** - OpenHands requires Python version 3.12 or higher (Python 3.14 is not currently supported) and `uvx` for the default `fetch` MCP server (more details below).
+**Note** - OpenHands requires Python version 3.12 or higher (Python 3.14 is not currently supported) and `uv` for the default `fetch` MCP server (more details below).
 
-1. Install OpenHands using pip:
-```bash
-pip install openhands-ai
-```
+#### Recommended: Using uv
 
-  Or if you prefer not to manage your own Python environment, you can use `uvx`:
+We recommend using [uv](https://docs.astral.sh/uv/) for the best OpenHands experience. uv provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers.
 
+1. **Install uv** (if you haven't already):
+
+   See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+2. **Launch OpenHands CLI**:
 ```bash
 uvx --python 3.12 --from openhands-ai openhands
 ```
 
 <AccordionGroup>
 
+<Accordion title="Alternative: Traditional pip installation">
+
+If you prefer to use pip:
+
+```bash
+# Install OpenHands
+pip install openhands-ai
+```
+
+Note that you'll still need `uv` installed for the default MCP servers to work properly.
+
+</Accordion>
+
 <Accordion title="Create shell aliases for easy access across environments">
 
 Add the following to your shell configuration file (`.bashrc`, `.zshrc`, etc.):
 
 ```bash
-# Add OpenHands aliases
+# Add OpenHands aliases (recommended)
 alias openhands="uvx --python 3.12 --from openhands-ai openhands"
 alias oh="uvx --python 3.12 --from openhands-ai openhands"
 ```
@@ -72,15 +87,10 @@ source ~/.bashrc  # or source ~/.zshrc
 
 </AccordionGroup>
 
-2. Launch an interactive OpenHands conversation from the command line:
-```bash
-openhands
-```
-
 <Note>
   If you have cloned the repository, you can also run the CLI directly using Poetry:
 
-  poetry run python -m openhands.cli.main
+  poetry run openhands
 </Note>
 
 3. Set your model, API key, and other preferences using the UI (or alternatively environment variables, below).
@@ -103,7 +113,7 @@ The conversation history will be saved in `~/.openhands/sessions`.
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e SANDBOX_VOLUMES=$SANDBOX_VOLUMES \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -112,8 +122,8 @@ docker run -it \
     -v ~/.openhands:/.openhands \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51 \
-    python -m openhands.cli.main --override-cli-mode true
+    docker.all-hands.dev/all-hands-ai/openhands:0.55 \
+    python -m openhands.cli.entry --override-cli-mode true
 ```
 
 <Note>

docs/usage/how-to/headless-mode.mdx

@@ -61,7 +61,7 @@ export GITHUB_TOKEN="your-token"  # Required for repository operations
 # Run OpenHands
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e SANDBOX_VOLUMES=$SANDBOX_VOLUMES \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -73,7 +73,7 @@ docker run -it \
     -v ~/.openhands:/.openhands \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.55 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
 

docs/usage/llms/local-llms.mdx

@@ -68,23 +68,23 @@ Download and install the LM Studio desktop app from [lmstudio.ai](https://lmstud
 1. Check [the installation guide](/usage/local-setup) and ensure all prerequisites are met before running OpenHands, then run:
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51
+    docker.all-hands.dev/all-hands-ai/openhands:0.55
 ```
 
 2. Wait until the server is running (see log below):
 ```
 Digest: sha256:e72f9baecb458aedb9afc2cd5bc935118d1868719e55d50da73190d3a85c674f
-Status: Image is up to date for docker.all-hands.dev/all-hands-ai/openhands:0.51
+Status: Image is up to date for docker.all-hands.dev/all-hands-ai/openhands:0.55
 Starting OpenHands...
 Running OpenHands as root
 14:22:13 - openhands:INFO: server_config.py:50 - Using config class None

docs/usage/local-setup.mdx

@@ -45,6 +45,13 @@ A system with a modern processor and a minimum of **4GB RAM** is recommended to
   1. [Install WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
   2. Run `wsl --version` in powershell and confirm `Default Version: 2`.
 
+  **Ubuntu (Linux Distribution)**
+
+  1. Install Ubuntu: `wsl --install -d Ubuntu` in PowerShell as Administrator.
+  2. Restart computer when prompted.
+  3. Open Ubuntu from Start menu to complete setup.
+  4. Verify installation: `wsl --list` should show Ubuntu.
+
   **Docker Desktop**
 
   1. [Install Docker Desktop on Windows](https://docs.docker.com/desktop/setup/install/windows-install).
@@ -53,7 +60,7 @@ A system with a modern processor and a minimum of **4GB RAM** is recommended to
   - Resources > WSL Integration: `Enable integration with my default WSL distro` is enabled.
 
   <Note>
-  The docker command below to start the app must be run inside the WSL terminal.
+  The docker command below to start the app must be run inside the WSL terminal. Use `wsl -d Ubuntu` in PowerShell or search "Ubuntu" in the Start menu to access the Ubuntu terminal.
   </Note>
 
   **Alternative: Windows without WSL**
@@ -66,9 +73,31 @@ A system with a modern processor and a minimum of **4GB RAM** is recommended to
 
 ### Start the App
 
-#### Option 1: Using the CLI Launcher (Recommended)
+#### Option 1: Using the CLI Launcher with uv (Recommended)
 
-If you have Python 3.12+ installed, you can use the CLI launcher for a simpler experience:
+We recommend using [uv](https://docs.astral.sh/uv/) for the best OpenHands experience. uv provides better isolation from your current project's virtual environment and is required for OpenHands' default MCP servers (like the [fetch MCP server](https://github.com/modelcontextprotocol/servers/tree/main/src/fetch)).
+
+**Install uv** (if you haven't already):
+
+See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for the latest installation instructions for your platform.
+
+**Launch OpenHands**:
+```bash
+# Launch the GUI server
+uvx --python 3.12 --from openhands-ai openhands serve
+
+# Or with GPU support (requires nvidia-docker)
+uvx --python 3.12 --from openhands-ai openhands serve --gpu
+
+# Or with current directory mounted
+uvx --python 3.12 --from openhands-ai openhands serve --mount-cwd
+```
+
+This will automatically handle Docker requirements checking, image pulling, and launching the GUI server. The `--gpu` flag enables GPU support via nvidia-docker, and `--mount-cwd` mounts your current directory into the container.
+
+<Accordion title="Alternative: Traditional pip installation">
+
+If you prefer to use pip and have Python 3.12+ installed:
 
 ```bash
 # Install OpenHands
@@ -76,34 +105,32 @@ pip install openhands-ai
 
 # Launch the GUI server
 openhands serve
-
-# Or with GPU support (requires nvidia-docker)
-openhands serve --gpu
-
-# Or with current directory mounted
-openhands serve --mount-cwd
 ```
 
-Or using `uvx --python 3.12 --from openhands-ai openhands serve` if you have [uv](https://docs.astral.sh/uv/) installed.
+Note that you'll still need `uv` installed for the default MCP servers to work properly.
 
-This will automatically handle Docker requirements checking, image pulling, and launching the GUI server. The `--gpu` flag enables GPU support via nvidia-docker, and `--mount-cwd` mounts your current directory into the container.
+</Accordion>
 
 #### Option 2: Using Docker Directly
 
+<Accordion title="Docker Command (Click to expand)">
+
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.51-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.55-nikolaik \
     -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -v ~/.openhands:/.openhands \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.51
+    docker.all-hands.dev/all-hands-ai/openhands:0.55
 ```
 
+</Accordion>
+
 > **Note**: If you used OpenHands before version 0.44, you may want to run `mv ~/.openhands-state ~/.openhands` to migrate your conversation history to the new location.
 
 You'll find OpenHands running at http://localhost:3000!

docs/usage/runtimes/docker.mdx

@@ -130,3 +130,28 @@ docker run # ... \
 <Note>
 **Docker Desktop Required**: Network isolation features, including custom networks and `host.docker.internal` routing, require Docker Desktop. Docker Engine alone does not support these features on localhost across custom networks. If you're using Docker Engine without Docker Desktop, network isolation may not work as expected.
 </Note>
+
+### Sidecar Containers
+
+If you want to run sidecar containers to the sandbox 'runner' containers without exposing the sandbox containers to the host network, you can use the `SANDBOX_ADDITIONAL_NETWORKS` environment variable to specify additional Docker network names that should be added to the sandbox containers.
+
+```bash
+docker network create openhands-sccache
+
+docker run -d \
+  --hostname openhandsredis \
+  --network openhands-sccache \
+  redis
+
+docker run # ...
+    -e SANDBOX_ADDITIONAL_NETWORKS='["openhands-sccache"]' \
+    # ...
+```
+
+Then all sandbox instances will have to access a shared redis instance at `openhandsredis:6379`.
+
+#### Docker Compose gotcha
+
+Note that Docker Compose adds a prefix (a scope) by default to created networks, which is not taken into account by the additional networks config. Therefore when using docker compose you have to either:
+- specify a network name via the `name` field to remove the scoping (https://docs.docker.com/reference/compose-file/networks/#name) 
+- or provide the scope within the given config (e.g. `SANDBOX_ADDITIONAL_NETWORKS: '["myscope_openhands-sccache"]'` where `myscope` is the docker-compose assigned prefix). 

docs/usage/runtimes/e2b.mdx

@@ -22,7 +22,7 @@ SDK to spawn and control these sandboxes.
 
 You can use the E2B CLI to create a custom sandbox with a Dockerfile. Read the full guide
 [here](https://e2b.dev/docs/guide/custom-sandbox). The premade OpenHands sandbox for E2B is set up in the `containers`
-directory. and it's called `openhands`.
+directory, and it's called `openhands`.
 
 ## Debugging
 

docs/usage/troubleshooting/troubleshooting.mdx

@@ -38,6 +38,23 @@ On initial prompt, an error is seen with `Permission Denied` or `PermissionError
 * If mounting a local directory, ensure your `WORKSPACE_BASE` has the necessary permissions for the user running
   OpenHands.
 
+### On Linux, Getting ConnectTimeout Error
+
+**Description**
+
+When running on Linux, you might run into the error `ERROR:root:<class 'httpx.ConnectTimeout'>: timed out`.
+
+**Resolution**
+
+If you installed Docker from your distribution’s package repository (e.g., docker.io on Debian/Ubuntu), be aware that
+these packages can sometimes be outdated or include changes that cause compatibility issues. try reinstalling Docker
+[using the official instructions](https://docs.docker.com/engine/install/) to ensure you are running a compatible version.
+
+If that does not solve the issue, try incrementally adding the following parameters to the docker run command:
+* `--network host`
+* `-e SANDBOX_USE_HOST_NETWORK=true`
+* `-e DOCKER_HOST_ADDR=127.0.0.1`
+
 ### Internal Server Error. Ports are not available
 
 **Description**

enterprise/LICENSE

@@ -0,0 +1,89 @@
+# PolyForm Free Trial License 1.0.0
+
+## Acceptance
+
+In order to get any license under these terms, you must agree
+to them as both strict obligations and conditions to all
+your licenses.
+
+## Copyright License
+
+The licensor grants you a copyright license for the software
+to do everything you might do with the software that would
+otherwise infringe the licensor's copyright in it for any
+permitted purpose.  However, you may only make changes or
+new works based on the software according to [Changes and New
+Works License](#changes-and-new-works-license), and you may
+not distribute copies of the software.
+
+## Changes and New Works License
+
+The licensor grants you an additional copyright license to
+make changes and new works based on the software for any
+permitted purpose.
+
+## Patent License
+
+The licensor grants you a patent license for the software that
+covers patent claims the licensor can license, or becomes able
+to license, that you would infringe by using the software.
+
+## Fair Use
+
+You may have "fair use" rights for the software under the
+law. These terms do not limit them.
+
+## Free Trial
+
+Use of the software for more than 30 days per calendar year is not allowed without a commercial license.
+
+## No Other Rights
+
+These terms do not allow you to sublicense or transfer any of
+your licenses to anyone else, or prevent the licensor from
+granting licenses to anyone else.  These terms do not imply
+any other licenses.
+
+## Patent Defense
+
+If you make any written claim that the software infringes or
+contributes to infringement of any patent, your patent license
+for the software granted under these terms ends immediately. If
+your company makes such a claim, your patent license ends
+immediately for work on behalf of your company.
+
+## Violations
+
+If you violate any of these terms, or do anything with the
+software not covered by your licenses, all your licenses
+end immediately.
+
+## No Liability
+
+***As far as the law allows, the software comes as is, without
+any warranty or condition, and the licensor will not be liable
+to you for any damages arising out of these terms or the use
+or nature of the software, under any kind of legal claim.***
+
+## Definitions
+
+The **licensor** is the individual or entity offering these
+terms, and the **software** is the software the licensor makes
+available under these terms.
+
+**You** refers to the individual or entity agreeing to these
+terms.
+
+**Your company** is any legal entity, sole proprietorship,
+or other kind of organization that you work for, plus all
+organizations that have control over, are under the control of,
+or are under common control with that organization.  **Control**
+means ownership of substantially all the assets of an entity,
+or the power to direct its management and policies by vote,
+contract, or otherwise.  Control can be direct or indirect.
+
+**Your licenses** are all the licenses granted to you for the
+software under these terms.
+
+**Use** means anything you do with the software requiring one
+of your licenses.

evaluation/benchmarks/EDA/run_infer.py

@@ -9,7 +9,8 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     compatibility_for_eval_history_pairs,
-    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -60,18 +61,15 @@ AGENT_CLS_TO_INST_SUFFIX = {
 def get_config(
     metadata: EvalMetadata,
 ) -> OpenHandsConfig:
-    sandbox_config = get_default_sandbox_config_for_eval()
-    sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    # Create config with EDA-specific container image
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
+
+    # Override the container image for EDA
+    config.sandbox.base_container_image = 'python:3.12-bookworm'
+
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
     agent_config.enable_prompt_extensions = False
@@ -146,7 +144,7 @@ def process_instance(
 
     logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
     test_result = game.reward()
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/agent_bench/run_infer.py

@@ -17,7 +17,8 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     compatibility_for_eval_history_pairs,
-    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -40,19 +41,12 @@ from openhands.utils.async_utils import call_async_from_sync
 def get_config(
     metadata: EvalMetadata,
 ) -> OpenHandsConfig:
-    sandbox_config = get_default_sandbox_config_for_eval()
-    sandbox_config.base_container_image = 'python:3.12-slim'
+    # Create config with agent_bench-specific container image
+    config = get_openhands_config_for_eval(metadata=metadata)
+
+    # Override the container image for agent_bench
+    config.sandbox.base_container_image = 'python:3.12-slim'
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
     agent_config.enable_prompt_extensions = False
@@ -273,7 +267,7 @@ def process_instance(
     # remove when it becomes unnecessary
     histories = compatibility_for_eval_history_pairs(state.history)
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/aider_bench/run_infer.py

@@ -17,6 +17,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -49,15 +51,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.11-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -246,7 +243,7 @@ def process_instance(
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
     histories = compatibility_for_eval_history_pairs(state.history)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/biocoder/run_infer.py

@@ -15,6 +15,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -60,15 +62,10 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = BIOCODER_BENCH_CONTAINER_IMAGE
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -294,7 +291,7 @@ def process_instance(
         raise ValueError('State should not be None.')
 
     test_result = complete_runtime(runtime, instance)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary

evaluation/benchmarks/bird/run_infer.py

@@ -18,6 +18,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -74,15 +76,10 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -422,7 +419,7 @@ def process_instance(
     # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
     if state is None:
         raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/browsing_delegation/run_infer.py

@@ -11,6 +11,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -39,14 +41,8 @@ def get_config(
     )
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        workspace_base=None,
-        workspace_mount_path=None,
+    config = get_openhands_config_for_eval(
+        metadata=metadata, runtime='docker', sandbox_config=sandbox_config
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -88,7 +84,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary

evaluation/benchmarks/commit0/run_infer.py

@@ -16,6 +16,8 @@ from evaluation.utils.shared import (
     assert_and_raise,
     codeact_user_response,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -113,16 +115,11 @@ def get_config(
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = base_container_image
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
-        enable_browser=RUN_WITH_BROWSING,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        enable_browser=RUN_WITH_BROWSING,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -480,7 +477,7 @@ def process_instance(
 
     # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
     histories = [event_to_dict(event) for event in state.history]
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(
@@ -506,7 +503,6 @@ def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
     Returns:
         Filtered dataset based on split type
     """
-
     filtered_dataset = pd.concat(
         [
             dataset[dataset['repo'].str.split('/').str[1] == repo]

evaluation/benchmarks/discoverybench/run_infer.py

@@ -17,6 +17,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -64,15 +66,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -89,8 +86,7 @@ def get_config(
 def get_dv_query_for_real(
     datasets, question, domain_knowledge=None, workflow_tags=None
 ):
-    """
-    Prepare a structured query for the agent to execute on the specified datasets.
+    """Prepare a structured query for the agent to execute on the specified datasets.
 
     This function constructs a query by compiling metadata from the provided datasets, along with any relevant domain knowledge and workflow tags.
 
@@ -104,7 +100,6 @@ def get_dv_query_for_real(
         query_to_dv: Query to be run on the dataset
         dataset_meta: Metadata of the dataset
     """
-
     dataset_meta = ''
     for dataset_metadata in datasets:
         dataset_meta += 'Dataset name: ' + dataset_metadata['name']
@@ -140,8 +135,7 @@ def get_dv_query_for_real(
 
 
 def initialize_runtime(runtime: Runtime, data_files: list[str]):
-    """
-    Initialize the runtime for the agent.
+    """Initialize the runtime for the agent.
 
     This function is called before the runtime is used to run the agent.
     """
@@ -231,8 +225,7 @@ def process_instance(
     metadata: EvalMetadata,
     reset_logger: bool = True,
 ):
-    """
-    Process and evaluate a single instance of the dataset.
+    """Process and evaluate a single instance of the dataset.
 
     This function executes the OpenHands agent
     for a specific instance of the dataset. It retrieves
@@ -247,7 +240,6 @@ def process_instance(
     Returns:
         output: EvalOutput object
     """
-
     config = get_config(metadata)
 
     # Setup the logger properly, so you can run
@@ -299,7 +291,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     test_result = complete_runtime(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
@@ -356,8 +348,7 @@ def list_csv_files(list_of_datasets):
 
 
 def create_dataset(repo_location: str, split: str = 'test'):
-    """
-    Create a dataset from the discoverybench repository
+    """Create a dataset from the discoverybench repository
     by walking through the repository and extracting metadata
     from the metadata_{}.json files
 
@@ -368,7 +359,6 @@ def create_dataset(repo_location: str, split: str = 'test'):
     Returns:
         df: DataFrame containing the dataset instances
     """
-
     data_dict = {}
 
     data_location = os.path.join(repo_location, 'discoverybench', 'real', split)

evaluation/benchmarks/gaia/run_infer.py

@@ -10,7 +10,6 @@ import huggingface_hub
 import pandas as pd
 from datasets import load_dataset
 from PIL import Image
-from pydantic import SecretStr
 
 from evaluation.benchmarks.gaia.scorer import question_scorer
 from evaluation.benchmarks.gaia.utils import (
@@ -23,6 +22,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -60,15 +61,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'nikolaik/python-nodejs:python3.12-nodejs22'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        sandbox_config=sandbox_config,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
     )
     config.set_llm_config(metadata.llm_config)
     if metadata.agent_config:
@@ -80,8 +76,7 @@ def get_config(
 
     config_copy = copy.deepcopy(config)
     load_from_toml(config_copy)
-    if config_copy.search_api_key:
-        config.search_api_key = SecretStr(config_copy.search_api_key)
+    config.search_api_key = config_copy.search_api_key
     return config
 
 
@@ -271,7 +266,7 @@ Here is the task:
         'model_answer': model_answer,
         'ground_truth': instance['Final answer'],
     }
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/gorilla/run_infer.py

@@ -12,6 +12,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -42,15 +44,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -108,7 +105,7 @@ def process_instance(
     # attempt to parse model_answer
     ast_eval_fn = instance['ast_eval']
     correct, hallucination = ast_eval_fn(instance_id, model_answer_raw)
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     logger.info(
         f'Final message: {model_answer_raw} | Correctness: {correct} | Hallucination: {hallucination}'
     )

evaluation/benchmarks/gpqa/run_infer.py

@@ -30,6 +30,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -63,15 +65,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -292,7 +289,7 @@ Ok now its time to start solving the question. Good luck!
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/humanevalfix/run_infer.py

@@ -23,6 +23,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -84,15 +86,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -248,7 +245,7 @@ def process_instance(
 
     if state is None:
         raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     test_result = complete_runtime(runtime, instance)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)

evaluation/benchmarks/lca_ci_build_repair/eval_infer.py

@@ -16,6 +16,7 @@ import ruamel.yaml
 from evaluation.utils.shared import (
     EvalMetadata,
     get_default_sandbox_config_for_eval,
+    get_openhands_config_for_eval,
     make_metadata,
 )
 from openhands.core.config import (
@@ -37,15 +38,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)

evaluation/benchmarks/lca_ci_build_repair/run_infer.py

@@ -22,6 +22,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -47,15 +49,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'python:3.12-bookworm'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -335,7 +332,7 @@ Be thorough in your exploration, testing, and reasoning. It's fine if your think
         )
     )
     assert state is not None
-    metrics = state.metrics.get() if state.metrics else {}
+    metrics = get_metrics(state)
 
     test_result = complete_runtime(runtime, instance)
 

evaluation/benchmarks/logic_reasoning/run_infer.py

@@ -10,6 +10,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -51,15 +53,10 @@ def get_config(
         '$OH_INTERPRETER_PATH -m pip install scitools-pyke'
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -247,7 +244,7 @@ def process_instance(
     )
     test_result['final_message'] = final_message
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary

evaluation/benchmarks/miniwob/run_infer.py

@@ -13,6 +13,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -57,15 +59,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'xingyaoww/od-eval-miniwob:v1.0'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -174,7 +171,7 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # Instruction is the first message from the USER
     instruction = ''

evaluation/benchmarks/mint/run_infer.py

@@ -15,6 +15,8 @@ from evaluation.utils.shared import (
     EvalOutput,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -109,15 +111,10 @@ def get_config(
         f'$OH_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}'
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -205,7 +202,7 @@ def process_instance(
         task_state = state.extra_data['task_state']
         logger.info('Task state: ' + str(task_state.to_dict()))
 
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/ml_bench/run_infer.py

@@ -26,6 +26,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -79,15 +81,10 @@ def get_config(
 ) -> OpenHandsConfig:
     sandbox_config = get_default_sandbox_config_for_eval()
     sandbox_config.base_container_image = 'public.ecr.aws/i5g0m1f6/ml-bench'
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime='docker',
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
@@ -250,7 +247,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
         )
     )
     assert state is not None
-    metrics = state.metrics.get() if state.metrics else {}
+    metrics = get_metrics(state)
 
     test_result = complete_runtime(runtime)
 

evaluation/benchmarks/multi_swe_bench/eval_infer.py

@@ -23,6 +23,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     get_default_sandbox_config_for_eval,
+    get_openhands_config_for_eval,
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
@@ -87,13 +88,9 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
         dataset_name=metadata.dataset,
         instance_id=instance['instance_id'],
     )
-    config = OpenHandsConfig(
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     return config
 
@@ -105,8 +102,7 @@ def process_instance(
     log_dir: str | None = None,
     runtime_failure_count: int = 0,
 ) -> EvalOutput:
-    """
-    Evaluate agent performance on a SWE-bench problem instance.
+    """Evaluate agent performance on a SWE-bench problem instance.
 
     Note that this signature differs from the expected input to `run_evaluation`. Use
     `functools.partial` to provide optional arguments before passing to the evaluation harness.

evaluation/benchmarks/multi_swe_bench/run_infer.py

@@ -21,6 +21,7 @@ from evaluation.utils.shared import (
     codeact_user_response,
     get_default_sandbox_config_for_eval,
     get_metrics,
+    get_openhands_config_for_eval,
     is_fatal_evaluation_error,
     make_metadata,
     prepare_dataset,
@@ -341,16 +342,11 @@ def get_config(
         instance_id=instance['instance_id'],
     )
 
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         enable_browser=RUN_WITH_BROWSING,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(

evaluation/benchmarks/nocode_bench/README.md

@@ -0,0 +1,45 @@
+# Evaluate OpenHands on NoCode-bench
+
+## LLM Setup
+
+Please follow [here](../../README.md#setup).
+
+
+## Docker image download
+
+Evaluating OpenHands on NoCode-bench need instance-level docker image.
+Please follow the instructions of NoCode-bench image setup to build or download all instance-level dokcer [here](https://github.com/NoCode-bench/NoCode-bench).
+
+## Generate patch
+
+Please follow the instructions [here](../swe_bench/README.md#running-locally-with-docker)
+For example,
+```bash
+bash ./evaluation/benchmarks/nocode_bench/scripts/run_infer_nc.sh llm.claude HEAD CodeActAgent 114 100 10 NoCode-bench/NoCode-bench_Verified test
+```
+The results will be generated in evaluation/evaluation_outputs/outputs/XXX/CodeActAgent/YYY/output.jsonl.
+
+## Runing evaluation
+
+First, install [NoCode-bench](https://github.com/NoCode-bench/NoCode-bench).
+
+Second, convert the output.jsonl to patch.jsonl with [script](scripts/eval/convert.py).
+
+```bash
+python evaluation/benchmarks/multi_swe_bench/scripts/eval/convert.py
+```
+
+Finally, evaluate with NoCode-bench.
+
+```bash
+export PYTHONPATH=$PYTHONPATH:$(pwd)
+python ./evaluation/eval.py \
+    --predictions_path ./all_preds.jsonl \  # <path_to_your_predictions>
+    --log_dir ./evaluation/logs \ # <path_to_your_log_dir>
+    --bench_tasks NoCode-bench/NoCode-bench_Verified \ # <dataset_name>
+    --max_workers 110 \ # <number_of_workers>
+    --output_file eval_result.txt \ # <path_to_your_output_file>
+    --image_level repo \ # <cache_image_level>
+    --timeout 600 \ # <timeout_in_seconds>
+    --proxy None # <proxy_if_needed>
+```

evaluation/benchmarks/nocode_bench/binary_patch_utils.py

@@ -0,0 +1,52 @@
+"""
+Utilities for handling binary files and patch generation in SWE-bench evaluation.
+"""
+
+
+def remove_binary_diffs(patch_text):
+    """
+    Remove binary file diffs from a git patch.
+
+    Args:
+        patch_text (str): The git patch text
+
+    Returns:
+        str: The cleaned patch text with binary diffs removed
+    """
+    lines = patch_text.splitlines()
+    cleaned_lines = []
+    block = []
+    is_binary_block = False
+
+    for line in lines:
+        if line.startswith('diff --git '):
+            if block and not is_binary_block:
+                cleaned_lines.extend(block)
+            block = [line]
+            is_binary_block = False
+        elif 'Binary files' in line:
+            is_binary_block = True
+            block.append(line)
+        else:
+            block.append(line)
+
+    if block and not is_binary_block:
+        cleaned_lines.extend(block)
+    return '\n'.join(cleaned_lines)
+
+
+def remove_binary_files_from_git():
+    """
+    Generate a bash command to remove binary files from git staging.
+
+    Returns:
+        str: A bash command that removes binary files from git staging
+    """
+    return """
+    for file in $(git status --porcelain | grep -E "^(M| M|\\?\\?|A| A)" | cut -c4-); do
+        if [ -f "$file" ] && (file "$file" | grep -q "executable" || git check-attr binary "$file" | grep -q "binary: set"); then
+            git rm -f "$file" 2>/dev/null || rm -f "$file"
+            echo "Removed: $file"
+        fi
+    done
+    """.strip()

evaluation/benchmarks/nocode_bench/consistants.py

@@ -0,0 +1,545 @@
+DOCPATH_PATTERNS = [
+    r'docs/',
+    r'^CHANGES\.rst$',
+    r'doc/',
+    r'ChangeLog',
+    r'^changelog/',
+    r'^CHANGES$',
+]
+
+MATPLOTLIB_CONFIG = {
+    k: {
+        'python': '3.11',
+        'conda_env': 'matplotlib_35',
+        'install': 'python -m pip install -e .',
+        'test_cmd': 'pytest -rA --color=no',
+    }
+    for k in ['3.5', '3.6', '3.7', '3.8', '3.9']
+}
+MATPLOTLIB_CONFIG.update(
+    {
+        k: {
+            'python': '3.8',
+            'conda_env': 'matplotlib_31',
+            'install': 'python -m pip install -e .',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['3.1', '3.2', '3.3', '3.4']
+    }
+)
+MATPLOTLIB_CONFIG.update(
+    {
+        k: {
+            'python': '3.5',
+            'install': 'python setup.py build; python setup.py install',
+            'conda_env': 'matplotlib_11',
+            'nonroot': True,
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['2.0', '2.1', '2.2', '1.0', '1.1', '1.2', '1.3', '1.4', '1.5']
+    }
+)
+for k in ['3.8', '3.9']:
+    MATPLOTLIB_CONFIG[k]['install'] = (
+        'python -m pip install --no-build-isolation -e ".[dev]"'
+    )
+
+
+SYMPY_CONFIG = {}
+SYMPY_CONFIG.update(
+    {
+        '1.0': {
+            'conda_env': 'sympy_10',
+            'install': 'pip install -e .',
+            'test_cmd': 'bin/test -C -v',
+            # testfile -k testname
+        }
+    }
+)
+
+REQUESTS_CONFIG = {}
+REQUESTS_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'requests_227',
+            'install': 'pip install -r requirements-dev.txt',
+            'test_cmd': 'pytest -rA',
+        }
+        for k in ['2.27']
+    }
+)
+REQUESTS_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'requests_226',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest -rA',
+        }
+        for k in ['2.26']
+    }
+)
+
+PYTEST_CONFIG = {}
+PYTEST_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pytest_33',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest -v --color=no',
+        }
+        for k in ['4.4', '4.1', '3.7', '3.4', '3.3']
+    }
+)
+
+PYLINT_CONFIG = {}
+PYLINT_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pylint_210',
+            'install': 'pip install -r requirements_test.txt',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in [
+            '2.10',
+            '2.11',
+            '2.13',
+            '2.14',
+            '2.15',
+            '2.16',
+            '2.17',
+            '3.0',
+            '3.1',
+            '3.2',
+            '3.3',
+        ]
+    }
+)
+PYLINT_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'pylint_210',
+            'pre_install': [
+                r"sed -i 's/setuptools==[0-9.]\+/setuptools==58.0.0/' requirements_test_min.txt"
+            ],
+            'install': 'pip install -r requirements_test.txt',
+            'test_cmd': 'pytest -rA --color=no',
+        }
+        for k in ['3.0', '3.1', '3.2', '3.3']
+    }
+)
+
+ASTROPY_CONFIG = {}
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_11',
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.1', '1.2', '1.3', '2.0']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_30',
+            'pre_install': """echo '[pytest]
+filterwarnings =
+    ignore::DeprecationWarning' > pytest.ini""",
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['3.0', '3.1', '3.2']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_40',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml"""
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.0']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_41',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml""",
+                """sed -i 's/^qt_no_exception_capture = 1$/; qt_no_exception_capture = 1/' setup.cfg""",
+                r"""sed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.tomlsed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.toml""",
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.1']
+    }
+)
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_42',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml""",
+                r"""sed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.tomlsed -i '/setuptools==68.0.0",/a \    "markupsafe==2.0.1",' pyproject.toml""",
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['4.2', '4.3', '5.0', '5.1']
+    }
+)
+
+ASTROPY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'astropy_52',
+            'pre_install': [
+                r"""sed -i 's/requires = \["setuptools",/requires = \["setuptools==68.0.0",/' pyproject.toml"""
+            ],
+            'install': 'python -m pip install -e .[test] --verbose',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['5.2', '5.3', '6.0', '6.1', '7.0']
+    }
+)
+
+DJANGO_CONFIG = {}
+DJANGO_CONFIG.update(
+    {
+        k: {
+            'install': 'pip install -e .',
+            'conda_env': 'django_22',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        }
+        for k in ['1.9', '2.2']
+    }
+)
+DJANGO_CONFIG.update(
+    {
+        '3.2': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_32',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+        '4.2': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_42',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+        '5.1': {
+            'install': 'pip install -e .',
+            'conda_env': 'django_51',
+            'test_cmd': 'python tests/runtests.py --verbosity 2',
+        },
+    }
+)
+SPHINX_CONFIG = {}
+SPHINX_CONFIG.update(
+    {  # 1.x 版本问题，实际无用
+        k: {
+            'conda_env': 'sphinx_20',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': ["sed -i 's/pytest/pytest -rA/' tox.ini"],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_20',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['2.0', '2.1', '2.2', '2.3', '2.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py",
+                "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['3.0', '3.1', '3.2', '3.3', '3.4', '3.5', '4.0']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                (
+                    "grep -q 'sphinxcontrib-htmlhelp>=2.0.0' setup.py && "
+                    "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py || "
+                    "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py"
+                ),
+                (
+                    "grep -q 'sphinxcontrib-serializinghtml>=1.1.5' setup.py && "
+                    "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py || "
+                    "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py"
+                ),
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.1']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+                "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+                "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+                "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+                "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+                "sed -i \"s/'packaging',/'packaging', 'markupsafe<=2.0.1',/\" setup.py",
+                "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py",
+                "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.2', '4.3', '4.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_30',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy37 -v --',
+        }
+        for k in ['4.5', '5.0', '5.1', '5.2']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_60',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy39 -v --',
+        }
+        for k in ['6.0', '6.2', '7.0', '7.1']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_72',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+                'apt-get update && apt-get install -y graphviz',
+            ],
+            'test_cmd': 'tox --current-env -epy39 -v --',
+        }
+        for k in ['7.2', '7.3', '7.4']
+    }
+)
+SPHINX_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'sphinx_80',
+            'install': 'python -m pip install -e .[test]',
+            'pre_install': [
+                "sed -i 's/pytest/pytest -rA/' tox.ini",
+            ],
+            'test_cmd': 'tox --current-env -epy310 -v --',
+        }
+        for k in ['8.0', '8.1']
+    }
+)
+
+
+SKLEARN_CONFIG = {}
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_020',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.20', '0.21', '0.22']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_100',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.23', '0.24', '1.00', '1.01', '1.02']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_104',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.03', '1.04', '1.05']
+    }
+)
+
+SEABORN_CONFIG = {}
+SEABORN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'seaborn_010',
+            'install': 'pip install -e .[dev]',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.3', '0.4', '0.5', '0.6', '0.11', '0.12', '0.13', '0.14']
+    }
+)
+
+XARRAY_CONFIG = {}
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_0014',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0014', '0015', '0016']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_0017',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0017', '0018', '0019', '0020', '0021']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2203',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['2203', '2206', '2209', '2210', '2211', '2212']
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2303',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in [
+            '2303',
+            '2304',
+            '2305',
+            '2306',
+            '2308',
+            '2309',
+            '2310',
+            '2311',
+            '2312',
+        ]
+    }
+)
+XARRAY_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'xarray_2401',
+            'install': 'pip install -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['2401', '2402', '2403', '2405', '2407', '2409', '2410', '2411']
+    }
+)
+
+SKLEARN_CONFIG = {}
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_020',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.20', '0.21', '0.22']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_100',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['0.23', '0.24', '1.00', '1.01', '1.02']
+    }
+)
+SKLEARN_CONFIG.update(
+    {
+        k: {
+            'conda_env': 'skl_104',
+            'install': 'pip install -v --no-use-pep517 --no-build-isolation -e .',
+            'test_cmd': 'pytest --color=no -rA',
+        }
+        for k in ['1.03', '1.04', '1.05', '1.06', '1.07']
+    }
+)
+
+
+MAP_REPO_TO_CONFIG = {
+    'pydata/xarray': XARRAY_CONFIG,
+    'mwaskom/seaborn': SEABORN_CONFIG,
+    'scikit-learn/scikit-learn': SKLEARN_CONFIG,
+    'sphinx-doc/sphinx': SPHINX_CONFIG,
+    'django/django': DJANGO_CONFIG,
+    'astropy/astropy': ASTROPY_CONFIG,
+    'pylint-dev/pylint': PYLINT_CONFIG,
+    'pytest-dev/pytest': PYTEST_CONFIG,
+    'psf/requests': REQUESTS_CONFIG,
+    'sympy/sympy': SYMPY_CONFIG,
+    'matplotlib/matplotlib': MATPLOTLIB_CONFIG,
+}

evaluation/benchmarks/nocode_bench/prompts/nc.j2

@@ -0,0 +1,65 @@
+<uploaded_files>
+/workspace/{{ workspace_dir_name }}
+</uploaded_files>
+
+I've uploaded a python code repository in the directory {{ workspace_dir_name }}. Consider the following issue description:
+
+<doc_change>
+{{ instance.problem_statement }}
+</doc_change>
+
+Can you help me add the new features to the repository based on the changes in the <doc_change>?
+I've already taken care of all changes to any of the test files described in the <doc_change>. This means you DON'T have to modify the testing logic or any of the tests in any way!
+Also the development Python environment is already set up for you (i.e., all dependencies already installed), so you don't need to install other packages.
+Your task is to make the minimal changes to non-test files in the /workspace/{{ workspace_dir_name }} directory to implement the new features required by the documentation updates.
+
+Follow these phases to resolve the issue:
+
+Phase 1. READING: read the requirements and reword it in clearer terms
+   1.1 If there are code or config snippets. Express in words any best practices or conventions in them.
+   1.2 Hightlight method names, variables, file names, stack traces, and technical details, particularly those related to new features.
+   1.3 Explain the new feature requirements in clear terms.
+   1.4 Specify functional scope and expected behavior of new features.
+   1.5 Hightlight any best practices to take into account when developing and testing the new feature.
+
+Phase 2. RUNNING: install and run the functionality in the repository to validate the new features
+   2.1 Follow the readme.
+   2.2 Install the environment and anything needed.
+   2.2 Iterate and figure out how to validate the newly added features.
+
+Phase 3. EXPLORATION: find the files related to the new features and possible implementation solutions
+   3.1 Use `grep` to search for relevant methods, classes, keywords and feature requirements.
+   3.2 Identify all files related to the new features.
+   3.3 Propose the methods and files to implement the new features and explain why.
+   3.4 From the possible file locations, select the most likely location to implement the new features.
+
+Phase 4. TEST CREATION: before implementing any new features, create a script to validate the feature's correctness.
+   4.1 Look at existing test files in the repository to understand the test format/structure.
+   4.2 Create a minimal validation script to verify the newly added features.
+   4.3 Run the validation script to confirm the new features are successfully added and working as expected.
+   4.4 Adjust the validation script as necessary to ensure the new features fully meet the requirements.
+
+Phase 5. FEATURE ANALYSIS: state clearly the new feature and how to implement it
+   5.1 State clearly what the new feature is.
+   5.2 State clearly where the feature should be implemented.
+   5.3 State clearly how the test validates the new feature.
+   5.4 State clearly the best practices to take into account when implementing the new feature.
+   5.5 State clearly how to implement the new feature.
+
+Phase 6. FEATURE IMPLEMENTATION: edit the source code to implement your chosen solution for the new feature
+   6.1 Make minimal, focused changes to implement the new feature.
+
+Phase 7. VERIFICATION: Test your new feature thoroughly.
+   7.1 Run your validation script to verify the new feature works as expected.
+   7.2 Add edge cases to your test script to ensure comprehensive coverage of the new feature.
+   7.3 Run existing tests related to the modified code to ensure you haven't broken anything.
+
+Phase 8. FINAL REVIEW: Carefully re-read the feature requirements and compare your changes with the base commit {{ instance.base_commit }}
+   8.1 Ensure you've fully implemented all required features.
+   8.2 Run any tests in the repository related to:
+     8.2.1 The new features you are adding
+     8.2.2 The files you modified
+     8.2.3 The functions you changed
+   8.3 If any tests fail, revise your implementation until all tests pass and the new feature works as expected.
+
+Be thorough in your exploration, testing, and reasoning. It's fine if your thinking process is lengthy - quality and completeness are more important than brevity.

evaluation/benchmarks/nocode_bench/resource/mapping.py

@@ -0,0 +1,39 @@
+"""Mapping instance_id to resource_factor.
+
+Different instances may have different resource requirements.
+e.g., some instances may require more memory/CPU to run inference.
+This file tracks the resource requirements of different instances.
+"""
+
+import json
+import os
+
+from openhands.core.logger import openhands_logger as logger
+
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_RUNTIME_RESOURCE_FACTOR = int(
+    os.environ.get('DEFAULT_RUNTIME_RESOURCE_FACTOR', 1)
+)
+
+# dataset to resource mapping
+_global_resource_mapping: dict[str, dict[str, float]] = {}
+
+
+def get_resource_mapping(dataset_name: str) -> dict[str, float]:
+    if dataset_name not in _global_resource_mapping:
+        file_path = os.path.join(CUR_DIR, f'{dataset_name}.json')
+        if not os.path.exists(file_path):
+            logger.info(f'Resource mapping for {dataset_name} not found.')
+            return None
+
+        with open(file_path, 'r') as f:
+            _global_resource_mapping[dataset_name] = json.load(f)
+        logger.debug(f'Loaded resource mapping for {dataset_name}')
+    return _global_resource_mapping[dataset_name]
+
+
+def get_instance_resource_factor(dataset_name: str, instance_id: str) -> int:
+    resource_mapping = get_resource_mapping(dataset_name)
+    if resource_mapping is None:
+        return DEFAULT_RUNTIME_RESOURCE_FACTOR
+    return int(resource_mapping.get(instance_id, DEFAULT_RUNTIME_RESOURCE_FACTOR))

evaluation/benchmarks/nocode_bench/run_infer_nc.py

@@ -0,0 +1,905 @@
+import asyncio
+import copy
+import json
+import os
+import tempfile
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+import toml
+from datasets import load_dataset
+from jinja2 import Environment, FileSystemLoader
+
+import openhands.agenthub
+from evaluation.benchmarks.nocode_bench.binary_patch_utils import (
+    remove_binary_diffs,
+    remove_binary_files_from_git,
+)
+from evaluation.benchmarks.nocode_bench.consistants import MAP_REPO_TO_CONFIG
+from evaluation.benchmarks.nocode_bench.resource.mapping import (
+    get_instance_resource_factor,
+)
+from evaluation.benchmarks.nocode_bench.scripts.utils.evaluation_utils import (
+    run_evaluation_nocode_bench,
+)
+from evaluation.utils.shared import (
+    EvalException,
+    EvalMetadata,
+    EvalOutput,
+    assert_and_raise,
+    codeact_user_response,
+    get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
+    is_fatal_evaluation_error,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    OpenHandsConfig,
+    get_evaluation_parser,
+    get_llm_config_arg,
+)
+from openhands.core.config.condenser_config import NoOpCondenserConfig
+from openhands.core.config.utils import get_condenser_config_arg
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.critic import AgentFinishedCritic
+from openhands.events.action import CmdRunAction, FileReadAction, MessageAction
+from openhands.events.observation import (
+    CmdOutputObservation,
+    ErrorObservation,
+    FileReadObservation,
+)
+from openhands.events.serialization.event import event_from_dict, event_to_dict
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+from openhands.utils.shutdown_listener import sleep_if_should_continue
+
+USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
+ENABLE_LLM_EDITOR = os.environ.get('ENABLE_LLM_EDITOR', 'false').lower() == 'true'
+BenchMode = Literal['swe', 'swt', 'swt-ci']
+
+# Global variable to track dataset type
+DATASET_TYPE = 'nc_bench'
+
+
+def set_dataset_type(dataset_name: str) -> str:
+    """Set dataset type based on dataset name."""
+    global DATASET_TYPE
+    DATASET_TYPE = 'nc_bench'
+
+    logger.info(f'Dataset type set to: {DATASET_TYPE}')
+
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
+
+
+def _get_swebench_workspace_dir_name(instance: pd.Series) -> str:
+    return f'{instance.repo.split("/")[-1]}'
+
+
+def get_instruction(instance: pd.Series, metadata: EvalMetadata) -> MessageAction:
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    metadata.details['mode']
+
+    # Determine the template file based on mode and LLM
+
+    template_name = 'nc.j2'
+
+    # Set up Jinja2 environment
+    # Assuming templates are in 'evaluation/benchmarks/swe_bench/prompts' relative to this script
+    prompts_dir = os.path.join(os.path.dirname(__file__), 'prompts')
+    env = Environment(loader=FileSystemLoader(prompts_dir))
+    template = env.get_template(template_name)
+
+    # Prepare context for rendering
+    context = {
+        'instance': instance,
+        'workspace_dir_name': workspace_dir_name,
+        'metadata': metadata,  # Pass metadata if needed in templates
+    }
+
+    context['test_instructions'] = ''  # Ensure it's defined for other modes
+
+    # Render the instruction
+    instruction = template.render(context)
+
+    if RUN_WITH_BROWSING:
+        instruction += (
+            '<IMPORTANT!>\nYou SHOULD NEVER attempt to browse the web. </IMPORTANT!>\n'
+        )
+
+    if 'image_assets' in instance:
+        assets = json.loads(instance['image_assets'])
+        assert 'problem_statement' in assets, (
+            'problem_statement is required in image_assets'
+        )
+        image_urls = assets['problem_statement']
+        return MessageAction(content=instruction, image_urls=image_urls)
+    return MessageAction(content=instruction)
+
+
+DEFAULT_DOCKER_IMAGE_PREFIX = os.environ.get(
+    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/xingyaoww/'
+)
+logger.info(f'Default docker image prefix: {DEFAULT_DOCKER_IMAGE_PREFIX}')
+
+
+def get_instance_docker_image(
+    instance_id: str,
+    swebench_official_image: bool = False,
+) -> str:
+    if swebench_official_image:
+        # Official NoCode-Bench image
+        image_name = f'ncbench_{instance_id}:latest'.lower()
+        logger.debug(f'Using official NoCode-Bench image: {image_name}')
+        return image_name
+    else:
+        raise
+
+
+def get_config(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+) -> OpenHandsConfig:
+    # We use a different instance image for the each instance of NoCode-bench eval
+    use_swebench_official_image = True
+
+    base_container_image = get_instance_docker_image(
+        instance['instance_id'],
+        swebench_official_image=use_swebench_official_image,
+    )
+    logger.info(
+        f'Using instance container image: {base_container_image}. '
+        f'Please make sure this image exists. '
+        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+    )
+
+    sandbox_config = get_default_sandbox_config_for_eval()
+    sandbox_config.base_container_image = base_container_image
+    sandbox_config.enable_auto_lint = True
+    sandbox_config.use_host_network = False
+    # Add platform to the sandbox config to solve issue 4401
+    sandbox_config.platform = 'linux/amd64'
+    sandbox_config.remote_runtime_resource_factor = get_instance_resource_factor(
+        dataset_name=metadata.dataset,
+        instance_id=instance['instance_id'],
+    )
+
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
+        runtime=os.environ.get('RUNTIME', 'docker'),
+        sandbox_config=sandbox_config,
+    )
+
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
+        )
+    )
+    # get 'draft_editor' config if exists
+    config.set_llm_config(get_llm_config_arg('draft_editor'), 'draft_editor')
+
+    agent_config = AgentConfig(
+        enable_jupyter=False,
+        enable_browsing=RUN_WITH_BROWSING,
+        enable_llm_editor=ENABLE_LLM_EDITOR,
+        enable_mcp=False,
+        condenser=metadata.condenser_config,
+        enable_prompt_extensions=False,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def make_serializable(obj):
+    if isinstance(obj, pd.Series):
+        obj = obj.to_dict()
+    if isinstance(obj, dict):
+        return {k: make_serializable(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [make_serializable(v) for v in obj]
+    elif isinstance(obj, tuple):
+        return tuple(make_serializable(v) for v in obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, pd.Timestamp):
+        return str(obj)
+    else:
+        return obj
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+    metadata: EvalMetadata,
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Initialization Fn')
+    logger.info('-' * 30)
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+    obs: CmdOutputObservation
+
+    # Set instance id and git configuration
+    action = CmdRunAction(
+        command=f"""echo 'export SWE_INSTANCE_ID={instance['instance_id']}' >> ~/.bashrc && echo 'export PIP_CACHE_DIR=~/.cache/pip' >> ~/.bashrc && echo "alias git='git --no-pager'" >> ~/.bashrc && git config --global core.pager "" && git config --global diff.binary false"""
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to export SWE_INSTANCE_ID and configure git: {str(obs)}',
+    )
+
+    action = CmdRunAction(command="""export USER=$(whoami); echo USER=${USER} """)
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to export USER: {str(obs)}')
+
+    # inject the init script
+    script_dir = os.path.dirname(__file__)
+
+    # inject the instance info
+    action = CmdRunAction(command='mkdir -p /swe_util/eval_data/instances')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to create /swe_util/eval_data/instances: {str(obs)}',
+    )
+
+    swe_instance_json_name = 'swe-bench-instance.json'
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Construct the full path for the desired file name within the temporary directory
+        temp_file_path = os.path.join(temp_dir, swe_instance_json_name)
+        # Write to the file with the desired name within the temporary directory
+
+        with open(temp_file_path, 'w') as f:
+            if not isinstance(instance, dict):
+                instance_dict = make_serializable(instance)
+            else:
+                instance_dict = dict(instance)
+
+            if DATASET_TYPE == 'nc_bench':
+                config = MAP_REPO_TO_CONFIG.get(instance['repo'], {}).get(
+                    instance['version'], []
+                )
+                docker_conda_env_name = config['conda_env']
+                instance_dict['conda_env'] = docker_conda_env_name
+
+            json.dump([instance_dict], f)
+
+        # Copy the file to the desired location
+        runtime.copy_to(temp_file_path, '/swe_util/eval_data/instances/')
+
+        # inject the instance swe entry
+        entry_script_path = 'instance_nc_entry.sh'
+
+        runtime.copy_to(
+            str(os.path.join(script_dir, f'scripts/setup/{entry_script_path}')),
+            '/swe_util/',
+        )
+
+    action = CmdRunAction(command='cat ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to cat ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command='source ~/.bashrc')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    if isinstance(obs, ErrorObservation):
+        logger.error(f'Failed to source ~/.bashrc: {str(obs)}')
+    assert_and_raise(obs.exit_code == 0, f'Failed to source ~/.bashrc: {str(obs)}')
+
+    action = CmdRunAction(command=f'source /swe_util/{entry_script_path}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to source /swe_util/{entry_script_path}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git reset --hard')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to git reset --hard: {str(obs)}')
+
+    action = CmdRunAction(
+        command='for remote_name in $(git remote); do git remote remove "${remote_name}"; done'
+    )
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(obs.exit_code == 0, f'Failed to remove git remotes: {str(obs)}')
+
+    if DATASET_TYPE != 'Multimodal' and DATASET_TYPE != 'SWE-bench-Live':
+        # Only for non-multimodal datasets, we need to activate the testbed environment for Python
+        # SWE-Bench multimodal datasets and SWE-bench-Live are not using the testbed environment
+        action = CmdRunAction(command='which python')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        assert_and_raise(
+            obs.exit_code == 0,
+            f'Expected to find python interpreter, but got: {str(obs)}',
+        )
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Initialization Fn')
+    logger.info('-' * 30)
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Completion Fn')
+    logger.info('-' * 30)
+    obs: CmdOutputObservation
+    workspace_dir_name = _get_swebench_workspace_dir_name(instance)
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    if obs.exit_code == -1:
+        # The previous command is still running
+        # We need to kill previous command
+        logger.info('The previous command is still running, trying to kill it...')
+        action = CmdRunAction(command='C-c')
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+        # Then run the command again
+        action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    if obs.exit_code == -1:
+        # The previous command is still running
+        # We need to kill previous command
+        logger.info('The previous command is still running, trying to ctrl+z it...')
+        action = CmdRunAction(command='C-z')
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+        # Then run the command again
+        action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+        action.set_hard_timeout(600)
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git config --global core.pager ""')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git config --global core.pager "": {str(obs)}',
+    )
+
+    # First check for any git repositories in subdirectories
+    action = CmdRunAction(command='find . -type d -name .git -not -path "./.git"')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to find git repositories: {str(obs)}',
+    )
+
+    git_dirs = [p for p in obs.content.strip().split('\n') if p]
+    if git_dirs:
+        # Remove all .git directories in subdirectories
+        for git_dir in git_dirs:
+            action = CmdRunAction(command=f'rm -rf "{git_dir}"')
+            action.set_hard_timeout(600)
+            logger.info(action, extra={'msg_type': 'ACTION'})
+            obs = runtime.run_action(action)
+            logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+            assert_and_raise(
+                isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+                f'Failed to remove git directory {git_dir}: {str(obs)}',
+            )
+
+    # add all files
+    action = CmdRunAction(command='git add -A')
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git add -A: {str(obs)}',
+    )
+
+    # Remove binary files from git staging
+    action = CmdRunAction(command=remove_binary_files_from_git())
+    action.set_hard_timeout(600)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to remove binary files: {str(obs)}',
+    )
+
+    n_retries = 0
+    git_patch = None
+    while n_retries < 5:
+        action = CmdRunAction(
+            command=f'git diff --no-color --cached {instance["base_commit"]} > patch.diff'
+        )
+        action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        n_retries += 1
+        if isinstance(obs, CmdOutputObservation):
+            if obs.exit_code == 0:
+                # Read the patch file
+                action = FileReadAction(path='patch.diff')
+                action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+                logger.info(action, extra={'msg_type': 'ACTION'})
+                obs = runtime.run_action(action)
+                logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+                if isinstance(obs, FileReadObservation):
+                    git_patch = obs.content
+                    break
+                elif isinstance(obs, ErrorObservation):
+                    # Fall back to cat "patch.diff" to get the patch
+                    assert 'File could not be decoded as utf-8' in obs.content
+                    action = CmdRunAction(command='cat patch.diff')
+                    action.set_hard_timeout(max(300 + 100 * n_retries, 600))
+                    logger.info(action, extra={'msg_type': 'ACTION'})
+                    obs = runtime.run_action(action)
+                    assert isinstance(obs, CmdOutputObservation) and obs.exit_code == 0
+                    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+                    git_patch = obs.content
+                    break
+                else:
+                    assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+            else:
+                logger.info('Failed to get git diff, retrying...')
+                sleep_if_should_continue(10)
+        elif isinstance(obs, ErrorObservation):
+            logger.error(f'Error occurred: {obs.content}. Retrying...')
+            sleep_if_should_continue(10)
+        else:
+            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+
+    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
+
+    # Remove binary diffs from the patch
+    git_patch = remove_binary_diffs(git_patch)
+
+    logger.info('-' * 30)
+    logger.info('END Runtime Completion Fn')
+    logger.info('-' * 30)
+    return {'git_patch': git_patch}
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+    runtime_failure_count: int = 0,
+) -> EvalOutput:
+    config = get_config(instance, metadata)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    # Increase resource_factor with increasing attempt_id
+    if runtime_failure_count > 0:
+        config.sandbox.remote_runtime_resource_factor = min(
+            config.sandbox.remote_runtime_resource_factor * (2**runtime_failure_count),
+            8,
+        )
+        logger.warning(
+            f'This is the {runtime_failure_count + 1}th attempt for instance {instance.instance_id}, setting resource factor to {config.sandbox.remote_runtime_resource_factor}'
+        )
+
+    metadata = copy.deepcopy(metadata)
+    metadata.details['runtime_failure_count'] = runtime_failure_count
+    metadata.details['remote_runtime_resource_factor'] = (
+        config.sandbox.remote_runtime_resource_factor
+    )
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+
+    try:
+        initialize_runtime(runtime, instance, metadata)
+
+        message_action = get_instruction(instance, metadata)
+
+        # Here's how you can run the agent (similar to the `main` function) and get the final task state
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=message_action,
+                runtime=runtime,
+                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                    metadata.agent_class
+                ],
+            )
+        )
+
+        # if fatal error, throw EvalError to trigger re-run
+        if is_fatal_evaluation_error(state.last_error):
+            raise EvalException('Fatal error detected: ' + state.last_error)
+
+        # ======= THIS IS SWE-Bench specific =======
+        # Get git patch
+        if DATASET_TYPE == 'SWE-bench-Live':
+            from evaluation.benchmarks.swe_bench.live_utils import (
+                complete_runtime as complete_runtime_fn,
+            )
+        else:
+            complete_runtime_fn = complete_runtime
+        return_val = complete_runtime_fn(runtime, instance)
+        git_patch = return_val['git_patch']
+        logger.info(
+            f'Got git diff for instance {instance.instance_id}:\n--------\n{git_patch}\n--------'
+        )
+    finally:
+        runtime.close()
+    # ==========================================
+
+    # ======= Attempt to evaluate the agent's edits =======
+    # we use eval_infer.sh to evaluate the agent's edits, not here
+    # because the agent may alter the environment / testcases
+    test_result = {
+        'git_patch': git_patch,
+    }
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
+    histories = [event_to_dict(event) for event in state.history]
+    metrics = get_metrics(state)
+
+    # Save the output
+    instruction = message_action.content
+    if message_action.image_urls:
+        instruction += (
+            '\n\n<image_urls>' + '\n'.join(message_action.image_urls) + '</image_urls>'
+        )
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        instance=instance.to_dict(),  # SWE Bench specific
+        test_result=test_result,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+    )
+    return output
+
+
+def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
+    file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.toml')
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as file:
+            data = toml.load(file)
+            if 'selected_ids' in data:
+                selected_ids = data['selected_ids']
+                logger.info(
+                    f'Filtering {len(selected_ids)} tasks from "selected_ids"...'
+                )
+                subset = dataset[dataset[filter_column].isin(selected_ids)]
+                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
+                return subset
+            if 'selected_repos' in data:
+                # repos for the swe-bench instances:
+                # ['astropy/astropy', 'django/django', 'matplotlib/matplotlib', 'mwaskom/seaborn', 'pallets/flask', 'psf/requests', 'pydata/xarray', 'pylint-dev/pylint', 'pytest-dev/pytest', 'scikit-learn/scikit-learn', 'sphinx-doc/sphinx', 'sympy/sympy']
+                selected_repos = data['selected_repos']
+                if isinstance(selected_repos, str):
+                    selected_repos = [selected_repos]
+                assert isinstance(selected_repos, list)
+                logger.info(
+                    f'Filtering {selected_repos} tasks from "selected_repos"...'
+                )
+                subset = dataset[dataset['repo'].isin(selected_repos)]
+                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
+                return subset
+
+    skip_ids = os.environ.get('SKIP_IDS', '').split(',')
+    if len(skip_ids) > 0:
+        logger.info(f'Filtering {len(skip_ids)} tasks from "SKIP_IDS"...')
+        return dataset[~dataset[filter_column].isin(skip_ids)]
+    return dataset
+
+
+if __name__ == '__main__':
+    parser = get_evaluation_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='NoCode-bench/NoCode-bench_Verified',
+        help='data set to evaluate on, either full-test or lite-test',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='split to evaluate on',
+    )
+    parser.add_argument(
+        '--mode',
+        type=str,
+        default='swe',
+        choices=['swe', 'swt', 'swt-ci'],
+        help="mode to run the evaluation, either 'swe', 'swt', or 'swt-ci'",
+    )
+
+    args, _ = parser.parse_known_args()
+
+    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
+    # so we don't need to manage file uploading to OpenHands's repo
+
+    dataset = load_dataset(args.dataset, args.split)
+
+    # Set the global dataset type based on dataset name
+    set_dataset_type(args.dataset)
+
+    swe_bench_tests = filter_dataset(dataset.to_pandas(), 'instance_id')
+    logger.info(
+        f'Loaded dataset {args.dataset} with split {args.split}: {len(swe_bench_tests)} tasks'
+    )
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        llm_config.log_completions = True
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    # Get condenser config from environment variable
+    condenser_name = os.environ.get('EVAL_CONDENSER')
+    if condenser_name:
+        condenser_config = get_condenser_config_arg(condenser_name)
+        if condenser_config is None:
+            raise ValueError(
+                f'Could not find Condenser config: EVAL_CONDENSER={condenser_name}'
+            )
+    else:
+        # If no specific condenser config is provided via env var, default to NoOpCondenser
+        condenser_config = NoOpCondenserConfig()
+        logger.debug(
+            'No Condenser config provided via EVAL_CONDENSER, using NoOpCondenser.'
+        )
+
+    details = {'mode': args.mode}
+    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
+
+    dataset_descrption = (
+        args.dataset.replace('/', '__') + '-' + args.split.replace('/', '__')
+    )
+    metadata = make_metadata(
+        llm_config,
+        dataset_descrption,
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        details=details,
+        condenser_config=condenser_config,
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    print(f'### OUTPUT FILE: {output_file} ###')
+
+    # Run evaluation in iterative mode:
+    # If a rollout fails to output AgentFinishAction, we will try again until it succeeds OR total 3 attempts have been made.
+    ITERATIVE_EVAL_MODE = (
+        os.environ.get('ITERATIVE_EVAL_MODE', 'false').lower() == 'true'
+    )
+    ITERATIVE_EVAL_MODE_MAX_ATTEMPTS = int(
+        os.environ.get('ITERATIVE_EVAL_MODE_MAX_ATTEMPTS', '3')
+    )
+
+    if not ITERATIVE_EVAL_MODE:
+        # load the dataset
+        instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
+        if len(instances) > 0 and not isinstance(
+            instances['PASS2PASS'][instances['PASS2PASS'].index[0]], str
+        ):
+            for col in ['PASS2PASS', 'FAIL2PASS']:
+                instances[col] = instances[col].apply(lambda x: str(x))
+
+        run_evaluation_nocode_bench(
+            instances,
+            metadata,
+            output_file,
+            args.eval_num_workers,
+            process_instance,
+            timeout_seconds=8
+            * 60
+            * 60,  # 8 hour PER instance should be more than enough
+            max_retries=5,
+        )
+    else:
+        critic = AgentFinishedCritic()
+
+        def get_cur_output_file_path(attempt: int) -> str:
+            return (
+                f'{output_file.removesuffix(".jsonl")}.critic_attempt_{attempt}.jsonl'
+            )
+
+        eval_ids = None
+        for attempt in range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1):
+            cur_output_file = get_cur_output_file_path(attempt)
+            logger.info(
+                f'Running evaluation with critic {critic.__class__.__name__} for attempt {attempt} of {ITERATIVE_EVAL_MODE_MAX_ATTEMPTS}.'
+            )
+
+            # For deterministic eval, we set temperature to 0.1 for (>1) attempt
+            # so hopefully we get slightly different results
+            if attempt > 1 and metadata.llm_config.temperature == 0:
+                logger.info(
+                    f'Detected temperature is 0 for (>1) attempt {attempt}. Setting temperature to 0.1...'
+                )
+                metadata.llm_config.temperature = 0.1
+
+            # Load instances - at first attempt, we evaluate all instances
+            # On subsequent attempts, we only evaluate the instances that failed the previous attempt determined by critic
+            instances = prepare_dataset(
+                swe_bench_tests, cur_output_file, args.eval_n_limit, eval_ids=eval_ids
+            )
+            if len(instances) > 0 and not isinstance(
+                instances['PASS2PASS'][instances['PASS2PASS'].index[0]], str
+            ):
+                for col in ['PASS2PASS', 'FAIL2PASS']:
+                    instances[col] = instances[col].apply(lambda x: str(x))
+
+            # Run evaluation - but save them to cur_output_file
+            logger.info(
+                f'Evaluating {len(instances)} instances for attempt {attempt}...'
+            )
+            run_evaluation_nocode_bench(
+                instances,
+                metadata,
+                cur_output_file,
+                args.eval_num_workers,
+                process_instance,
+                timeout_seconds=8
+                * 60
+                * 60,  # 8 hour PER instance should be more than enough
+                max_retries=5,
+            )
+
+            # When eval is done, we update eval_ids to the instances that failed the current attempt
+            instances_failed = []
+            logger.info(
+                f'Use critic {critic.__class__.__name__} to check {len(instances)} instances for attempt {attempt}...'
+            )
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    try:
+                        history = [
+                            event_from_dict(event) for event in instance['history']
+                        ]
+                        critic_result = critic.evaluate(
+                            history, instance['test_result'].get('git_patch', '')
+                        )
+                        if not critic_result.success:
+                            instances_failed.append(instance['instance_id'])
+                    except Exception as e:
+                        logger.error(
+                            f'Error loading history for instance {instance["instance_id"]}: {e}'
+                        )
+                        instances_failed.append(instance['instance_id'])
+            logger.info(
+                f'{len(instances_failed)} instances failed the current attempt {attempt}: {instances_failed}'
+            )
+            eval_ids = instances_failed
+
+            # If no instances failed, we break
+            if len(instances_failed) == 0:
+                break
+
+        # Then we should aggregate the results from all attempts into the original output file
+        # and remove the intermediate files
+        logger.info(
+            'Aggregating results from all attempts into the original output file...'
+        )
+        fout = open(output_file, 'w')
+        added_instance_ids = set()
+        for attempt in reversed(range(1, ITERATIVE_EVAL_MODE_MAX_ATTEMPTS + 1)):
+            cur_output_file = get_cur_output_file_path(attempt)
+            if not os.path.exists(cur_output_file):
+                logger.warning(
+                    f'Intermediate output file {cur_output_file} does not exist. Skipping...'
+                )
+                continue
+
+            with open(cur_output_file, 'r') as f:
+                for line in f:
+                    instance = json.loads(line)
+                    # Also make sure git_patch is not empty - otherwise we fall back to previous attempt (empty patch is worse than anything else)
+                    if (
+                        instance['instance_id'] not in added_instance_ids
+                        and instance['test_result'].get('git_patch', '').strip()
+                    ):
+                        fout.write(line)
+                        added_instance_ids.add(instance['instance_id'])
+            logger.info(
+                f'Aggregated instances from {cur_output_file}. Total instances added so far: {len(added_instance_ids)}'
+            )
+        fout.close()
+        logger.info(
+            f'Done! Total {len(added_instance_ids)} instances added to {output_file}'
+        )

evaluation/benchmarks/nocode_bench/scripts/eval/convert.py

@@ -0,0 +1,33 @@
+import argparse
+import json
+
+
+def main(output_jsonl: str):
+    with open(output_jsonl, 'r') as f:
+        for line in f:
+            try:
+                output = json.loads(line)
+                pred = {
+                    'instance_id': output['instance_id'],
+                    'model_name_or_path': output['metadata']['llm_config']['model'],
+                    'model_patch': output['test_result']['git_patch'],
+                }
+            except Exception as e:
+                print(
+                    f'Error while reading output of instance {output["instance_id"]}: {e}'
+                )
+
+            print(json.dumps(pred))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--output_jsonl',
+        type=str,
+        required=True,
+        help='Path to the prediction file (.../outputs.jsonl)',
+    )
+    args = parser.parse_args()
+
+    main(args.output_jsonl)

evaluation/benchmarks/nocode_bench/scripts/eval/verify_costs.py

@@ -0,0 +1,104 @@
+import argparse
+
+import pandas as pd
+
+from openhands.core.logger import openhands_logger as logger
+
+
+def verify_instance_costs(row: pd.Series) -> float:
+    """
+    Verifies that the accumulated_cost matches the sum of individual costs in metrics.
+    Also checks for duplicate consecutive costs which might indicate buggy counting.
+    If the consecutive costs are identical, the file is affected by this bug:
+    https://github.com/All-Hands-AI/OpenHands/issues/5383
+
+    Args:
+        row: DataFrame row containing instance data with metrics
+    Returns:
+        float: The verified total cost for this instance (corrected if needed)
+    """
+    try:
+        metrics = row.get('metrics')
+        if not metrics:
+            logger.warning(f'Instance {row["instance_id"]}: No metrics found')
+            return 0.0
+
+        accumulated = metrics.get('accumulated_cost')
+        costs = metrics.get('costs', [])
+
+        if accumulated is None:
+            logger.warning(
+                f'Instance {row["instance_id"]}: No accumulated_cost in metrics'
+            )
+            return 0.0
+
+        # Check for duplicate consecutive costs and systematic even-odd pairs
+        has_duplicate = False
+        all_pairs_match = True
+
+        # Check each even-odd pair (0-1, 2-3, etc.)
+        for i in range(0, len(costs) - 1, 2):
+            if abs(costs[i]['cost'] - costs[i + 1]['cost']) < 1e-6:
+                has_duplicate = True
+                logger.debug(
+                    f'Instance {row["instance_id"]}: Possible buggy double-counting detected! '
+                    f'Steps {i} and {i + 1} have identical costs: {costs[i]["cost"]:.2f}'
+                )
+            else:
+                all_pairs_match = False
+                break
+
+        # Calculate total cost, accounting for buggy double counting if detected
+        if len(costs) >= 2 and has_duplicate and all_pairs_match:
+            paired_steps_cost = sum(
+                cost_entry['cost']
+                for cost_entry in costs[: -1 if len(costs) % 2 else None]
+            )
+            real_paired_cost = paired_steps_cost / 2
+
+            unpaired_cost = costs[-1]['cost'] if len(costs) % 2 else 0
+            total_cost = real_paired_cost + unpaired_cost
+
+        else:
+            total_cost = sum(cost_entry['cost'] for cost_entry in costs)
+
+        if not abs(total_cost - accumulated) < 1e-6:
+            logger.warning(
+                f'Instance {row["instance_id"]}: Cost mismatch: '
+                f'accumulated: {accumulated:.2f}, sum of costs: {total_cost:.2f}, '
+            )
+
+        return total_cost
+
+    except Exception as e:
+        logger.error(
+            f'Error verifying costs for instance {row.get("instance_id", "UNKNOWN")}: {e}'
+        )
+        return 0.0
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Verify costs in SWE-bench output file'
+    )
+    parser.add_argument(
+        'input_filepath', type=str, help='Path to the output.jsonl file'
+    )
+    args = parser.parse_args()
+
+    try:
+        # Load and verify the JSONL file
+        df = pd.read_json(args.input_filepath, lines=True)
+        logger.info(f'Loaded {len(df)} instances from {args.input_filepath}')
+
+        # Verify costs for each instance and sum up total
+        total_cost = df.apply(verify_instance_costs, axis=1).sum()
+        logger.info(f'Total verified cost across all instances: ${total_cost:.2f}')
+
+    except Exception as e:
+        logger.error(f'Failed to process file: {e}')
+        raise
+
+
+if __name__ == '__main__':
+    main()

evaluation/benchmarks/nocode_bench/scripts/run_infer_nc.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+MAX_ITER=$5
+NUM_WORKERS=$6
+DATASET=$7
+SPLIT=$8
+N_RUNS=$9
+MODE=${10}
+
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$MAX_ITER" ]; then
+  echo "MAX_ITER not specified, use default 100"
+  MAX_ITER=100
+fi
+
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
+
+if [ -z "$DATASET" ]; then
+  echo "DATASET not specified, use default princeton-nlp/SWE-bench_Lite"
+  DATASET="princeton-nlp/SWE-bench_Lite"
+fi
+
+if [ -z "$SPLIT" ]; then
+  echo "SPLIT not specified, use default test"
+  SPLIT="test"
+fi
+
+if [ -z "$MODE" ]; then
+  MODE="swe"
+  echo "MODE not specified, use default $MODE"
+fi
+
+if [ -n "$EVAL_CONDENSER" ]; then
+  echo "Using Condenser Config: $EVAL_CONDENSER"
+else
+  echo "No Condenser Config provided via EVAL_CONDENSER, use default (NoOpCondenser)."
+fi
+
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+echo "SPLIT: $SPLIT"
+echo "MAX_ITER: $MAX_ITER"
+echo "NUM_WORKERS: $NUM_WORKERS"
+echo "COMMIT_HASH: $COMMIT_HASH"
+echo "MODE: $MODE"
+echo "EVAL_CONDENSER: $EVAL_CONDENSER"
+
+# Default to NOT use Hint
+if [ -z "$USE_HINT_TEXT" ]; then
+  export USE_HINT_TEXT=false
+fi
+echo "USE_HINT_TEXT: $USE_HINT_TEXT"
+EVAL_NOTE="$OPENHANDS_VERSION"
+# if not using Hint, add -no-hint to the eval note
+if [ "$USE_HINT_TEXT" = false ]; then
+  EVAL_NOTE="$EVAL_NOTE-no-hint"
+fi
+
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
+if [ -n "$EXP_NAME" ]; then
+  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+# if mode != swe, add mode to the eval note
+if [ "$MODE" != "swe" ]; then
+  EVAL_NOTE="${EVAL_NOTE}-${MODE}"
+fi
+# Add condenser config to eval note if provided
+if [ -n "$EVAL_CONDENSER" ]; then
+  EVAL_NOTE="${EVAL_NOTE}-${EVAL_CONDENSER}"
+fi
+
+function run_eval() {
+  local eval_note="${1}"
+  COMMAND="poetry run python evaluation/benchmarks/nocode_bench/run_infer_nc.py \
+    --agent-cls $AGENT \
+    --llm-config $MODEL_CONFIG \
+    --max-iterations $MAX_ITER \
+    --eval-num-workers $NUM_WORKERS \
+    --eval-note $eval_note \
+    --dataset $DATASET \
+    --split $SPLIT \
+    --mode $MODE"
+
+
+
+  if [ -n "$EVAL_LIMIT" ]; then
+    echo "EVAL_LIMIT: $EVAL_LIMIT"
+    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+  fi
+
+  # Run the command
+  eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+if [ -z "$N_RUNS" ]; then
+  N_RUNS=1
+  echo "N_RUNS not specified, use default $N_RUNS"
+fi
+
+# Skip runs if the run number is in the SKIP_RUNS list
+# read from env variable SKIP_RUNS as a comma separated list of run numbers
+SKIP_RUNS=(${SKIP_RUNS//,/ })
+for i in $(seq 1 $N_RUNS); do
+  if [[ " ${SKIP_RUNS[@]} " =~ " $i " ]]; then
+    echo "Skipping run $i"
+    continue
+  fi
+  current_eval_note="$EVAL_NOTE-run_$i"
+  echo "EVAL_NOTE: $current_eval_note"
+  run_eval $current_eval_note
+done
+
+checkout_original_branch

evaluation/benchmarks/nocode_bench/scripts/setup/compare_patch_filename.py

@@ -0,0 +1,54 @@
+"""This script compares gold patches with OpenHands-generated patches and check whether
+OpenHands found the right (set of) files to modify.
+"""
+
+import argparse
+import json
+import re
+
+
+def extract_modified_files(patch):
+    modified_files = set()
+    file_pattern = re.compile(r'^diff --git a/(.*?) b/')
+
+    for line in patch.split('\n'):
+        match = file_pattern.match(line)
+        if match:
+            modified_files.add(match.group(1))
+
+    return modified_files
+
+
+def process_report(oh_output_file):
+    succ = 0
+    fail = 0
+    for line in open(oh_output_file):
+        line = json.loads(line)
+        instance_id = line['instance_id']
+        gold_patch = line['swe_instance']['patch']
+        generated_patch = line['git_patch']
+        gold_modified_files = extract_modified_files(gold_patch)
+        # swe-bench lite only: a gold patch always contains exactly one file
+        assert len(gold_modified_files) == 1
+        generated_modified_files = extract_modified_files(generated_patch)
+
+        # Check if all files in gold_patch are also in generated_patch
+        all_files_in_generated = gold_modified_files.issubset(generated_modified_files)
+        if all_files_in_generated:
+            succ += 1
+        else:
+            fail += 1
+            print(
+                f'{instance_id}: file mismatch, gold = {gold_modified_files}, generated = {generated_modified_files}'
+            )
+    print(
+        f'\nSUMMARY: {succ} out of {succ + fail} instances found correct files to edit, success rate = {succ / float(succ + fail)}'
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--oh_output_file', help='Path to the OH output file')
+    args = parser.parse_args()
+
+    process_report(args.oh_output_file)

evaluation/benchmarks/nocode_bench/scripts/setup/instance_nc_entry.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+source ~/.bashrc
+SWEUTIL_DIR=/swe_util
+
+if [ -z "$SWE_INSTANCE_ID" ]; then
+    echo "Error: SWE_INSTANCE_ID is not set." >&2
+    exit 1
+fi
+
+
+item=$(jq --arg INSTANCE_ID "$SWE_INSTANCE_ID" '.[] | select(.instance_id == $INSTANCE_ID)' $SWEUTIL_DIR/eval_data/instances/swe-bench-instance.json)
+
+
+if [[ -z "$item" ]]; then
+  echo "No item found for the provided instance ID."
+  exit 1
+fi
+
+REPO_NAME=$(echo "$item" | jq -r '.repo | split("/")[-1]')
+WORKSPACE_NAME="$REPO_NAME"
+
+
+echo "WORKSPACE_NAME: $WORKSPACE_NAME"
+
+
+# Clear the workspace
+if [ -d /workspace ]; then
+    rm -rf /workspace/*
+else
+    mkdir /workspace
+fi
+# Copy repo to workspace
+if [ -d /workspace/$WORKSPACE_NAME ]; then
+    rm -rf /workspace/$WORKSPACE_NAME
+fi
+mkdir -p /workspace
+
+SRC_DIR="/root/$REPO_NAME"
+DEST_DIR="/workspace/$WORKSPACE_NAME"
+
+cp -r "$SRC_DIR" "$DEST_DIR"
+
+
+
+echo ">> Extracting conda environment name..."
+CONDA_ENV_NAME=$(echo "$item" | jq -r '.conda_env // empty')
+
+# Activate instance-specific environment
+if [ -d /opt/miniconda3 ]; then
+    . /opt/miniconda3/etc/profile.d/conda.sh
+    conda activate $CONDA_ENV_NAME
+fi

evaluation/benchmarks/nocode_bench/scripts/utils/evaluation_utils.py

@@ -0,0 +1,154 @@
+import json
+import multiprocessing as mp
+from typing import Awaitable, Callable, TextIO
+
+import numpy as np
+import pandas as pd
+from pydantic import SecretStr
+from tqdm import tqdm
+
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    _process_instance_wrapper,
+    _process_instance_wrapper_mp,
+)
+from openhands.core.logger import openhands_logger as logger
+
+
+def update_progress_nc(
+    result: EvalOutput,
+    pbar: tqdm,
+    output_fp: TextIO,
+):
+    """Update the progress bar and write the result to the output file."""
+    pbar.update(1)
+    pbar.set_description(f'Instance {result.instance_id}')
+    pbar.set_postfix_str(f'Test Result: {str(result.test_result)[:300]}...')
+    logger.info(
+        f'Finished evaluation for instance {result.instance_id}: '
+        f'{str(result.test_result)[:300]}...\n'
+    )
+
+    def make_serializable(obj):
+        if isinstance(obj, pd.Series):
+            return make_serializable(obj.to_dict())
+
+        if isinstance(obj, dict):
+            return {k: make_serializable(v) for k, v in obj.items()}
+
+        elif isinstance(obj, (list, tuple, set)):
+            converted = [make_serializable(v) for v in obj]
+            if isinstance(obj, list):
+                return converted
+            elif isinstance(obj, tuple):
+                return tuple(converted)
+            else:  # set
+                return converted
+
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+
+        elif isinstance(obj, np.generic):
+            return obj.item()
+
+        elif isinstance(obj, pd.Timestamp):
+            return obj.isoformat()
+
+        elif SecretStr is not None and isinstance(obj, SecretStr):
+            return str(obj)
+
+        else:
+            return obj
+
+    try:
+        raw_data = result.model_dump(mode='python', round_trip=False)
+        safe_data = make_serializable(raw_data)
+        output_fp.write(json.dumps(safe_data, ensure_ascii=False) + '\n')
+        output_fp.flush()
+
+    except Exception as e:
+        logger.error(f'Failed to write full result: {e}')
+
+        fallback = {
+            'instance_id': result.instance_id,
+            'model_patch': result.test_result.get('git_patch', ''),
+        }
+        try:
+            output_fp.write(json.dumps(fallback, ensure_ascii=False) + '\n')
+            output_fp.flush()
+            logger.info(
+                f'Wrote fallback result for instance {result.instance_id}: only instance_id and model_patch.'
+            )
+        except Exception as e2:
+            logger.error(f'Failed to write fallback result: {e2}')
+
+
+def cleanup():
+    print('Cleaning up child processes...')
+    for process in mp.active_children():
+        print(f'Terminating child process: {process.name}')
+        process.terminate()
+        process.join()
+
+
+def run_evaluation_nocode_bench(
+    dataset: pd.DataFrame,
+    metadata: EvalMetadata | None,
+    output_file: str,
+    num_workers: int,
+    process_instance_func: Callable[
+        [pd.Series, EvalMetadata, bool], Awaitable[EvalOutput]
+    ],
+    max_retries: int = 5,  # number of retries for each instance
+    timeout_seconds: int | None = None,
+):
+    use_multiprocessing = num_workers > 1
+
+    if metadata is not None:
+        logger.info(
+            f'Evaluation started with Agent {metadata.agent_class}:\n'
+            f'model {metadata.llm_config.model}, max iterations {metadata.max_iterations}.\n'
+        )
+    else:
+        logger.warning('Running evaluation without metadata.')
+        logger.info(f'Evaluation started with {num_workers} workers.')
+
+    total_instances = len(dataset)
+    pbar = tqdm(total=total_instances, desc='Instances processed')
+    output_fp = open(output_file, 'a')
+
+    try:
+        if use_multiprocessing:
+            with mp.Pool(num_workers) as pool:
+                args_iter = (
+                    (
+                        process_instance_func,
+                        instance,
+                        metadata,
+                        True,
+                        max_retries,
+                        timeout_seconds,
+                    )
+                    for _, instance in dataset.iterrows()
+                )
+                results = pool.imap_unordered(_process_instance_wrapper_mp, args_iter)
+                for result in results:
+                    update_progress_nc(result, pbar, output_fp)
+        else:
+            for _, instance in dataset.iterrows():
+                result = _process_instance_wrapper(
+                    process_instance_func=process_instance_func,
+                    instance=instance,
+                    metadata=metadata,
+                    use_mp=False,
+                    max_retries=max_retries,
+                )
+                update_progress_nc(result, pbar, output_fp)
+
+    except KeyboardInterrupt:
+        print('\nKeyboardInterrupt received. Cleaning up...\n')
+        cleanup()
+
+    output_fp.close()
+    logger.info('\nEvaluation finished.\n')

evaluation/benchmarks/scienceagentbench/run_infer.py

@@ -12,6 +12,8 @@ from evaluation.utils.shared import (
     codeact_user_response,
     compatibility_for_eval_history_pairs,
     get_default_sandbox_config_for_eval,
+    get_metrics,
+    get_openhands_config_for_eval,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -63,16 +65,10 @@ def get_config(
     sandbox_config.base_container_image = (
         'docker.io/xingyaoww/openhands-eval-scienceagentbench'
     )
-    config = OpenHandsConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
+        metadata=metadata,
         runtime=os.environ.get('RUNTIME', 'docker'),
-        max_budget_per_task=4,
-        max_iterations=metadata.max_iterations,
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     config.set_llm_config(
         update_llm_config_for_completions_logging(
@@ -218,7 +214,7 @@ If the program uses some packages that are incompatible, please figure out alter
     # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
     if state is None:
         raise ValueError('State should not be None.')
-    metrics = state.metrics.get() if state.metrics else None
+    metrics = get_metrics(state)
 
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here

evaluation/benchmarks/swe_bench/README.md

@@ -2,6 +2,8 @@
 
 This folder contains the evaluation harness that we built on top of the original [SWE-Bench benchmark](https://www.swebench.com/) ([paper](https://arxiv.org/abs/2310.06770)).
 
+**UPDATE (8/12/2025): We now support running SWE-rebench evaluation (see the paper [here](https://arxiv.org/abs/2505.20411))! For how to run it, checkout [this README](./SWE-rebench.md).**
+
 **UPDATE (6/15/2025): We now support running SWE-bench-Live evaluation (see the paper [here](https://arxiv.org/abs/2505.23419))! For how to run it, checkout [this README](./SWE-bench-Live.md).**
 
 **UPDATE (5/26/2025): We now support running interactive SWE-Bench evaluation (see the paper [here](https://arxiv.org/abs/2502.13069))! For how to run it, checkout [this README](./SWE-Interact.md).**
@@ -91,6 +93,9 @@ export USE_HINT_TEXT=true # Ignore this if you are not sure.
 
 # Specify a condenser configuration for memory management (default: NoOpCondenser)
 export EVAL_CONDENSER=summarizer_for_eval # Name of the condenser config group in config.toml
+
+# Specify the instruction prompt template file name
+export INSTRUCTION_TEMPLATE_NAME=swe_custom.j2 # Name of the file in the swe_bench/prompts folder.
 ```
 
 Let's say you'd like to run 10 instances using `llm.eval_gpt4_1106_preview` and CodeActAgent,

evaluation/benchmarks/swe_bench/SWE-rebench.md

@@ -0,0 +1,84 @@
+# SWE-rebench
+
+<p align="center">
+<a href="https://arxiv.org/abs/2505.20411">📃 Paper</a>
+•
+<a href="https://huggingface.co/datasets/nebius/SWE-rebench">🤗 HuggingFace</a>
+•
+<a href="https://swe-rebench.com/leaderboard">📊 Leaderboard</a>
+</p>
+
+SWE-rebench is a large-scale dataset for verifiable software engineering tasks.
+It comes in **two datasets**:
+
+* **[`nebius/SWE-rebench-leaderboard`](https://huggingface.co/datasets/nebius/SWE-rebench-leaderboard)** – updatable benchmark used for [leaderboard evaluation](https://swe-rebench.com/leaderboard).
+* **[`nebius/SWE-rebench`](https://huggingface.co/datasets/nebius/SWE-rebench)** – full dataset with **21,302 tasks**, suitable for training or large-scale offline evaluation.
+
+This document explains how to run OpenHands on SWE-rebench, using the leaderboard split as the main example.
+To run on the full dataset, simply replace the dataset name.
+
+
+## Setting Up
+
+Set up your development environment and configure your LLM provider by following the [SWE-bench README](README.md) in this directory.
+
+
+## Running Inference
+
+Use the existing SWE-bench inference script, changing the dataset to `nebius/SWE-rebench-leaderboard` and selecting the split (`test` for leaderboard submission):
+
+```bash
+./evaluation/benchmarks/swe_bench/scripts/run_infer.sh \
+    llm.your_llm HEAD CodeActAgent 30 50 1 nebius/SWE-rebench-leaderboard test
+```
+
+Arguments:
+
+* `llm.your_llm` – your model configuration key
+* `HEAD` – commit reference for reproducibility
+* `CodeActAgent` – agent type
+* `10` – number of examples to evaluate
+* `50` – maximum iterations per task (increase if needed)
+* `1` – number of workers
+* `nebius/SWE-rebench-leaderboard` – Hugging Face dataset name
+* `test` – dataset split
+
+**Tip:** To run on the **full 21k dataset**, replace `nebius/SWE-rebench-leaderboard` with `nebius/SWE-rebench`.
+
+
+## Evaluating Results
+
+After inference completes, evaluate using the [SWE-bench-fork evaluation harness](https://github.com/SWE-rebench/SWE-bench-fork).
+
+1. Convert the OpenHands output to SWE-bench evaluation format:
+
+```bash
+python evaluation/benchmarks/swe_bench/scripts/live/convert.py \
+  --output_jsonl path/to/evaluation/output.jsonl > preds.jsonl
+```
+
+2. Clone the SWE-bench-fork repo (https://github.com/SWE-rebench/SWE-bench-fork) and follow its README to install dependencies.
+
+
+3. Run the evaluation using the fork:
+
+```bash
+python -m swebench.harness.run_evaluation \
+    --dataset_name nebius/SWE-rebench-leaderboard \
+    --split test \
+    --predictions_path preds.jsonl \
+    --max_workers 10 \
+    --run_id openhands
+```
+
+
+## Citation
+
+```bibtex
+@article{badertdinov2025swerebench,
+  title={SWE-rebench: An Automated Pipeline for Task Collection and Decontaminated Evaluation of Software Engineering Agents},
+  author={Badertdinov, Ibragim and Golubev, Alexander and Nekrashevich, Maksim and Shevtsov, Anton and Karasik, Simon and Andriushchenko, Andrei and Trofimova, Maria and Litvintseva, Daria and Yangel, Boris},
+  journal={arXiv preprint arXiv:2505.20411},
+  year={2025}
+}
+```

evaluation/benchmarks/swe_bench/binary_patch_utils.py

@@ -1,11 +1,8 @@
-"""
-Utilities for handling binary files and patch generation in SWE-bench evaluation.
-"""
+"""Utilities for handling binary files and patch generation in SWE-bench evaluation."""
 
 
 def remove_binary_diffs(patch_text):
-    """
-    Remove binary file diffs from a git patch.
+    """Remove binary file diffs from a git patch.
 
     Args:
         patch_text (str): The git patch text
@@ -36,8 +33,7 @@ def remove_binary_diffs(patch_text):
 
 
 def remove_binary_files_from_git():
-    """
-    Generate a bash command to remove binary files from git staging.
+    """Generate a bash command to remove binary files from git staging.
 
     Returns:
         str: A bash command that removes binary files from git staging

evaluation/benchmarks/swe_bench/eval_infer.py

@@ -19,6 +19,7 @@ from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     get_default_sandbox_config_for_eval,
+    get_openhands_config_for_eval,
     prepare_dataset,
     reset_logger_for_multiprocessing,
     run_evaluation,
@@ -83,13 +84,9 @@ def get_config(metadata: EvalMetadata, instance: pd.Series) -> OpenHandsConfig:
         dataset_name=metadata.dataset,
         instance_id=instance['instance_id'],
     )
-    config = OpenHandsConfig(
-        run_as_openhands=False,
+    config = get_openhands_config_for_eval(
         runtime=os.environ.get('RUNTIME', 'docker'),
-        sandbox=sandbox_config,
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
+        sandbox_config=sandbox_config,
     )
     return config
 
@@ -111,8 +108,7 @@ def process_instance(
     runtime_failure_count: int = 0,
     conditional_imports: ConditionalImports | None = None,
 ) -> EvalOutput:
-    """
-    Evaluate agent performance on a SWE-bench problem instance.
+    """Evaluate agent performance on a SWE-bench problem instance.
 
     Note that this signature differs from the expected input to `run_evaluation`. Use
     `functools.partial` to provide optional arguments before passing to the evaluation harness.