add cloud sql

Still needs testing and integration of proper client (Rather than the mock I've been using)

.github/dependabot.yml

@@ -16,9 +16,6 @@ updates:
       chromadb:
         patterns:
           - "chromadb"
-      browsergym:
-        patterns:
-          - "browsergym"
       security-all:
         applies-to: "security-updates"
         patterns:

.github/workflows/eval-runner.yml

@@ -3,6 +3,8 @@ name: Run Evaluation
 on:
   pull_request:
     types: [labeled]
+  schedule:
+    - cron: "0 1 * * *" # Run daily at 1 AM UTC
   workflow_dispatch:
     inputs:
       reason:

.github/workflows/lint-fix.yml

@@ -1,62 +0,0 @@
-name: Lint Fix
-
-on:
-  pull_request:
-    types: [labeled]
-
-jobs:
-  lint-fix:
-    if: github.event.label.name == 'lint-fix'
-    name: Fix linting issues
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      # Frontend lint fixes
-      - name: Install Node.js 20
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-      - name: Install frontend dependencies
-        run: |
-          cd frontend
-          npm install --frozen-lockfile
-      - name: Fix frontend lint issues
-        run: |
-          cd frontend
-          npm run lint:fix
-
-      # Python lint fixes
-      - name: Set up python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-          cache: 'pip'
-      - name: Install pre-commit
-        run: pip install pre-commit==3.7.0
-      - name: Fix python lint issues
-        run: |
-          # Run all pre-commit hooks and continue even if they modify files (exit code 1)
-          pre-commit run --config ./dev_config/python/.pre-commit-config.yaml --files openhands/**/* evaluation/**/* tests/**/* || true
-
-      # Commit and push changes if any
-      - name: Check for changes
-        id: git-check
-        run: |
-          git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT
-      - name: Commit and push if there are changes
-        if: steps.git-check.outputs.changes == 'true'
-        run: |
-          git config --local user.email "openhands@all-hands.dev"
-          git config --local user.name "OpenHands Bot"
-          git add -A
-          git commit -m "🤖 Auto-fix linting issues"
-          git push

.github/workflows/openhands-resolver.yml

@@ -1,291 +1,15 @@
-name: Auto-Fix Tagged Issue with OpenHands
+name: Resolve Issues with OpenHands
 
 on:
-  workflow_call:
-    inputs:
-      max_iterations:
-        required: false
-        type: number
-        default: 50
-      macro:
-        required: false
-        type: string
-        default: "@openhands-agent"
-      target_branch:
-        required: false
-        type: string
-        default: "main"
-        description: "Target branch to pull and create PR against"
-    secrets:
-      LLM_MODEL:
-        required: true
-      LLM_API_KEY:
-        required: true
-      LLM_BASE_URL:
-        required: false
-      PAT_TOKEN:
-        required: true
-      PAT_USERNAME:
-        required: true
-
   issues:
     types: [labeled]
   pull_request:
     types: [labeled]
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  pull_request_review:
-    types: [submitted]
-
-permissions:
-  contents: write
-  pull-requests: write
-  issues: write
 
 jobs:
-  auto-fix:
-    if: |
-      github.event_name == 'workflow_call' ||
-      github.event.label.name == 'fix-me' ||
-      github.event.label.name == 'fix-me-experimental' ||
-
-      (
-        ((github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
-        contains(github.event.comment.body, inputs.macro || '@openhands-agent') &&
-        (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER')
-        ) ||
-
-        (github.event_name == 'pull_request_review' &&
-        contains(github.event.review.body, inputs.macro || '@openhands-agent') &&
-        (github.event.review.author_association == 'OWNER' || github.event.review.author_association == 'COLLABORATOR' || github.event.review.author_association == 'MEMBER')
-        )
-      )
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-
-      - name: Get latest versions and create requirements.txt
-        run: |
-          python -m pip index versions openhands-ai > openhands_versions.txt
-          OPENHANDS_VERSION=$(head -n 1 openhands_versions.txt | awk '{print $2}' | tr -d '()')
-          echo "openhands-ai==${OPENHANDS_VERSION}" >> requirements.txt
-          cat requirements.txt
-
-      - name: Cache pip dependencies
-        if: |
-          !(
-            github.event.label.name == 'fix-me-experimental' ||
-            (
-              (github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
-              contains(github.event.comment.body, '@openhands-agent-exp')
-            ) ||
-            (
-              github.event_name == 'pull_request_review' &&
-              contains(github.event.review.body, '@openhands-agent-exp')
-            )
-          )
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/*
-          key: ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
-
-      - name: Check required environment variables
-        env:
-          LLM_MODEL: ${{ secrets.LLM_MODEL }}
-          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
-          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
-          PAT_USERNAME: ${{ secrets.PAT_USERNAME }}
-        run: |
-          required_vars=("LLM_MODEL" "LLM_API_KEY" "PAT_TOKEN" "PAT_USERNAME")
-          for var in "${required_vars[@]}"; do
-            if [ -z "${!var}" ]; then
-              echo "Error: Required environment variable $var is not set."
-              exit 1
-            fi
-          done
-
-      - name: Set environment variables
-        run: |
-          if [ -n "${{ github.event.review.body }}" ]; then
-            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          elif [ -n "${{ github.event.issue.pull_request }}" ]; then
-            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
-            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          elif [ -n "${{ github.event.pull_request.number }}" ]; then
-            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
-          else
-            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
-            echo "ISSUE_TYPE=issue" >> $GITHUB_ENV
-          fi
-
-          if [ -n "${{ github.event.review.body }}" ]; then
-            echo "COMMENT_ID=${{ github.event.review.id || 'None' }}" >> $GITHUB_ENV
-          else
-            echo "COMMENT_ID=${{ github.event.comment.id || 'None' }}" >> $GITHUB_ENV
-          fi
-
-          echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV
-          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
-
-          # Set branch variables
-          echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
-
-      - name: Comment on issue with start message
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const issueType = process.env.ISSUE_TYPE;
-            github.rest.issues.createComment({
-              issue_number: ${{ env.ISSUE_NUMBER }},
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
-            });
-
-      - name: Install OpenHands
-        run: |
-          if [[ "${{ github.event.label.name }}" == "fix-me-experimental" ]] ||
-             ([[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]] &&
-              [[ "${{ github.event.comment.body }}" == "@openhands-agent-exp"* ]]) ||
-             ([[ "${{ github.event_name }}" == "pull_request_review" ]] &&
-              [[ "${{ github.event.review.body }}" == "@openhands-agent-exp"* ]]); then
-            python -m pip install --upgrade pip
-            pip install git+https://github.com/all-hands-ai/openhands.git
-          else
-            python -m pip install --upgrade -r requirements.txt
-          fi
-
-      - name: Attempt to resolve issue
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
-          LLM_MODEL: ${{ secrets.LLM_MODEL }}
-          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
-          PYTHONPATH: ""
-        run: |
-          cd /tmp && python -m openhands.resolver.resolve_issue \
-            --repo ${{ github.repository }} \
-            --issue-number ${{ env.ISSUE_NUMBER }} \
-            --issue-type ${{ env.ISSUE_TYPE }} \
-            --max-iterations ${{ env.MAX_ITERATIONS }} \
-            --comment-id ${{ env.COMMENT_ID }} \
-
-      - name: Check resolution result
-        id: check_result
-        run: |
-          if cd /tmp && grep -q '"success":true' output/output.jsonl; then
-            echo "RESOLUTION_SUCCESS=true" >> $GITHUB_OUTPUT
-          else
-            echo "RESOLUTION_SUCCESS=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Upload output.jsonl as artifact
-        uses: actions/upload-artifact@v4
-        if: always() # Upload even if the previous steps fail
-        with:
-          name: resolver-output
-          path: /tmp/output/output.jsonl
-          retention-days: 30 # Keep the artifact for 30 days
-
-      - name: Create draft PR or push branch
-        if: always() # Create PR or branch even if the previous steps fail
-        env:
-          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
-          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
-          LLM_MODEL: ${{ secrets.LLM_MODEL }}
-          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
-          PYTHONPATH: ""
-        run: |
-          if [ "${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}" == "true" ]; then
-            cd /tmp && python -m openhands.resolver.send_pull_request \
-              --issue-number ${{ env.ISSUE_NUMBER }} \
-              --pr-type draft | tee pr_result.txt && \
-              grep "draft created" pr_result.txt | sed 's/.*\///g' > pr_number.txt
-          else
-            cd /tmp && python -m openhands.resolver.send_pull_request \
-              --issue-number ${{ env.ISSUE_NUMBER }} \
-              --pr-type branch \
-              --send-on-failure | tee branch_result.txt && \
-              grep "branch created" branch_result.txt | sed 's/.*\///g; s/.expand=1//g' > branch_name.txt
-          fi
-
-      - name: Comment on issue
-        uses: actions/github-script@v7
-        if: always() # Comment on issue even if the previous steps fail
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const fs = require('fs');
-            const issueNumber = ${{ env.ISSUE_NUMBER }};
-            const success = ${{ steps.check_result.outputs.RESOLUTION_SUCCESS }};
-
-            let prNumber = '';
-            let branchName = '';
-            let logContent = '';
-            const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`;
-
-            try {
-              if (success){
-                logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim();
-              } else {
-                logContent = fs.readFileSync('/tmp/branch_result.txt', 'utf8').trim();
-              }
-            } catch (error) {
-              console.error('Error reading results file:', error);
-            }
-
-            try {
-              if (success) {
-                prNumber = fs.readFileSync('/tmp/pr_number.txt', 'utf8').trim();
-              } else {
-                branchName = fs.readFileSync('/tmp/branch_name.txt', 'utf8').trim();
-              }
-            } catch (error) {
-              console.error('Error reading file:', error);
-            }
-
-            if (logContent.includes(noChangesMessage)) {
-              github.rest.issues.createComment({
-                issue_number: issueNumber,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.`
-              });
-            } else if (success && prNumber) {
-              github.rest.issues.createComment({
-                issue_number: issueNumber,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: `A potential fix has been generated and a draft PR #${prNumber} has been created. Please review the changes.`
-              });
-            } else if (!success && branchName) {
-              github.rest.issues.createComment({
-                issue_number: issueNumber,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`
-              });
-            } else {
-              github.rest.issues.createComment({
-                issue_number: issueNumber,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.`
-              });
-            }
+  call-openhands-resolver:
+    uses: All-Hands-AI/openhands-resolver/.github/workflows/openhands-resolver.yml@main
+    if: github.event.label.name == 'fix-me'
+    with:
+      max_iterations: 50
+    secrets: inherit

.github/workflows/review-pr.yml

@@ -0,0 +1,81 @@
+# Workflow that uses OpenHands to review a pull request. PR must be labeled 'review-this'
+name: Use OpenHands to Review Pull Request
+
+on:
+  pull_request:
+    types: [synchronize, labeled]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  dogfood:
+    if: contains(github.event.pull_request.labels.*.name, 'review-this')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: install git, github cli
+      run: |
+        sudo apt-get install -y git gh
+        git config --global --add safe.directory $PWD
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.base.ref }} # check out the target branch
+    - name: Download Diff
+      run: |
+        curl -O "${{ github.event.pull_request.diff_url }}" -L
+    - name: Write Task File
+      run: |
+        echo "Your coworker wants to apply a pull request to this project." > task.txt
+        echo "Read and review ${{ github.event.pull_request.number }}.diff file. Create a review-${{ github.event.pull_request.number }}.txt and write your concise comments and suggestions there." >> task.txt
+        echo "Do not ask me for confirmation at any point." >> task.txt
+        echo "" >> task.txt
+        echo "Title" >> task.txt
+        echo "${{ github.event.pull_request.title }}" >> task.txt
+        echo "" >> task.txt
+        echo "Description" >> task.txt
+        echo "${{ github.event.pull_request.body }}" >> task.txt
+        echo "" >> task.txt
+        echo "Diff file is: ${{ github.event.pull_request.number }}.diff" >> task.txt
+    - name: Set up environment
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        export PATH="/github/home/.local/bin:$PATH"
+        poetry install --without evaluation,llama-index
+        poetry run playwright install --with-deps chromium
+    - name: Run OpenHands
+      env:
+        LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+        LLM_MODEL: ${{ vars.LLM_MODEL }}
+      run: |
+        # Append path to launch poetry
+        export PATH="/github/home/.local/bin:$PATH"
+        # Append path to correctly import package, note: must set pwd at first
+        export PYTHONPATH=$(pwd):$PYTHONPATH
+        export WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE
+        export WORKSPACE_BASE=$GITHUB_WORKSPACE
+        echo -e "/exit\n" | poetry run python openhands/core/main.py -i 50 -f task.txt
+        rm task.txt
+    - name: Check if review file is non-empty
+      id: check_file
+      run: |
+        ls -la
+        if [[ -s review-${{ github.event.pull_request.number }}.txt ]]; then
+          echo "non_empty=true" >> $GITHUB_OUTPUT
+        fi
+      shell: bash
+    - name: Create PR review if file is non-empty
+      env:
+        GH_TOKEN: ${{ github.token }}
+      if: steps.check_file.outputs.non_empty == 'true'
+      run: |
+        gh pr review ${{ github.event.pull_request.number }} --comment --body-file "review-${{ github.event.pull_request.number }}.txt"

.github/workflows/run-eval.yml

@@ -1,53 +0,0 @@
-# Run evaluation on a PR
-name: Run Eval
-
-# Runs when a PR is labeled with one of the "run-eval-" labels
-on:
-  pull_request:
-    types: [labeled]
-
-jobs:
-  trigger-job:
-    name: Trigger remote eval job
-    if: ${{ github.event.label.name == 'run-eval-xs' || github.event.label.name == 'run-eval-s' || github.event.label.name == 'run-eval-m' }}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout PR branch
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-
-      - name: Trigger remote job
-        run: |
-          REPO_URL="https://github.com/${{ github.repository }}"
-          PR_BRANCH="${{ github.head_ref }}"
-          echo "Repository URL: $REPO_URL"
-          echo "PR Branch: $PR_BRANCH"
-
-          if [[ "${{ github.event.label.name }}" == "run-eval-xs" ]]; then
-            EVAL_INSTANCES="1"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-s" ]]; then
-            EVAL_INSTANCES="5"
-          elif [[ "${{ github.event.label.name }}" == "run-eval-m" ]]; then
-            EVAL_INSTANCES="30"
-          fi
-
-          curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
-            -H "Accept: application/vnd.github+json" \
-            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
-            https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
-
-          # Send Slack message
-          PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
-          slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
-          curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
-            https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
-
-      - name: Comment on PR
-        uses: KeisukeYamashita/create-comment@v1
-        with:
-          unique: false
-          comment: |
-            Running evaluation on the PR. Once eval is done, the results will be posted.

.gitignore

@@ -175,10 +175,6 @@ evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
 evaluation/scienceagentbench/benchmark
-evaluation/commit0_bench/repos
-
-# openhands resolver
-output/
 
 # frontend
 

COMMUNITY.md

@@ -1,43 +0,0 @@
-# 🙌 The OpenHands Community
-
-The OpenHands community is built around the belief that (1) AI and AI agents are going to fundamentally change the way
-we build software, and (2) if this is true, we should do everything we can to make sure that the benefits provided by
-such powerful technology are accessible to everyone.
-
-If this resonates with you, we'd love to have you join us in our quest!
-
-## 🤝 How to Join
-
-Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
-
-## 💪 Becoming a Contributor
-
-We welcome contributions from everyone! Whether you're a developer, a researcher, or simply enthusiastic about advancing
-the field of software engineering with AI, there are many ways to get involved:
-
-- **Code Contributions:** Help us develop new core functionality, improve our agents, improve the frontend and other
-interfaces, or anything else that would help make OpenHands better.
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in
-evaluating the models, or suggest improvements.
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
-
-For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
-
-## Code of Conduct
-
-We have a [Code of Conduct](./CODE_OF_CONDUCT.md) that we expect all contributors to adhere to.
-Long story short, we are aiming for an open, welcoming, diverse, inclusive, and healthy community.
-All contributors are expected to contribute to building this sort of community.
-
-## 🛠️ Becoming a Maintainer
-
-For contributors who have made significant and sustained contributions to the project, there is a possibility of joining
-the maintainer team. The process for this is as follows:
-
-1. Any contributor who has made sustained and high-quality contributions to the codebase can be nominated by any
-maintainer. If you feel that you may qualify you can reach out to any of the maintainers that have reviewed your PRs and ask if you can be nominated.
-2. Once a maintainer nominates a new maintainer, there will be a discussion period among the maintainers for at least 3 days.
-3. If no concerns are raised the nomination will be accepted by acclamation, and if concerns are raised there will be a discussion and possible vote.
-
-Note that just making many PRs does not immediately imply that you will become a maintainer. We will be looking
-at sustained high-quality contributions over a period of time, as well as good teamwork and adherence to our [Code of Conduct](./CODE_OF_CONDUCT.md).

CONTRIBUTING.md

@@ -54,7 +54,7 @@ The agent needs a place to run code and commands. When you run OpenHands on your
 to do this by default. But there are other ways of creating a sandbox for the agent.
 
 If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
-by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
+by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/runtime.py).
 
 #### Testing
 When you write code, it is also good to write tests. Please navigate to the `tests` folder to see existing test suites.
@@ -92,32 +92,3 @@ You may also check out previous PRs in the [PR list](https://github.com/All-Hand
 
 If your changes are user-facing (e.g. a new feature in the UI, a change in behavior, or a bugfix)
 please include a short message that we can add to our changelog.
-
-## How to Make Effective Contributions
-
-### Opening Issues
-
-If you notice any bugs or have any feature requests please open them via the [issues page](https://github.com/All-Hands-AI/OpenHands/issues). We will triage based on how critical the bug is or how potentially useful the improvement is, discuss, and implement the ones that the community has interest/effort for.
-
-Further, if you see an issue you like, please leave a "thumbs-up" or a comment, which will help us prioritize.
-
-### Making Pull Requests
-
-We're generally happy to consider all PRs, with the evaluation process varying based on the type of change:
-
-#### For Small Improvements
-
-Small improvements with few downsides are typically reviewed and approved quickly.
-One thing to check when making changes is to ensure that all continuous integration tests pass, which you can check before getting a review.
-
-#### For Core Agent Changes
-
-We need to be more careful with changes to the core agent, as it is imperative to maintain high quality. These PRs are evaluated based on three key metrics:
-
-1. **Accuracy**
-2. **Efficiency**
-3. **Code Complexity**
-
-If it improves accuracy, efficiency, or both with only a minimal change to code quality, that's great we're happy to merge it in!
-If there are bigger tradeoffs (e.g. helping efficiency a lot and hurting accuracy a little) we might want to put it behind a feature flag.
-Either way, please feel free to discuss on github issues or slack, and we will give guidance and preliminary feedback.

Development.md

@@ -38,9 +38,7 @@ make build
 ```
 
 ### 3. Configuring the Language Model
-OpenHands supports a diverse array of Language Models (LMs) through the powerful [litellm](https://docs.litellm.ai) library.
-By default, we've chosen Claude Sonnet 3.5 as our go-to model, but the world is your oyster! You can unleash the
-potential of any other LM that piques your interest.
+OpenHands supports a diverse array of Language Models (LMs) through the powerful [litellm](https://docs.litellm.ai) library. By default, we've chosen the mighty GPT-4 from OpenAI as our go-to model, but the world is your oyster! You can unleash the potential of Anthropic's suave Claude, the enigmatic Llama, or any other LM that piques your interest.
 
 To configure the LM of your choice, run:
 
@@ -54,7 +52,10 @@ To configure the LM of your choice, run:
    Environment variables > config.toml variables > default variables
 
 **Note on Alternative Models:**
-See [our documentation](https://docs.all-hands.dev/modules/usage/llms) for recommended models.
+Some alternative models may prove more challenging to tame than others. Fear not, brave adventurer! We shall soon unveil LLM-specific documentation to guide you on your quest.
+And if you've already mastered the art of wielding a model other than OpenAI's GPT, we encourage you to share your setup instructions with us by creating instructions and adding it [to our documentation](https://github.com/All-Hands-AI/OpenHands/tree/main/docs/modules/usage/llms).
+
+For a full list of the LM providers and models available, please consult the [litellm documentation](https://docs.litellm.ai/docs/providers).
 
 ### 4. Running the application
 #### Option A: Run the Full Application
@@ -97,10 +98,9 @@ poetry run pytest ./tests/unit/test_*.py
 2. Update the poetry.lock file via `poetry lock --no-update`
 
 ### 9. Use existing Docker image
-To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by
-setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
-
-Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.14-nikolaik`
+To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image. Follow these steps:
+1. Set the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
+2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.13-nikolaik
 
 ## Develop inside Docker container
 

ISSUE_TRIAGE.md

@@ -6,9 +6,9 @@ These are the procedures and guidelines on how issues are triaged in this repo b
 * Issues may be tagged with what it relates to (**backend**, **frontend**, **agent quality**, etc.)
 
 ## Severity
-* **Low**: Minor issues or affecting single user.
-* **Medium**: Affecting multiple users.
-* **Critical**: Affecting all users or potential security issues.
+* **Low**: Minor issues, single user report
+* **Medium**: Affecting multiple users
+* **Critical**: Affecting all users or potential security issues
 
 ## Effort
 * Issues may be estimated with effort required (**small effort**, **medium effort**, **large effort**)
@@ -17,9 +17,9 @@ These are the procedures and guidelines on how issues are triaged in this repo b
 * Issues with low implementation difficulty may be tagged with **good first issue**
 
 ## Not Enough Information
-* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear.
-* If an issue is unclear and the author does not provide more information or respond to a request, the issue may be closed as **not planned** (Usually after a week).
+* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear
+* If an issue is unclear and the author does not provide more information or respond to a request, the issue may be closed as **not planned** (Usually after a week)
 
 ## Multiple Requests/Fixes in One Issue
-* These issues will be narrowed down to one request/fix so the issue is more easily tracked and fixed.
-* Issues may be broken down into multiple issues if required.
+* These issues will be narrowed down to one request/fix so the issue is more easily tracked and fixed
+* Issues may be broken down into multiple issues if required

Makefile

@@ -184,10 +184,6 @@ test:
 	@$(MAKE) -s test-frontend
 
 build-frontend:
-	@echo "$(YELLOW)Cleaning TypeScript build cache...$(RESET)"
-	@cd frontend && npx tsc --build --clean
-	@echo "$(YELLOW)Cleaning Git cache for casing issues...$(RESET)"
-	@cd frontend && git rm -r --cached . && git add . && git commit -m "Fix Git cache" || echo "No changes to commit"
 	@echo "$(YELLOW)Building frontend...$(RESET)"
 	@cd frontend && npm run build
 

README.md

@@ -38,16 +38,16 @@ See the [Installation](https://docs.all-hands.dev/modules/usage/installation) gu
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik
 
 docker run -it --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
-    -e LOG_ALL_EVENTS=true \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
+    -e LOG_ALL_EVENTS=true \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.14
+    docker.all-hands.dev/all-hands-ai/openhands:0.13
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
@@ -61,7 +61,7 @@ works best, but you have [many options](https://docs.all-hands.dev/modules/usage
 You can also [connect OpenHands to your local filesystem](https://docs.all-hands.dev/modules/usage/runtimes),
 run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
 interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode),
-or run it on tagged issues with [a github action](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
+or run it on tagged issues with [a github action](https://github.com/All-Hands-AI/OpenHands-resolver).
 
 Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
@@ -77,21 +77,28 @@ To learn more about the project, and for tips on using OpenHands,
 There you'll find resources on how to use different LLM providers,
 troubleshooting resources, and advanced configuration options.
 
-## 🤝 How to Join the Community
+## 🤝 How to Contribute
 
-OpenHands is a community-driven project, and we welcome contributions from everyone. We do most of our communication
-through Slack, so this is the best place to start, but we also are happy to have you contact us on Discord or Github:
+OpenHands is a community-driven project, and we welcome contributions from everyone.
+Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of
+software engineering with AI, there are many ways to get involved:
 
-- [Join our Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
-- [Join our Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
-- [Read or post Github Issues](https://github.com/All-Hands-AI/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
+- **Code Contributions:** Help us develop new agents, core functionality, the frontend and other interfaces, or sandboxing solutions.
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements.
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
 
-See more about the community in [COMMUNITY.md](./COMMUNITY.md) or find details on contributing in [CONTRIBUTING.md](./CONTRIBUTING.md).
+For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
+
+## 🤖 Join Our Community
+
+Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
+Let's make software engineering better together!
+
+- [Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
+- [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
 
 ## 📈 Progress
 
-See the monthly OpenHands roadmap [here](https://github.com/orgs/All-Hands-AI/projects/1) (updated at the maintainer's meeting at the end of each month).
-
 <p align="center">
   <a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
     <img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">

compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.13-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.14-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.13-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -14,4 +14,4 @@ Pour utiliser l'Action GitHub OpenHands dans le dépôt OpenHands, un mainteneur
 
 ## Installation de l'Action dans un nouveau dépôt
 
-Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
+Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -12,4 +12,4 @@
 
 ## 在新仓库中安装 Action
 
-要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md) 进行操作。
+要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow) 进行操作。

docs/modules/usage/about.md

@@ -1,6 +1,6 @@
-# About OpenHands
+# 📚 Misc
 
-## Research Strategy
+## ⭐️ Research Strategy
 
 Achieving full replication of production-grade applications with LLMs is a complex endeavor. Our strategy involves:
 
@@ -9,11 +9,34 @@ Achieving full replication of production-grade applications with LLMs is a compl
 3. **Task Planning:** Developing capabilities for bug detection, codebase management, and optimization
 4. **Evaluation:** Establishing comprehensive evaluation metrics to better understand and improve our models
 
-## Default Agent
+## 🚧 Default Agent
 
 Our default Agent is currently the [CodeActAgent](agents), which is capable of generating code and handling files.
 
-## Built With
+## 🤝 How to Contribute
+
+OpenHands is a community-driven project, and we welcome contributions from everyone. Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of software engineering with AI, there are many ways to get involved:
+
+- **Code Contributions:** Help us develop the core functionalities, frontend interface, or sandboxing solutions
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability
+
+For details, please check [this document](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md).
+
+## 🤖 Join Our Community
+
+We have both Slack workspace for the collaboration on building OpenHands and Discord server for discussion about anything related, e.g., this project, LLM, agent, etc.
+
+- [Slack workspace](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
+- [Discord server](https://discord.gg/ESHStjSjD4)
+
+If you would love to contribute, feel free to join our community. Let's simplify software engineering together!
+
+🐚 **Code less, make more with OpenHands.**
+
+[![Star History Chart](https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date)](https://star-history.com/#All-Hands-AI/OpenHands&Date)
+
+## 🛠️ Built With
 
 OpenHands is built using a combination of powerful frameworks and libraries, providing a robust foundation for its development. Here are the key technologies used in the project:
 
@@ -21,6 +44,6 @@ OpenHands is built using a combination of powerful frameworks and libraries, pro
 
 Please note that the selection of these technologies is in progress, and additional technologies may be added or existing ones may be removed as the project evolves. We strive to adopt the most suitable and efficient tools to enhance the capabilities of OpenHands.
 
-## License
+## 📜 License
 
-Distributed under MIT [License](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE).
+Distributed under the MIT License. See [our license](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE) for more information.

docs/modules/usage/how-to/cli-mode.md

@@ -50,7 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -59,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.13 \
     python -m openhands.core.cli
 ```
 

docs/modules/usage/how-to/custom-sandbox-guide.md

@@ -62,3 +62,25 @@ Run OpenHands by running ```make run``` in the top level directory.
 ## Technical Explanation
 
 Please refer to [custom docker image section of the runtime documentation](https://docs.all-hands.dev/modules/usage/architecture/runtime#advanced-how-openhands-builds-and-maintains-od-runtime-images) for more details.
+
+## Troubleshooting / Errors
+
+### Error: ```useradd: UID 1000 is not unique```
+
+If you see this error in the console output it is because OpenHands is trying to create the openhands user in the sandbox with a UID of 1000, however this UID is already being used in the image (for some reason). To fix this change the sandbox_user_id field in the config.toml file to a different value:
+
+```toml
+[core]
+workspace_base="./workspace"
+run_as_openhands=true
+sandbox_base_container_image="custom_image"
+sandbox_user_id="1001"
+```
+
+### Port use errors
+
+If you see an error about a port being in use or unavailable, try deleting all running Docker Containers (run `docker ps` and `docker rm` relevant containers) and then re-running ```make run``` .
+
+## Discuss
+
+For other issues or questions join the [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) or [Discord](https://discord.gg/ESHStjSjD4) and ask!

docs/modules/usage/how-to/github-action.md

@@ -4,92 +4,12 @@ This guide explains how to use the OpenHands GitHub Action, both within the Open
 
 ## Using the Action in the OpenHands Repository
 
-To use the OpenHands GitHub Action in a repository, you can:
+To use the OpenHands GitHub Action in the OpenHands repository, an OpenHands maintainer can:
 
 1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue or leave a comment on the issue starting with `@openhands-agent`.
-
-The action will automatically trigger and attempt to resolve the issue.
+2. Add the `fix-me` label to the issue.
+3. The action will automatically trigger and attempt to resolve the issue.
 
 ## Installing the Action in a New Repository
 
-To install the OpenHands GitHub Action in your own repository, follow
-the [README for the OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
-
-## Usage Tips
-
-### Iterative resolution
-
-1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue, or leave a comment starting with `@openhands-agent`
-3. Review the attempt to resolve the issue by checking the pull request
-4. Follow up with feedback through general comments, review comments, or inline thread comments
-5. Add the `fix-me` label to the pull request, or address a specific comment by starting with `@openhands-agent`
-
-### Label versus Macro
-
-- Label (`fix-me`): Requests OpenHands to address the **entire** issue or pull request.
-- Macro (`@openhands-agent`): Requests OpenHands to consider only the issue/pull request description and **the specific comment**.
-
-## Advanced Settings
-
-### Add custom repository settings
-
-You can provide custom directions for OpenHands by following the [README for the resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md#providing-custom-instructions).
-
-### Configure custom macro
-
-To customize the default macro (`@openhands-agent`):
-
-1. [Create a repository variable](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#creating-configuration-variables-for-a-repository) named `OPENHANDS_MACRO`
-2. Assign the variable a custom value
-
-## Writing Effective .openhands_instructions Files
-
-The `.openhands_instructions` file is a file that you can put in the root directory of your repository to guide OpenHands in understanding and working with your repository effectively. Here are key tips for writing high-quality instructions:
-
-### Core Principles
-
-1. **Concise but Informative**: Provide a clear, focused overview of the repository that emphasizes the most common actions OpenHands will need to perform.
-
-2. **Repository Structure**: Explain the key directories and their purposes, especially highlighting where different types of code (e.g., frontend, backend) are located.
-
-3. **Development Workflows**: Document the essential commands for:
-   - Building and setting up the project
-   - Running tests
-   - Linting and code quality checks
-   - Any environment-specific requirements
-
-4. **Testing Guidelines**: Specify:
-   - Where tests are located
-   - How to run specific test suites
-   - Any testing conventions or requirements
-
-### Example Structure
-
-```markdown
-# Repository Overview
-[Brief description of the project]
-
-## General Setup
-- Main build command
-- Development environment setup
-- Pre-commit checks
-
-## Backend
-- Location and structure
-- Testing instructions
-- Environment requirements
-
-## Frontend
-- Setup prerequisites
-- Build and test commands
-- Environment variables
-
-## Additional Guidelines
-- Code style requirements
-- Special considerations
-- Common workflows
-```
-
-For a real-world example, refer to the [OpenHands repository's .openhands_instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/.openhands_instructions).
+To install the OpenHands GitHub Action in your own repository, follow the [directions in the OpenHands Resolver repo](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).

docs/modules/usage/how-to/headless-mode.md

@@ -44,7 +44,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -54,6 +54,6 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.14 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.13 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```

docs/modules/usage/installation.mdx

@@ -11,16 +11,16 @@
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.14-nikolaik \
-    -e LOG_ALL_EVENTS=true \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.13-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
+    -e LOG_ALL_EVENTS=true \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.14
+    docker.all-hands.dev/all-hands-ai/openhands:0.13
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/modules/usage/runtimes.md

@@ -49,7 +49,7 @@ but seems to work well on most systems.
 
 ## All Hands Runtime
 The All Hands Runtime is currently in beta. You can request access by joining
-the #remote-runtime-limited-beta channel on Slack ([see the README](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-join-our-community) for an invite).
+the #remote-runtime-limited-beta channel on Slack (see the README for an invite).
 
 To use the All Hands Runtime, set the following environment variables when
 starting OpenHands:
@@ -66,7 +66,7 @@ docker run # ...
 ## Modal Runtime
 Our partners at [Modal](https://modal.com/) have also provided a runtime for OpenHands.
 
-To use the Modal Runtime, create an account, and then [create an API key.](https://modal.com/settings)
+To use the Modal Runtime, create an account, and then [create an API key](https://modal.com/settings)
 
 You'll then need to set the following environment variables when starting OpenHands:
 ```bash

docs/package.json

@@ -15,10 +15,10 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.6.3",
-    "@docusaurus/plugin-content-pages": "^3.6.3",
-    "@docusaurus/preset-classic": "^3.6.3",
-    "@docusaurus/theme-mermaid": "^3.6.3",
+    "@docusaurus/core": "^3.6.0",
+    "@docusaurus/plugin-content-pages": "^3.6.0",
+    "@docusaurus/preset-classic": "^3.6.0",
+    "@docusaurus/theme-mermaid": "^3.6.0",
     "@mdx-js/react": "^3.1.0",
     "clsx": "^2.0.0",
     "prism-react-renderer": "^2.4.0",
@@ -29,7 +29,7 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.1",
-    "@docusaurus/tsconfig": "^3.6.3",
+    "@docusaurus/tsconfig": "^3.6.0",
     "@docusaurus/types": "^3.5.1",
     "typescript": "~5.6.3"
   },

evaluation/EDA/game.py

@@ -87,7 +87,9 @@ class Q20Game:
         # others
         bingo, anwser_reply = self.judge_winner(response)
         if bingo:
-            return 'You are bingo! Use the "finish" tool to finish the interaction.\n'
+            return (
+                'You are bingo! quit now, run: <execute_bash> exit </execute_bash>.\n'
+            )
         if self.curr_turn == self.num_turns - 2:
             anwser_reply += " You must guess now, what's it?"
         return anwser_reply

evaluation/aider_bench/README.md

@@ -56,20 +56,6 @@ You can update the arguments in the script
 ./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
 ```
 
-### Run Inference on `RemoteRuntime` (experimental)
-
-This is in limited beta. Contact Xingyao over slack if you want to try this out!
-
-```bash
-./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
-
-# Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
-export ALLHANDS_API_KEY="YOUR-API-KEY"
-export RUNTIME=remote
-export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
-./evaluation/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
-```
-
 ## Summarize Results
 
 ```bash

evaluation/aider_bench/run_infer.py

@@ -58,9 +58,6 @@ def get_config(
             use_host_network=False,
             timeout=100,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
-            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_runtime_alive=False,
-            remote_runtime_init_timeout=1800,
         ),
         # do not mount workspace
         workspace_base=None,

evaluation/biocoder/run_infer.py

@@ -40,7 +40,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 FILE_EXT_MAP = {

evaluation/bird/run_infer.py

@@ -40,7 +40,7 @@ from openhands.utils.async_utils import call_async_from_sync
 def codeact_user_response(state: State) -> str:
     msg = (
         'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have completed the SQL, please finish the interaction using the "finish" tool.\n'
+        'If you think you have completed the SQL, please run the following command: <execute_bash> exit </execute_bash>.\n'
         'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
     )
     if state.history:
@@ -54,7 +54,7 @@ def codeact_user_response(state: State) -> str:
             # let the agent know that it can give up when it has tried 3 times
             return (
                 msg
-                + 'If you want to give up, use the "finish" tool to finish the interaction.\n'
+                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
             )
     return msg
 
@@ -64,7 +64,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 

evaluation/commit0_bench/README.md

@@ -1,82 +0,0 @@
-# Commit0 Evaluation with OpenHands
-
-This folder contains the evaluation harness that we built on top of the original [Commit0](https://commit-0.github.io/) ([paper](TBD)).
-
-The evaluation consists of three steps:
-
-1. Environment setup: [install python environment](../README.md#development-environment), [configure LLM config](../README.md#configure-openhands-and-your-llm).
-2. [Run Evaluation](#run-inference-on-commit0-instances): Generate a edit patch for each Commit0 Repo, and get the evaluation results
-
-## Setup Environment and LLM Configuration
-
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
-
-## OpenHands Commit0 Instance-level Docker Support
-
-OpenHands supports using the Commit0 Docker for **[inference](#run-inference-on-commit0-instances).
-This is now the default behavior.
-
-
-## Run Inference on Commit0 Instances
-
-Make sure your Docker daemon is running, and you have ample disk space (at least 200-500GB, depends on the Commit0 set you are running on) for the [instance-level docker image](#openhands-commit0-instance-level-docker-support).
-
-When the `run_infer.sh` script is started, it will automatically pull the `lite` split in Commit0. For example, for instance ID `commit-0/minitorch`, it will try to pull our pre-build docker image `wentingzhao/minitorch` from DockerHub. This image will be used create an OpenHands runtime image where the agent will operate on.
-
-```bash
-./evaluation/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
-
-# Example
-./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 16 100 8 wentingzhao/commit0_combined test
-```
-
-where `model_config` is mandatory, and the rest are optional.
-
-- `repo_split`, e.g. `lite`, is the split of the Commit0 dataset you would like to evaluate on. Available options are `lite`, `all` and each individual repo.
-- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`.
-- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
-like to evaluate. It could also be a release tag like `0.6.2`.
-- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
-default, the script evaluates the `lite` split of the Commit0 dataset (16 repos). Note:
-in order to use `eval_limit`, you must also set `agent`.
-- `max_iter`, e.g. `20`, is the maximum number of iterations for the agent to run. By
-default, it is set to 30.
-- `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
-default, it is set to 1.
-- `dataset`, a huggingface dataset name. e.g. `wentingzhao/commit0_combined`, specifies which dataset to evaluate on.
-- `dataset_split`, split for the huggingface dataset. Notice only `test` is supported for Commit0.
-
-Note that the `USE_INSTANCE_IMAGE` environment variable is always set to `true` for Commit0.
-
-Let's say you'd like to run 10 instances using `llm.eval_sonnet` and CodeActAgent,
-
-then your command would be:
-
-```bash
-./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
-```
-
-### Run Inference on `RemoteRuntime` (experimental)
-
-This is in limited beta. Contact Xingyao over slack if you want to try this out!
-
-```bash
-./evaluation/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
-
-# Example - This runs evaluation on CodeActAgent for 10 instances on "wentingzhao/commit0_combined"'s test set, with max 30 iteration per instances, with 1 number of workers running in parallel
-ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev" EVAL_DOCKER_IMAGE_PREFIX="docker.io/wentingzhao" \
-./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
-```
-
-To clean-up all existing runtime you've already started, run:
-
-```bash
-ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/commit0_bench/scripts/cleanup_remote_runtime.sh
-```
-
-### Specify a subset of tasks to run infer
-
-If you would like to specify a list of tasks you'd like to benchmark on, you just need to pass selected repo through `repo_split` option.

evaluation/commit0_bench/run_infer.py

@@ -1,606 +0,0 @@
-import asyncio
-import json
-import os
-from collections import Counter
-from typing import Any
-
-import pandas as pd
-from commit0.harness.constants import SPLIT
-from datasets import load_dataset
-
-import openhands.agenthub
-from evaluation.utils.shared import (
-    EvalException,
-    EvalMetadata,
-    EvalOutput,
-    assert_and_raise,
-    codeact_user_response,
-    make_metadata,
-    prepare_dataset,
-    reset_logger_for_multiprocessing,
-    run_evaluation,
-    update_llm_config_for_completions_logging,
-)
-from openhands.controller.state.state import State
-from openhands.core.config import (
-    AgentConfig,
-    AppConfig,
-    SandboxConfig,
-    get_llm_config_arg,
-    get_parser,
-)
-from openhands.core.logger import openhands_logger as logger
-from openhands.core.main import create_runtime, run_controller
-from openhands.events.action import CmdRunAction, MessageAction
-from openhands.events.observation import CmdOutputObservation, ErrorObservation
-from openhands.events.serialization.event import event_to_dict
-from openhands.runtime.base import Runtime
-from openhands.utils.async_utils import call_async_from_sync
-from openhands.utils.shutdown_listener import sleep_if_should_continue
-
-USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
-USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
-RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-    'CodeActCommit0Agent': codeact_user_response,
-}
-
-
-def _get_commit0_workspace_dir_name(instance: pd.Series) -> str:
-    return instance['repo'].split('/')[1]
-
-
-def get_instruction(instance: pd.Series, metadata: EvalMetadata):
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-    # Prepare instruction
-    test_cmd = instance['test']['test_cmd']
-    test_dir = instance['test']['test_dir']
-    # Instruction based on Anthropic's official trajectory
-    # https://github.com/eschluntz/swe-bench-experiments/tree/main/evaluation/verified/20241022_tools_claude-3-5-sonnet-updated/trajs
-    instruction = (
-        '<uploaded_files>\n'
-        f'/workspace/{workspace_dir_name}\n'
-        '</uploaded_files>\n'
-        f"I've uploaded a python code repository in the directory {workspace_dir_name}. Here is your task:\n\n"
-        'Here is your task:\n\n'
-        '  You need to complete the implementations for all functions (i.e., those with pass\n'
-        '  statements) and pass the unit tests.\n\n'
-        '  Do not change the names of existing functions or classes, as they may be referenced\n'
-        '  from other code like unit tests, etc.\n\n'
-        '  When you generate code, you must maintain the original formatting of the function\n'
-        '  stubs (such as whitespaces), otherwise we will not able to search/replace blocks\n'
-        '  for code modifications, and therefore you will receive a score of 0 for your generated\n'
-        '  code.'
-        '\n\n'
-        'Here is the command to run the unit tests:\n'
-        '<test_command>\n'
-        f'{test_cmd} {test_dir}\n'
-        '</test_command>\n\n'
-        'Make a local git commit for each agent step for all code changes. If there is not change in current step, do not make a commit.'
-    )
-
-    if RUN_WITH_BROWSING:
-        instruction += (
-            '<IMPORTANT!>\n'
-            'You SHOULD NEVER attempt to browse the web. '
-            '</IMPORTANT!>\n'
-        )
-    return instruction
-
-
-# TODO: migrate all swe-bench docker to ghcr.io/openhands
-DOCKER_IMAGE_PREFIX = os.environ.get(
-    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/wentingzhao/'
-)
-logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
-
-
-def get_instance_docker_image(repo_name: str) -> str:
-    return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + repo_name).lower() + ':v0'
-
-
-def get_config(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-) -> AppConfig:
-    # COMMIT0_CONTAINER_IMAGE = 'wentingzhao/'
-    assert USE_INSTANCE_IMAGE
-    # We use a different instance image for the each instance of commit0 eval
-    repo_name = instance['repo'].split('/')[1]
-    base_container_image = get_instance_docker_image(repo_name)
-    logger.info(
-        f'Using instance container image: {base_container_image}. '
-        f'Please make sure this image exists. '
-        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
-    )
-    # else:
-    #     raise
-    # base_container_image = SWE_BENCH_CONTAINER_IMAGE
-    # logger.info(f'Using swe-bench container image: {base_container_image}')
-
-    config = AppConfig(
-        default_agent=metadata.agent_class,
-        run_as_openhands=False,
-        max_iterations=metadata.max_iterations,
-        runtime=os.environ.get('RUNTIME', 'eventstream'),
-        sandbox=SandboxConfig(
-            base_container_image=base_container_image,
-            enable_auto_lint=True,
-            use_host_network=False,
-            # large enough timeout, since some testcases take very long to run
-            timeout=300,
-            api_key=os.environ.get('ALLHANDS_API_KEY', None),
-            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            keep_runtime_alive=False,
-            remote_runtime_init_timeout=3600,
-        ),
-        # do not mount workspace
-        workspace_base=None,
-        workspace_mount_path=None,
-    )
-    config.set_llm_config(
-        update_llm_config_for_completions_logging(
-            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
-        )
-    )
-    agent_config = AgentConfig(
-        codeact_enable_jupyter=False,
-        codeact_enable_browsing=RUN_WITH_BROWSING,
-        codeact_enable_llm_editor=False,
-    )
-    config.set_agent_config(agent_config)
-    return config
-
-
-def initialize_runtime(
-    runtime: Runtime,
-    instance: pd.Series,  # this argument is not required
-):
-    """Initialize the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    """
-    logger.info('-' * 30)
-    logger.info('BEGIN Runtime Initialization Fn')
-    logger.info('-' * 30)
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-    obs: CmdOutputObservation
-
-    action = CmdRunAction(
-        command=f'git clone -b commit0_combined https://github.com/{instance["repo"]}.git'
-    )
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to git clone -b commit0_combined https://github.com/{instance["repo"]}.git: {str(obs)}',
-    )
-
-    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
-    )
-
-    action = CmdRunAction(command='git checkout -b openhands')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0, f'Failed to git checkout new branch openhands: {str(obs)}'
-    )
-
-    # Install commit0
-    action = CmdRunAction(command='/root/.cargo/bin/uv pip install commit0')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        obs.exit_code == 0,
-        f'Failed to install commit0: {str(obs)}',
-    )
-    logger.info('-' * 30)
-    logger.info('END Runtime Initialization Fn')
-    logger.info('-' * 30)
-
-
-def complete_runtime(
-    runtime: Runtime,
-    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
-) -> dict[str, Any]:
-    """Complete the runtime for the agent.
-
-    This function is called before the runtime is used to run the agent.
-    If you need to do something in the sandbox to get the correctness metric after
-    the agent has run, modify this function.
-    """
-    logger.info('-' * 30)
-    logger.info('BEGIN Runtime Completion Fn')
-    logger.info('-' * 30)
-    obs: CmdOutputObservation
-    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
-
-    action = CmdRunAction(command='git add .')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
-        f'Failed to git add -A: {str(obs)}',
-    )
-
-    action = CmdRunAction(command='git commit -m "openhands edits"')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation)
-        and (obs.exit_code == 0 or obs.exit_code == 1),
-        f'Failed to git commit -m "openhands": {str(obs)}',
-    )
-
-    # Generate diff patch compared to base commit, excluding spec.pdf.bz2 files
-    n_retries = 0
-    git_patch = None
-    while n_retries < 5:
-        action = CmdRunAction(
-            command=f"git diff {instance['base_commit']} HEAD -- . ':(exclude)spec.pdf.bz2'"
-        )
-        action.timeout = 600 + 100 * n_retries
-        logger.info(action, extra={'msg_type': 'ACTION'})
-        obs = runtime.run_action(action)
-        # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-        n_retries += 1
-        if isinstance(obs, CmdOutputObservation):
-            if obs.exit_code == 0:
-                git_patch = obs.content.strip()
-                break
-            else:
-                logger.info('Failed to get git diff, retrying...')
-                sleep_if_should_continue(10)
-        elif isinstance(obs, ErrorObservation):
-            logger.error(f'Error occurred: {obs.content}. Retrying...')
-            sleep_if_should_continue(10)
-        else:
-            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
-
-    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
-
-    test_dir = instance['test']['test_dir']
-    action = CmdRunAction(
-        command=f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors {test_dir} > test_output.txt 2>&1"
-    )
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to run test command: {str(obs)}',
-    )
-    # Read test output
-    action = CmdRunAction(command='cat test_output.txt')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to read test output: {str(obs)}',
-    )
-    test_output = obs.content.strip()
-    # logger.info(f'Test output: {test_output}')
-
-    # Save pytest exit code
-    action = CmdRunAction(command='echo $?')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
-        f'Failed to save pytest exit code: {str(obs)}',
-    )
-    pytest_exit_code = obs.content.strip()
-    # logger.info(f'Pytest exit code: {pytest_exit_code}')
-
-    # Read the test report
-    action = CmdRunAction(command='cat report.json')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    assert_and_raise(
-        isinstance(obs, CmdOutputObservation),
-        f'Failed to read test report: {str(obs)}',
-    )
-    # Get test IDs from instance
-    repo_name = instance['repo'].split('/')[1]
-    repo_name = repo_name.replace('.', '-')
-    action = CmdRunAction(command=f'commit0 get-tests {repo_name}')
-    action.timeout = 600
-    logger.info(action, extra={'msg_type': 'ACTION'})
-    obs = runtime.run_action(action)
-    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
-    test_ids = obs.content.strip().split('\n')
-
-    try:
-        report = json.loads(obs.content)
-        tests = {x['nodeid']: x['call'] for x in report['tests'] if 'call' in x}
-
-        # Calculate test statistics
-        status = []
-        runtimes = []
-        no_runs = 0
-
-        for test_id in test_ids:
-            if test_id in tests and tests[test_id] is not None:
-                status.append(tests[test_id]['outcome'])
-                runtimes.append(tests[test_id]['duration'])
-                no_runs += 1
-            else:
-                status.append('failed')
-                runtimes.append(0)
-
-        status_counts = Counter(status)
-        total_runtime = sum(runtimes) if no_runs > 0 else 0
-        num_passed = status_counts.get('passed', 0) + status_counts.get('xfail', 0)
-        passed_ratio = num_passed / len(status) if status else 0
-
-        eval_result = {
-            'name': workspace_dir_name,
-            'sum': total_runtime,
-            'passed': passed_ratio,
-            'num_passed': num_passed,
-            'num_tests': len(test_ids),
-        }
-
-    except json.JSONDecodeError:
-        logger.error('Failed to parse test report JSON')
-        eval_result = {
-            'name': workspace_dir_name,
-            'sum': 0,
-            'passed': 0,
-            'num_passed': 0,
-            'num_tests': len(test_ids),
-        }
-
-    # Create tarball of workspace
-    temp_zip = runtime.copy_from(f'/workspace/{workspace_dir_name}')
-
-    commit0_dir = os.path.dirname(__file__)
-    persistent_zip = os.path.join(commit0_dir, f'{workspace_dir_name}.zip')
-    with open(temp_zip, 'rb') as src, open(persistent_zip, 'wb') as dst:
-        dst.write(src.read())
-    zip_file = persistent_zip
-    return {
-        'eval_result': eval_result,
-        'git_patch': git_patch,
-        'test_output': test_output,
-        'pytest_exit_code': pytest_exit_code,
-        'zip_file': zip_file,
-    }
-
-
-def process_instance(
-    instance: pd.Series,
-    metadata: EvalMetadata,
-    reset_logger: bool = True,
-) -> EvalOutput:
-    config = get_config(instance, metadata)
-    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
-    if reset_logger:
-        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
-        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
-    else:
-        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
-
-    runtime = create_runtime(config)
-    call_async_from_sync(runtime.connect)
-    try:
-        initialize_runtime(runtime, instance)
-
-        instruction = get_instruction(instance, metadata)
-
-        # Here's how you can run the agent (similar to the `main` function) and get the final task state
-        state: State | None = asyncio.run(
-            run_controller(
-                config=config,
-                initial_user_action=MessageAction(content=instruction),
-                runtime=runtime,
-                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
-                    metadata.agent_class
-                ],
-            )
-        )
-
-        # if fatal error, throw EvalError to trigger re-run
-        if (
-            state.last_error
-            and 'fatal error during agent execution' in state.last_error
-            and 'stuck in a loop' not in state.last_error
-        ):
-            raise EvalException('Fatal error detected: ' + state.last_error)
-
-        # ======= THIS IS Commit0 specific =======
-        # Get git patch
-        return_val = complete_runtime(runtime, instance)
-        eval_result = return_val['eval_result']
-        git_patch = return_val['git_patch']
-        test_output = return_val['test_output']
-        pytest_exit_code = return_val['pytest_exit_code']
-        zip_file = return_val['zip_file']
-
-        repo_name = instance['repo'].split('/')[1]
-        zip_dest = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}.zip'
-        )
-        patch_file = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_patch.diff'
-        )
-        test_output_file = os.path.join(
-            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_test_output.txt'
-        )
-        pytest_exit_code_file = os.path.join(
-            metadata.eval_output_dir,
-            'repos',
-            repo_name,
-            f'{repo_name}_pytest_exit_code.txt',
-        )
-
-        os.makedirs(os.path.dirname(zip_dest), exist_ok=True)
-        os.rename(zip_file, zip_dest)
-
-        write_targets = [
-            (patch_file, git_patch),
-            (test_output_file, test_output),
-            (pytest_exit_code_file, pytest_exit_code),
-        ]
-
-        for write_target in write_targets:
-            with open(write_target[0], 'w') as f:
-                f.write(write_target[1])
-
-        logger.info(
-            f'Got evaluation result for repo {instance.instance_id}:\n--------\n{eval_result}\n--------'
-        )
-    finally:
-        runtime.close()
-    # ==========================================
-
-    # ======= Attempt to evaluate the agent's edits =======
-    # we use eval_infer.sh to evaluate the agent's edits, not here
-    # because the agent may alter the environment / testcases
-    test_result = {
-        'eval_result': eval_result,
-    }
-
-    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
-    histories = [event_to_dict(event) for event in state.history]
-    metrics = state.metrics.get() if state.metrics else None
-
-    # Save the output
-    output = EvalOutput(
-        instance_id=instance.instance_id,
-        instruction=instruction,
-        instance=instance.to_dict(),
-        test_result=test_result,
-        metadata=metadata,
-        history=histories,
-        metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
-    )
-    return output
-
-
-def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
-    """Setup Commit0 dataset based on split type.
-
-    Args:
-        dataset: Full Commit0 dataset
-        repo_split: Split type ('all', 'lite' or specific repo name)
-
-    Returns:
-        Filtered dataset based on split type
-    """
-
-    filtered_dataset = pd.concat(
-        [
-            dataset[dataset['repo'].str.split('/').str[1] == repo]
-            for repo in SPLIT.get(repo_split, [])
-        ]
-    )
-
-    # Drop setup column if it exists
-    if 'setup' in filtered_dataset.columns:
-        filtered_dataset = filtered_dataset.drop('setup', axis=1)
-
-    # Replace all forward slashes in instance_id with hyphens
-    filtered_dataset['instance_id'] = filtered_dataset['repo'].str.split('/').str[1]
-
-    return filtered_dataset
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--dataset',
-        type=str,
-        default='wentingzhao/commit0_combined',
-        help='dataset to evaluate on, only test split exists for this HF dataset',
-    )
-    parser.add_argument(
-        '--split',
-        type=str,
-        default='test',
-        help='this is the HF dataset split',
-    )
-    parser.add_argument(
-        '--repo-split',
-        type=str,
-        default='lite',
-        help='all, lite, or each repo name',
-    )
-    args, _ = parser.parse_known_args()
-
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenHands's repo
-    dataset = load_dataset(args.dataset, split=args.split)
-
-    commit0_datasets = commit0_setup(dataset.to_pandas(), args.repo_split)
-
-    logger.info(f'Loaded dataset {args.dataset} with reposplit {args.repo_split}')
-
-    llm_config = None
-    if args.llm_config:
-        llm_config = get_llm_config_arg(args.llm_config)
-        llm_config.log_completions = True
-
-    if llm_config is None:
-        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
-
-    details = {}
-    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
-
-    dataset_descrption = (
-        args.dataset.replace('/', '__') + '-' + args.repo_split.replace('/', '__')
-    )
-    metadata = make_metadata(
-        llm_config,
-        dataset_descrption,
-        args.agent_cls,
-        args.max_iterations,
-        args.eval_note,
-        args.eval_output_dir,
-        details=details,
-    )
-
-    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
-
-    instances = prepare_dataset(commit0_datasets, output_file, args.eval_n_limit)
-
-    run_evaluation(
-        instances,
-        metadata,
-        output_file,
-        args.eval_num_workers,
-        process_instance,
-        timeout_seconds=120 * 60,  # 2 hour PER instance should be more than enough
-    )

evaluation/commit0_bench/scripts/cleanup_remote_runtime.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-
-# API base URL
-BASE_URL="https://runtime.eval.all-hands.dev"
-
-# Get the list of runtimes
-response=$(curl --silent --location --request GET "${BASE_URL}/list" \
-  --header "X-API-Key: ${ALLHANDS_API_KEY}")
-
-n_runtimes=$(echo $response | jq -r '.total')
-echo "Found ${n_runtimes} runtimes. Stopping them..."
-
-runtime_ids=$(echo $response | jq -r '.runtimes | .[].runtime_id')
-
-# Function to stop a single runtime
-stop_runtime() {
-  local runtime_id=$1
-  local counter=$2
-  echo "Stopping runtime ${counter}/${n_runtimes}: ${runtime_id}"
-  curl --silent --location --request POST "${BASE_URL}/stop" \
-    --header "X-API-Key: ${ALLHANDS_API_KEY}" \
-    --header "Content-Type: application/json" \
-    --data-raw "{\"runtime_id\": \"${runtime_id}\"}"
-  echo
-}
-export -f stop_runtime
-export BASE_URL ALLHANDS_API_KEY n_runtimes
-
-# Use GNU Parallel to stop runtimes in parallel
-echo "$runtime_ids" | parallel -j 16 --progress stop_runtime {} {#}
-
-echo "All runtimes have been stopped."

evaluation/commit0_bench/scripts/run_infer.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-source "evaluation/utils/version_control.sh"
-
-REPO_SPLIT=$1
-MODEL_CONFIG=$2
-COMMIT_HASH=$3
-AGENT=$4
-EVAL_LIMIT=$5
-MAX_ITER=$6
-NUM_WORKERS=$7
-DATASET=$8
-SPLIT=$9
-N_RUNS=${10}
-
-if [ -z "$NUM_WORKERS" ]; then
-  NUM_WORKERS=1
-  echo "Number of workers not specified, use default $NUM_WORKERS"
-fi
-checkout_eval_branch
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-if [ -z "$MAX_ITER" ]; then
-  echo "MAX_ITER not specified, use default 100"
-  MAX_ITER=100
-fi
-
-if [ -z "$USE_INSTANCE_IMAGE" ]; then
-  echo "USE_INSTANCE_IMAGE not specified, use default true"
-  USE_INSTANCE_IMAGE=true
-fi
-
-if [ -z "$RUN_WITH_BROWSING" ]; then
-  echo "RUN_WITH_BROWSING not specified, use default false"
-  RUN_WITH_BROWSING=false
-fi
-
-
-if [ -z "$DATASET" ]; then
-  echo "DATASET not specified, use default wentingzhao/commit0_combined"
-  DATASET="wentingzhao/commit0_combined"
-fi
-
-if [ -z "$REPO_SPLIT" ]; then
-  echo "REPO_SPLIT not specified, use default lite"
-  REPO_SPLIT=0
-fi
-
-if [ -z "$SPLIT" ]; then
-  echo "HF SPLIT not specified, use default test"
-  SPLIT="test"
-fi
-
-export USE_INSTANCE_IMAGE=$USE_INSTANCE_IMAGE
-echo "USE_INSTANCE_IMAGE: $USE_INSTANCE_IMAGE"
-export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
-echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
-
-get_agent_version
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-echo "DATASET: $DATASET"
-echo "HF SPLIT: $SPLIT"
-echo "REPO SPLIT: $REPO_SPLIT"
-
-# Default to NOT use Hint
-if [ -z "$USE_HINT_TEXT" ]; then
-  export USE_HINT_TEXT=false
-fi
-echo "USE_HINT_TEXT: $USE_HINT_TEXT"
-EVAL_NOTE="$AGENT_VERSION"
-# if not using Hint, add -no-hint to the eval note
-if [ "$USE_HINT_TEXT" = false ]; then
-  EVAL_NOTE="$EVAL_NOTE-no-hint"
-fi
-
-if [ "$RUN_WITH_BROWSING" = true ]; then
-  EVAL_NOTE="$EVAL_NOTE-with-browsing"
-fi
-
-if [ -n "$EXP_NAME" ]; then
-  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
-fi
-
-function run_eval() {
-  local eval_note=$1
-  COMMAND="poetry run python evaluation/commit0_bench/run_infer.py \
-    --agent-cls $AGENT \
-    --llm-config $MODEL_CONFIG \
-    --max-iterations $MAX_ITER \
-    --eval-num-workers $NUM_WORKERS \
-    --eval-note $eval_note \
-    --dataset $DATASET \
-    --split $SPLIT \
-    --repo-split $REPO_SPLIT"
-
-  if [ -n "$EVAL_LIMIT" ]; then
-    echo "EVAL_LIMIT: $EVAL_LIMIT"
-    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-  fi
-
-  # Run the command
-  eval $COMMAND
-}
-
-unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
-if [ -z "$N_RUNS" ]; then
-  N_RUNS=1
-  echo "N_RUNS not specified, use default $N_RUNS"
-fi
-
-for i in $(seq 1 $N_RUNS); do
-  current_eval_note="$EVAL_NOTE-run_$i"
-  echo "EVAL_NOTE: $current_eval_note"
-  run_eval $current_eval_note
-done
-
-checkout_original_branch

evaluation/discoverybench/run_infer.py

@@ -55,7 +55,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 
@@ -250,6 +250,9 @@ def process_instance(
 
     config = get_config(metadata)
 
+    # use a session id for concurrent evaluation
+    sid = 'ID_' + str(instance.instance_id)
+
     # Setup the logger properly, so you can run
     # multi-processing to parallelize the evaluation
     if reset_logger:
@@ -281,7 +284,7 @@ def process_instance(
     instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
 
     # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    runtime = create_runtime(config)
+    runtime = create_runtime(config, sid=sid)
     call_async_from_sync(runtime.connect)
     initialize_runtime(runtime, instance.data_files)
 

evaluation/gorilla/run_infer.py

@@ -33,7 +33,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the request, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have completed the request, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 

evaluation/gpqa/run_infer.py

@@ -87,10 +87,11 @@ def gpqa_codeact_user_response(
     msg = (
         'Please continue working on the task on whatever approach you think is suitable.\n'
         'Feel free to use all tools for calculations and solving the problem, and web-search for finding relevant facts during the process if needed\n'
-        'If you have finished reporting the answer in the expected format, (and only once that is done), please use the "finish" tool to finish the interaction.\n'
+        'If you have finished reporting the answer in the expected format, (and only once that is done), please run the following command to submit: <execute_bash> exit </execute_bash>.\n'
         'Again you are being told a million times to first report the answer in the requested format (see again below for reference) before exiting. DO NOT EXIT WITHOUT REPORTING THE ANSWER FIRST.\n'
         'That is, when you have decided on the answer report in the following format:\n'
         f'{ACTION_FORMAT}\n'
+        '<execute_bash> exit </execute_bash>\n'
         'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP TO SOLVE THIS TASK.\n'
     )
     return msg
@@ -99,7 +100,7 @@ def gpqa_codeact_user_response(
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {'CodeActAgent': gpqa_codeact_user_response}
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': '\n\n SUPER IMPORTANT: When you think you have solved the question, first report it back to the user in the requested format. Only once that is done, in the next turn, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': '\n\n SUPER IMPORTANT: When you think you have solved the question, first report it back to the user in the requested format. Only once that is done, in the next turn, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 
@@ -204,11 +205,12 @@ Additional Instructions:
 - Do not try to solve the question in a single step. Break it down into smaller steps.
 - You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.
 
-- SUPER IMPORTANT: When you have reported the answer to the user in the requested format, (and only once that is done) in the next turn, please finish the interaction using the "finish" tool.
+- SUPER IMPORTANT: When you have reported the answer to the user in the requested format, (and only once that is done) in the next turn, please run the following command: <execute_bash> exit </execute_bash>.
 - Again you are being told a million times to first report the answer in the requested format (see again below for reference) before exiting. DO NOT EXIT WITHOUT REPORTING THE ANSWER FIRST.
     That is, when you have decided on the answer report in the following format:
 
 {ACTION_FORMAT}
+<execute_bash> exit </execute_bash>
 
 Again do not quit without reporting the answer first.
 Ok now its time to start solving the question. Good luck!

evaluation/humanevalfix/README.md

@@ -23,7 +23,7 @@ For each problem, OpenHands is given a set number of iterations to fix the faili
 ```
 {
     "task_id": "Python/2",
-    "instruction": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
+    "instruction": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
     "metadata": {
         "agent_class": "CodeActAgent",
         "model_name": "gpt-4",
@@ -38,10 +38,10 @@ For each problem, OpenHands is given a set number of iterations to fix the faili
                 "id": 27,
                 "timestamp": "2024-05-22T20:57:24.688651",
                 "source": "user",
-                "message": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
+                "message": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
                 "action": "message",
                 "args": {
-                    "content": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n",
+                    "content": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
                     "wait_for_response": false
                 }
             },

evaluation/humanevalfix/run_infer.py

@@ -75,7 +75,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 

evaluation/miniwob/README.md

@@ -16,20 +16,6 @@ Access with browser the above MiniWoB URLs and see if they load correctly.
 ./evaluation/miniwob/scripts/run_infer.sh llm.claude-35-sonnet-eval
 ```
 
-### Run Inference on `RemoteRuntime` (experimental)
-
-This is in limited beta. Contact Xingyao over slack if you want to try this out!
-
-```bash
-./evaluation/miniwob/scripts/run_infer.sh [model_config] [git-version] [agent] [note] [eval_limit] [num_workers]
-
-# Example - This runs evaluation on BrowsingAgent for 125 instances on miniwob, with 2 workers running in parallel
-export ALLHANDS_API_KEY="YOUR-API-KEY"
-export RUNTIME=remote
-export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
-./evaluation/miniwob/scripts/run_infer.sh llm.eval HEAD BrowsingAgent "" 125 2
-```
-
 Results will be in `evaluation/evaluation_outputs/outputs/miniwob/`
 
 To calculate the average reward, run:

evaluation/miniwob/get_avg_reward.py

@@ -23,7 +23,7 @@ if __name__ == '__main__':
             data = json.loads(line)
             actual_num += 1
             total_cost += data['metrics']['accumulated_cost']
-            total_reward += data['test_result']['reward']
+            total_reward += data['test_result']
 
     avg_reward = total_reward / total_num
     print('Avg Reward: ', avg_reward)

evaluation/miniwob/run_infer.py

@@ -47,7 +47,6 @@ SUPPORTED_AGENT_CLS = {'BrowsingAgent', 'CodeActAgent'}
 
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
     'CodeActAgent': codeact_user_response,
-    'BrowsingAgent': 'Continue the task. IMPORTANT: do not talk to the user until you have finished the task',
 }
 
 
@@ -67,9 +66,7 @@ def get_config(
             browsergym_eval_env=env_id,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            remote_runtime_init_timeout=1800,
             keep_runtime_alive=False,
-            timeout=120,
         ),
         # do not mount workspace
         workspace_base=None,

evaluation/miniwob/scripts/run_infer.sh

@@ -33,7 +33,7 @@ echo "MODEL_CONFIG: $MODEL_CONFIG"
 
 EVAL_NOTE="${AGENT_VERSION}_${NOTE}"
 
-COMMAND="export PYTHONPATH=evaluation/miniwob:\$PYTHONPATH && poetry run python evaluation/miniwob/run_infer.py \
+COMMAND="poetry run python evaluation/miniwob/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 10 \

evaluation/mint/run_infer.py

@@ -70,7 +70,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'IMPORTANT: When your answer is confirmed by the user to be correct, you can use the "finish" tool to finish the interaction.\n'
+    'CodeActAgent': '\nIMPORTANT: When your answer is confirmed by the user to be correct, you can exit using the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 with open(os.path.join(os.path.dirname(__file__), 'requirements.txt'), 'r') as f:

evaluation/ml_bench/README.md

@@ -55,7 +55,7 @@ Here's an example of the evaluation output for a single task instance:
 {
   "instance_id": 3,
   "repo": "https://github.com/dmlc/dgl",
-  "instruction": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
+  "instruction": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
   "metadata": {
     "agent_class": "CodeActAgent",
     "model_name": "gpt-4-1106-preview",
@@ -70,10 +70,10 @@ Here's an example of the evaluation output for a single task instance:
         "id": 0,
         "timestamp": "2024-05-26T17:40:41.060009",
         "source": "user",
-        "message": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
+        "message": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
         "action": "message",
         "args": {
-          "content": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please finish the interaction using the "finish" tool.\n",
+          "content": "Please complete the Machine Learning task in the following repository: dgl\n\nThe task is: DGL Implementation of NGCF model\n\nI have a deep desire to embark on a journey brimming with knowledge and expertise. My objective is to train a cutting-edge NGCF Model, known for its unparalleled capabilities, on the illustrious dataset known as gowalla. To ensure swift execution, I kindly request your assistance in crafting the code, making use of the powerful GPU #3 and an embedding size of 32. Can you lend a helping hand to transform this dream into a reality?\n\nYou should create a script named `run.sh` under the specified path in the repo to run the task.\n\nYou can find the task repo at: /workspace/dgl/examples/pytorch/NGCF/NGCF\n\nYou should terminate the subprocess after running the task (e.g., call subprocess.Popen(args).wait()).When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n",
           "wait_for_response": false
         }
       },

evaluation/ml_bench/run_infer.py

@@ -52,7 +52,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the task, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have completed the task, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 ID2CONDA = {

evaluation/swe_bench/eval_infer.py

@@ -84,7 +84,7 @@ def get_config(instance: pd.Series) -> AppConfig:
             timeout=1800,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
-            remote_runtime_init_timeout=3600,
+            remote_runtime_init_timeout=1800,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -263,29 +263,23 @@ def process_instance(
                         test_output_path = os.path.join(log_dir, 'test_output.txt')
                         with open(test_output_path, 'w') as f:
                             f.write(test_output)
-                        try:
-                            _report = get_eval_report(
-                                test_spec=test_spec,
-                                prediction={
-                                    'model_patch': model_patch,
-                                    'instance_id': instance_id,
-                                },
-                                log_path=test_output_path,
-                                include_tests_status=True,
-                            )
-                            report = _report[instance_id]
-                            logger.info(
-                                f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}"
-                            )
-                            instance['test_result']['report']['resolved'] = report[
-                                'resolved'
-                            ]
-                        except Exception as e:
-                            logger.error(
-                                f'[{instance_id}] Error when getting eval report: {e}'
-                            )
-                            instance['test_result']['report']['resolved'] = False
-                            instance['test_result']['report']['error_eval'] = True
+
+                        _report = get_eval_report(
+                            test_spec=test_spec,
+                            prediction={
+                                'model_patch': model_patch,
+                                'instance_id': instance_id,
+                            },
+                            log_path=test_output_path,
+                            include_tests_status=True,
+                        )
+                        report = _report[instance_id]
+                        logger.info(
+                            f"[{instance_id}] report: {report}\nResult for {instance_id}: resolved: {report['resolved']}"
+                        )
+                        instance['test_result']['report']['resolved'] = report[
+                            'resolved'
+                        ]
             else:
                 logger.info(f'[{instance_id}] Error when starting eval:\n{obs.content}')
                 instance['test_result']['report']['error_eval'] = True
@@ -361,7 +355,7 @@ if __name__ == '__main__':
 
     if 'model_patch' not in predictions.columns:
         predictions['model_patch'] = predictions['test_result'].apply(
-            lambda x: x.get('git_patch', '')
+            lambda x: x['git_patch']
         )
     assert {'instance_id', 'model_patch'}.issubset(
         set(predictions.columns)

evaluation/swe_bench/prompt.py

@@ -1,6 +1,6 @@
 CODEACT_SWE_PROMPT = """Now, you're going to solve this issue on your own. Your terminal session has started and you're in the repository's root directory. You can use any bash commands or the special interface to help you. Edit all the files you need to and run any checks or tests that you want.
 Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command.
-When you're satisfied with all of the changes you've made, you can use the "finish" tool to finish the interaction.
+When you're satisfied with all of the changes you've made, you can run the following command: <execute_bash> exit </execute_bash>.
 Note however that you cannot use any interactive session commands (e.g. vim) in this environment, but you can write scripts and run them. E.g. you can write a python script and then run it with `python <script_name>.py`.
 
 NOTE ABOUT THE EDIT COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate indentation before each line!

evaluation/swe_bench/run_infer.py

@@ -146,7 +146,7 @@ def get_config(
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
             remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
             keep_runtime_alive=False,
-            remote_runtime_init_timeout=3600,
+            remote_runtime_init_timeout=1800,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -534,10 +534,5 @@ if __name__ == '__main__':
             instances[col] = instances[col].apply(lambda x: str(x))
 
     run_evaluation(
-        instances,
-        metadata,
-        output_file,
-        args.eval_num_workers,
-        process_instance,
-        timeout_seconds=120 * 60,  # 2 hour PER instance should be more than enough
+        instances, metadata, output_file, args.eval_num_workers, process_instance
     )

evaluation/toolqa/run_infer.py

@@ -34,7 +34,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have completed the request, please finish the interaction using the "finish" tool.\n'
+    'CodeActAgent': 'When you think you have completed the request, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 

evaluation/utils/shared.py

@@ -3,11 +3,9 @@ import logging
 import multiprocessing as mp
 import os
 import pathlib
-import signal
 import subprocess
 import time
 import traceback
-from contextlib import contextmanager
 from typing import Any, Awaitable, Callable, TextIO
 
 import pandas as pd
@@ -94,27 +92,6 @@ class EvalException(Exception):
     pass
 
 
-class EvalTimeoutException(Exception):
-    pass
-
-
-@contextmanager
-def timeout(seconds: int):
-    def timeout_handler(signum, frame):
-        raise EvalTimeoutException(f'Function timed out after {seconds} seconds')
-
-    # Set up the signal handler
-    original_handler = signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(seconds)
-
-    try:
-        yield
-    finally:
-        # Restore the original handler and disable the alarm
-        signal.alarm(0)
-        signal.signal(signal.SIGALRM, original_handler)
-
-
 def codeact_user_response(
     state: State,
     encapsulate_solution: bool = False,
@@ -160,7 +137,7 @@ def codeact_user_response(
             # let the agent know that it can give up when it has tried 3 times
             return (
                 msg
-                + 'If you want to give up, use the "finish" tool to finish the interaction.\n'
+                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
             )
     return msg
 
@@ -303,33 +280,15 @@ def _process_instance_wrapper(
     metadata: EvalMetadata,
     use_mp: bool,
     max_retries: int = 5,
-    timeout_seconds: int | None = None,
 ) -> EvalOutput:
-    """Wrap the process_instance_func to handle retries and errors."""
+    """Wrap the process_instance_func to handle retries and errors.
+
+    Retry an instance up to max_retries times if it fails (e.g., due to transient network/runtime issues).
+    """
     for attempt in range(max_retries + 1):
         try:
-            if timeout_seconds is not None:
-                with timeout(timeout_seconds):
-                    result = process_instance_func(instance, metadata, use_mp)
-            else:
-                result = process_instance_func(instance, metadata, use_mp)
+            result = process_instance_func(instance, metadata, use_mp)
             return result
-        except EvalTimeoutException as e:
-            error = f'Timeout after {timeout_seconds} seconds'
-            stacktrace = traceback.format_exc()
-            msg = (
-                '-' * 10
-                + '\n'
-                + f'Timeout ({timeout_seconds} seconds) in instance [{instance.instance_id}], Stopped evaluation for this instance.'
-                + '\n'
-                + '-' * 10
-            )
-            logger.exception(e)
-            return EvalOutput(
-                instance_id=instance.instance_id,
-                test_result={},
-                error=error,
-            )
         except Exception as e:
             error = str(e)
             stacktrace = traceback.format_exc()
@@ -378,7 +337,6 @@ def run_evaluation(
         [pd.Series, EvalMetadata, bool], Awaitable[EvalOutput]
     ],
     max_retries: int = 5,  # number of retries for each instance
-    timeout_seconds: int | None = None,
 ):
     use_multiprocessing = num_workers > 1
 
@@ -399,14 +357,7 @@ def run_evaluation(
         if use_multiprocessing:
             with mp.Pool(num_workers) as pool:
                 args_iter = (
-                    (
-                        process_instance_func,
-                        instance,
-                        metadata,
-                        True,
-                        max_retries,
-                        timeout_seconds,
-                    )
+                    (process_instance_func, instance, metadata, True, max_retries)
                     for _, instance in dataset.iterrows()
                 )
                 results = pool.imap_unordered(_process_instance_wrapper_mp, args_iter)

frontend/.eslintrc

@@ -10,8 +10,7 @@
     "plugin:@typescript-eslint/eslint-recommended",
     "plugin:@typescript-eslint/recommended",
     "plugin:react/recommended",
-    "plugin:react-hooks/recommended",
-    "plugin:@tanstack/query/recommended"
+    "plugin:react-hooks/recommended"
   ],
   "plugins": [
     "prettier"

frontend/__tests__/clear-session.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { clearSession } from "../src/utils/clear-session";
+import store from "../src/store";
+import { initialState as browserInitialState } from "../src/state/browserSlice";
+
+describe("clearSession", () => {
+  beforeEach(() => {
+    // Mock localStorage
+    const localStorageMock = {
+      getItem: vi.fn(),
+      setItem: vi.fn(),
+      removeItem: vi.fn(),
+      clear: vi.fn(),
+    };
+    vi.stubGlobal("localStorage", localStorageMock);
+
+    // Set initial browser state to non-default values
+    store.dispatch({
+      type: "browser/setUrl",
+      payload: "https://example.com",
+    });
+    store.dispatch({
+      type: "browser/setScreenshotSrc",
+      payload: "base64screenshot",
+    });
+  });
+
+  it("should clear localStorage and reset browser state", () => {
+    clearSession();
+
+    // Verify localStorage items were removed
+    expect(localStorage.removeItem).toHaveBeenCalledWith("token");
+    expect(localStorage.removeItem).toHaveBeenCalledWith("repo");
+
+    // Verify browser state was reset
+    const state = store.getState();
+    expect(state.browser.url).toBe(browserInitialState.url);
+    expect(state.browser.screenshotSrc).toBe(browserInitialState.screenshotSrc);
+  });
+});

frontend/__tests__/components/Browser.test.tsx

@@ -1,7 +1,7 @@
 import { screen } from "@testing-library/react";
 import { describe, it, expect } from "vitest";
 import { renderWithProviders } from "../../test-utils";
-import BrowserPanel from "#/components/browser";
+import BrowserPanel from "#/components/Browser";
 
 describe("Browser", () => {
   it("renders a message if no screenshotSrc is provided", () => {

frontend/__tests__/components/chat/chat-input.test.tsx

@@ -171,9 +171,9 @@ describe("ChatInput", () => {
     // Fire paste event with text data
     fireEvent.paste(input!, {
       clipboardData: {
-        getData: (type: string) => (type === "text/plain" ? "test paste" : ""),
-        files: [],
-      },
+        getData: (type: string) => type === 'text/plain' ? 'test paste' : '',
+        files: []
+      }
     });
   });
 
@@ -187,16 +187,14 @@ describe("ChatInput", () => {
     expect(input).toBeTruthy();
 
     // Create a paste event with an image file
-    const file = new File(["dummy content"], "image.png", {
-      type: "image/png",
-    });
+    const file = new File(["dummy content"], "image.png", { type: "image/png" });
 
     // Fire paste event with image data
     fireEvent.paste(input!, {
       clipboardData: {
-        getData: () => "",
-        files: [file],
-      },
+        getData: () => '',
+        files: [file]
+      }
     });
 
     // Verify image paste was handled

frontend/__tests__/components/chat/chat-interface.test.tsx

@@ -3,9 +3,9 @@ import { act, screen, waitFor, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { renderWithProviders } from "test-utils";
 import { ChatInterface } from "#/components/chat-interface";
-import { addUserMessage } from "#/state/chat-slice";
+import { addUserMessage } from "#/state/chatSlice";
 import { SUGGESTIONS } from "#/utils/suggestions";
-import * as ChatSlice from "#/state/chat-slice";
+import * as ChatSlice from "#/state/chatSlice";
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const renderChatInterface = (messages: (Message | ErrorMessage)[]) =>
@@ -21,11 +21,6 @@ describe("Empty state", () => {
   }));
 
   beforeAll(() => {
-    vi.mock("@remix-run/react", async (importActual) => ({
-      ...(await importActual<typeof import("@remix-run/react")>()),
-      useRouteLoaderData: vi.fn(() => ({})),
-    }));
-
     vi.mock("#/context/socket", async (importActual) => ({
       ...(await importActual<typeof import("#/context/ws-client-provider")>()),
       useWsClient: useWsClientMock,

frontend/__tests__/components/feedback-form.test.tsx

@@ -1,7 +1,6 @@
 import { render, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { afterEach, describe, expect, it, vi } from "vitest";
-import { renderWithProviders } from "test-utils";
 import { FeedbackForm } from "#/components/feedback-form";
 
 describe("FeedbackForm", () => {
@@ -13,9 +12,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should render correctly", () => {
-    renderWithProviders(
-      <FeedbackForm polarity="positive" onClose={onCloseMock} />,
-    );
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
 
     screen.getByLabelText("Email");
     screen.getByLabelText("Private");
@@ -26,9 +23,7 @@ describe("FeedbackForm", () => {
   });
 
   it("should switch between private and public permissions", async () => {
-    renderWithProviders(
-      <FeedbackForm polarity="positive" onClose={onCloseMock} />,
-    );
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
     const privateRadio = screen.getByLabelText("Private");
     const publicRadio = screen.getByLabelText("Public");
 
@@ -45,11 +40,10 @@ describe("FeedbackForm", () => {
   });
 
   it("should call onClose when the close button is clicked", async () => {
-    renderWithProviders(
-      <FeedbackForm polarity="positive" onClose={onCloseMock} />,
-    );
+    render(<FeedbackForm polarity="positive" onClose={onCloseMock} />);
     await user.click(screen.getByRole("button", { name: "Cancel" }));
 
     expect(onCloseMock).toHaveBeenCalled();
   });
+
 });

frontend/__tests__/components/file-explorer/ExplorerTree.test.tsx

@@ -1,7 +1,7 @@
 import { screen } from "@testing-library/react";
 import { renderWithProviders } from "test-utils";
 import { describe, afterEach, vi, it, expect } from "vitest";
-import ExplorerTree from "#/components/file-explorer/explorer-tree";
+import ExplorerTree from "#/components/file-explorer/ExplorerTree";
 
 const FILES = ["file-1-1.ts", "folder-1-2"];
 

frontend/__tests__/components/file-explorer/FileExplorer.test.tsx

@@ -3,8 +3,8 @@ import userEvent from "@testing-library/user-event";
 import { renderWithProviders } from "test-utils";
 import { describe, it, expect, vi, Mock, afterEach } from "vitest";
 import toast from "#/utils/toast";
-import AgentState from "#/types/agent-state";
-import FileExplorer from "#/components/file-explorer/file-explorer";
+import AgentState from "#/types/AgentState";
+import FileExplorer from "#/components/file-explorer/FileExplorer";
 import OpenHands from "#/api/open-hands";
 
 const toastSpy = vi.spyOn(toast, "error");
@@ -16,13 +16,16 @@ vi.mock("../../services/fileService", async () => ({
 }));
 
 const renderFileExplorerWithRunningAgentState = () =>
-  renderWithProviders(<FileExplorer isOpen onToggle={() => {}} />, {
-    preloadedState: {
-      agent: {
-        curAgentState: AgentState.RUNNING,
+  renderWithProviders(
+    <FileExplorer error={null} isOpen onToggle={() => {}} />,
+    {
+      preloadedState: {
+        agent: {
+          curAgentState: AgentState.RUNNING,
+        },
       },
     },
-  });
+  );
 
 describe.skip("FileExplorer", () => {
   afterEach(() => {

frontend/__tests__/components/file-explorer/TreeNode.test.tsx

@@ -2,7 +2,7 @@ import { screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { renderWithProviders } from "test-utils";
 import { vi, describe, afterEach, it, expect } from "vitest";
-import TreeNode from "#/components/file-explorer/tree-node";
+import TreeNode from "#/components/file-explorer/TreeNode";
 import OpenHands from "#/api/open-hands";
 
 const getFileSpy = vi.spyOn(OpenHands, "getFile");

frontend/__tests__/components/modals/ConnectToGitHubByTokenModal.test.tsx

@@ -0,0 +1,6 @@
+import { describe, it } from "vitest";
+
+describe("ConnectToGitHubByTokenModal", () => {
+  it.todo("should render the form");
+  it.todo("should set the token in local storage");
+});

frontend/__tests__/components/modals/base-modal/BaseModal.test.tsx

@@ -1,7 +1,7 @@
 import { render, screen, act } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { describe, it, vi, expect } from "vitest";
-import BaseModal from "#/components/modals/base-modal/base-modal";
+import BaseModal from "#/components/modals/base-modal/BaseModal";
 
 describe("BaseModal", () => {
   it("should render if the modal is open", () => {

frontend/__tests__/components/modals/settings/ModelSelector.test.tsx

@@ -1,7 +1,7 @@
 import { describe, it, expect } from "vitest";
 import { render, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
-import { ModelSelector } from "#/components/modals/settings/model-selector";
+import { ModelSelector } from "#/components/modals/settings/ModelSelector";
 
 describe("ModelSelector", () => {
   const models = {

frontend/__tests__/components/terminal/Terminal.test.tsx

@@ -1,8 +1,8 @@
 import { act, screen } from "@testing-library/react";
 import { renderWithProviders } from "test-utils";
 import { vi, describe, afterEach, it, expect } from "vitest";
-import { Command, appendInput, appendOutput } from "#/state/command-slice";
-import Terminal from "#/components/terminal/terminal";
+import { Command, appendInput, appendOutput } from "#/state/commandSlice";
+import Terminal from "#/components/terminal/Terminal";
 
 global.ResizeObserver = vi.fn().mockImplementation(() => ({
   observe: vi.fn(),

frontend/__tests__/components/user-actions.test.tsx

@@ -1,6 +1,7 @@
 import { render, screen } from "@testing-library/react";
 import { describe, expect, it, test, vi, afterEach } from "vitest";
 import userEvent from "@testing-library/user-event";
+import * as Remix from "@remix-run/react";
 import { UserActions } from "#/components/user-actions";
 
 describe("UserActions", () => {
@@ -8,9 +9,14 @@ describe("UserActions", () => {
   const onClickAccountSettingsMock = vi.fn();
   const onLogoutMock = vi.fn();
 
+  const useFetcherSpy = vi.spyOn(Remix, "useFetcher");
+  // @ts-expect-error - Only returning the relevant properties for the test
+  useFetcherSpy.mockReturnValue({ state: "idle" });
+
   afterEach(() => {
     onClickAccountSettingsMock.mockClear();
     onLogoutMock.mockClear();
+    useFetcherSpy.mockClear();
   });
 
   it("should render", () => {
@@ -105,8 +111,10 @@ describe("UserActions", () => {
     expect(onLogoutMock).not.toHaveBeenCalled();
   });
 
-  // FIXME: Spinner now provided through useQuery
-  it.skip("should display the loading spinner", () => {
+  it("should display the loading spinner", () => {
+    // @ts-expect-error - Only returning the relevant properties for the test
+    useFetcherSpy.mockReturnValue({ state: "loading" });
+
     render(
       <UserActions
         onClickAccountSettings={onClickAccountSettingsMock}

frontend/__tests__/hooks/use-click-outside-element.test.tsx

@@ -1,7 +1,7 @@
 import { render, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { expect, test, vi } from "vitest";
-import { useClickOutsideElement } from "#/hooks/use-click-outside-element";
+import { useClickOutsideElement } from "#/hooks/useClickOutsideElement";
 
 interface ClickOutsideTestComponentProps {
   callback: () => void;

frontend/__tests__/hooks/use-rate.test.ts

@@ -1,93 +0,0 @@
-import { act, renderHook } from "@testing-library/react";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { useRate } from "#/utils/use-rate";
-
-describe("useRate", () => {
-  beforeEach(() => {
-    vi.useFakeTimers();
-  });
-
-  afterEach(() => {
-    vi.useRealTimers();
-  });
-
-  it("should initialize", () => {
-    const { result } = renderHook(() => useRate());
-
-    expect(result.current.items).toHaveLength(0);
-    expect(result.current.rate).toBeNull();
-    expect(result.current.lastUpdated).toBeNull();
-    expect(result.current.isUnderThreshold).toBe(true);
-  });
-
-  it("should handle the case of a single element", () => {
-    const { result } = renderHook(() => useRate());
-
-    act(() => {
-      result.current.record(123);
-    });
-
-    expect(result.current.items).toHaveLength(1);
-    expect(result.current.lastUpdated).not.toBeNull();
-  });
-
-  it("should return the difference between the last two elements", () => {
-    const { result } = renderHook(() => useRate());
-
-    vi.setSystemTime(500);
-    act(() => {
-      result.current.record(4);
-    });
-
-    vi.advanceTimersByTime(500);
-    act(() => {
-      result.current.record(9);
-    });
-
-    expect(result.current.items).toHaveLength(2);
-    expect(result.current.rate).toBe(5);
-    expect(result.current.lastUpdated).toBe(1000);
-  });
-
-  it("should update isUnderThreshold after [threshold]ms of no activity", () => {
-    const { result } = renderHook(() => useRate({ threshold: 500 }));
-
-    expect(result.current.isUnderThreshold).toBe(true);
-
-    act(() => {
-      // not sure if fake timers is buggy with intervals,
-      // but I need to call it twice to register
-      vi.advanceTimersToNextTimer();
-      vi.advanceTimersToNextTimer();
-    });
-
-    expect(result.current.isUnderThreshold).toBe(false);
-  });
-
-  it("should return an isUnderThreshold boolean", () => {
-    const { result } = renderHook(() => useRate({ threshold: 500 }));
-
-    vi.setSystemTime(500);
-    act(() => {
-      result.current.record(400);
-    });
-    act(() => {
-      result.current.record(1000);
-    });
-
-    expect(result.current.isUnderThreshold).toBe(false);
-
-    act(() => {
-      result.current.record(1500);
-    });
-
-    expect(result.current.isUnderThreshold).toBe(true);
-
-    act(() => {
-      vi.advanceTimersToNextTimer();
-      vi.advanceTimersToNextTimer();
-    });
-
-    expect(result.current.isUnderThreshold).toBe(false);
-  });
-});

frontend/__tests__/hooks/use-terminal.test.tsx

@@ -1,10 +1,10 @@
 import { beforeAll, describe, expect, it, vi } from "vitest";
 import { render } from "@testing-library/react";
 import { afterEach } from "node:test";
-import { ReactNode } from "react";
-import { useTerminal } from "#/hooks/use-terminal";
-import { Command } from "#/state/command-slice";
+import { useTerminal } from "#/hooks/useTerminal";
+import { Command } from "#/state/commandSlice";
 import { WsClientProvider } from "#/context/ws-client-provider";
+import { ReactNode } from "react";
 
 interface TestTerminalComponentProps {
   commands: Command[];
@@ -23,17 +23,11 @@ interface WrapperProps {
   children: ReactNode;
 }
 
-function Wrapper({ children }: WrapperProps) {
+
+function Wrapper({children}: WrapperProps) {
   return (
-    <WsClientProvider
-      enabled
-      token="NO_JWT"
-      ghToken="NO_GITHUB"
-      settings={null}
-    >
-      {children}
-    </WsClientProvider>
-  );
+    <WsClientProvider enabled={true} token="NO_JWT" ghToken="NO_GITHUB" settings={null}>{children}</WsClientProvider>
+  )
 }
 
 describe("useTerminal", () => {

frontend/__tests__/initial-query.test.tsx

@@ -1,9 +1,6 @@
 import { describe, it, expect } from "vitest";
 import store from "../src/store";
-import {
-  setInitialQuery,
-  clearInitialQuery,
-} from "../src/state/initial-query-slice";
+import { setInitialQuery, clearInitialQuery } from "../src/state/initial-query-slice";
 
 describe("Initial Query Behavior", () => {
   it("should clear initial query when clearInitialQuery is dispatched", () => {

frontend/__tests__/routes/_oh.test.tsx

@@ -1,153 +1,35 @@
-import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
-import { createRemixStub } from "@remix-run/testing";
-import { screen, waitFor, within } from "@testing-library/react";
-import { renderWithProviders } from "test-utils";
-import userEvent from "@testing-library/user-event";
-import MainApp from "#/routes/_oh";
-import * as CaptureConsent from "#/utils/handle-capture-consent";
-import i18n from "#/i18n";
+import { describe, it, test } from "vitest";
 
 describe("frontend/routes/_oh", () => {
-  const RemixStub = createRemixStub([{ Component: MainApp, path: "/" }]);
-
-  const { userIsAuthenticatedMock, settingsAreUpToDateMock } = vi.hoisted(
-    () => ({
-      userIsAuthenticatedMock: vi.fn(),
-      settingsAreUpToDateMock: vi.fn(),
-    }),
-  );
-
-  beforeAll(() => {
-    vi.mock("#/utils/user-is-authenticated", () => ({
-      userIsAuthenticated: userIsAuthenticatedMock.mockReturnValue(true),
-    }));
-
-    vi.mock("#/services/settings", async (importOriginal) => ({
-      ...(await importOriginal<typeof import("#/services/settings")>()),
-      settingsAreUpToDate: settingsAreUpToDateMock,
-    }));
-  });
-
-  afterEach(() => {
-    vi.clearAllMocks();
-    localStorage.clear();
-  });
-
-  it("should render", async () => {
-    renderWithProviders(<RemixStub />);
-    await screen.findByTestId("root-layout");
-  });
-
-  it("should render the AI config modal if the user is authed", async () => {
-    // Our mock return value is true by default
-    renderWithProviders(<RemixStub />);
-    await screen.findByTestId("ai-config-modal");
-  });
-
-  it("should render the AI config modal if settings are not up-to-date", async () => {
-    settingsAreUpToDateMock.mockReturnValue(false);
-    renderWithProviders(<RemixStub />);
-
-    await screen.findByTestId("ai-config-modal");
-  });
-
-  it("should not render the AI config modal if the settings are up-to-date", async () => {
-    settingsAreUpToDateMock.mockReturnValue(true);
-    renderWithProviders(<RemixStub />);
-
-    await waitFor(() => {
-      expect(screen.queryByTestId("ai-config-modal")).not.toBeInTheDocument();
-    });
-  });
-
-  it("should capture the user's consent", async () => {
-    const user = userEvent.setup();
-    const handleCaptureConsentSpy = vi.spyOn(
-      CaptureConsent,
-      "handleCaptureConsent",
+  describe("brand logo", () => {
+    it.todo("should not do anything if the user is in the main screen");
+    it.todo(
+      "should be clickable and redirect to the main screen if the user is not in the main screen",
     );
-
-    renderWithProviders(<RemixStub />);
-
-    // The user has not consented to tracking
-    const consentForm = await screen.findByTestId("user-capture-consent-form");
-    expect(handleCaptureConsentSpy).not.toHaveBeenCalled();
-    expect(localStorage.getItem("analytics-consent")).toBeNull();
-
-    const submitButton = within(consentForm).getByRole("button", {
-      name: /confirm preferences/i,
-    });
-    await user.click(submitButton);
-
-    // The user has now consented to tracking
-    expect(handleCaptureConsentSpy).toHaveBeenCalledWith(true);
-    expect(localStorage.getItem("analytics-consent")).toBe("true");
-    expect(
-      screen.queryByTestId("user-capture-consent-form"),
-    ).not.toBeInTheDocument();
   });
 
-  it("should not render the user consent form if the user has already made a decision", async () => {
-    localStorage.setItem("analytics-consent", "true");
-    renderWithProviders(<RemixStub />);
+  describe("user menu", () => {
+    it.todo("should open the user menu when clicked");
 
-    await waitFor(() => {
-      expect(
-        screen.queryByTestId("user-capture-consent-form"),
-      ).not.toBeInTheDocument();
+    describe("logged out", () => {
+      it.todo("should display a placeholder");
+      test.todo("the logout option in the user menu should be disabled");
+    });
+
+    describe("logged in", () => {
+      it.todo("should display the user's avatar");
+      it.todo("should log the user out when the logout option is clicked");
     });
   });
 
-  it("should render a new project button if a token is set", async () => {
-    localStorage.setItem("token", "test-token");
-    const { rerender } = renderWithProviders(<RemixStub />);
-
-    await screen.findByTestId("new-project-button");
-
-    localStorage.removeItem("token");
-    rerender(<RemixStub />);
-
-    await waitFor(() => {
-      expect(
-        screen.queryByTestId("new-project-button"),
-      ).not.toBeInTheDocument();
-    });
-  });
-
-  // TODO: Move to e2e tests
-  it.skip("should update the i18n language when the language settings change", async () => {
-    const changeLanguageSpy = vi.spyOn(i18n, "changeLanguage");
-    const { rerender } = renderWithProviders(<RemixStub />);
-
-    // The default language is English
-    expect(changeLanguageSpy).toHaveBeenCalledWith("en");
-
-    localStorage.setItem("LANGUAGE", "es");
-
-    rerender(<RemixStub />);
-    expect(changeLanguageSpy).toHaveBeenCalledWith("es");
-
-    rerender(<RemixStub />);
-    // The language has not changed, so the spy should not have been called again
-    expect(changeLanguageSpy).toHaveBeenCalledTimes(2);
-  });
-
-  // FIXME: logoutCleanup has been replaced with a hook
-  it.skip("should call logoutCleanup after a logout", async () => {
-    const user = userEvent.setup();
-    localStorage.setItem("ghToken", "test-token");
-
-    // const logoutCleanupSpy = vi.spyOn(LogoutCleanup, "logoutCleanup");
-    renderWithProviders(<RemixStub />);
-
-    const userActions = await screen.findByTestId("user-actions");
-    const userAvatar = within(userActions).getByTestId("user-avatar");
-    await user.click(userAvatar);
-
-    const logout = within(userActions).getByRole("button", { name: /logout/i });
-    await user.click(logout);
-
-    // expect(logoutCleanupSpy).toHaveBeenCalled();
-    expect(localStorage.getItem("ghToken")).toBeNull();
+  describe("config", () => {
+    it.todo("should open the config modal when clicked");
+    it.todo(
+      "should not save the config and close the config modal when the close button is clicked",
+    );
+    it.todo(
+      "should save the config when the save button is clicked and close the modal",
+    );
+    it.todo("should warn the user about saving the config when in /app");
   });
 });

frontend/__tests__/utils/cache.test.ts

@@ -0,0 +1,53 @@
+import { afterEach } from "node:test";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { cache } from "#/utils/cache";
+
+describe("Cache", () => {
+  const testKey = "key";
+  const testData = { message: "Hello, world!" };
+  const testTTL = 1000; // 1 second
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("gets data from memory if not expired", () => {
+    cache.set(testKey, testData, testTTL);
+
+    expect(cache.get(testKey)).toEqual(testData);
+  });
+
+  it("should expire after 5 minutes by default", () => {
+    cache.set(testKey, testData);
+    expect(cache.get(testKey)).not.toBeNull();
+
+    vi.advanceTimersByTime(5 * 60 * 1000 + 1);
+
+    expect(cache.get(testKey)).toBeNull();
+  });
+
+  it("returns null if cached data is expired", () => {
+    cache.set(testKey, testData, testTTL);
+
+    vi.advanceTimersByTime(testTTL + 1);
+    expect(cache.get(testKey)).toBeNull();
+  });
+
+  it("deletes data from memory", () => {
+    cache.set(testKey, testData, testTTL);
+    cache.delete(testKey);
+    expect(cache.get(testKey)).toBeNull();
+  });
+
+  it("clears all data with the app prefix from memory", () => {
+    cache.set(testKey, testData, testTTL);
+    cache.set("anotherKey", { data: "More data" }, testTTL);
+    cache.clearAll();
+    expect(cache.get(testKey)).toBeNull();
+    expect(cache.get("anotherKey")).toBeNull();
+  });
+});

frontend/__tests__/utils/extract-next-page-from-link.test.ts

@@ -1,13 +0,0 @@
-import { expect, test } from "vitest";
-import { extractNextPageFromLink } from "#/utils/extract-next-page-from-link";
-
-test("extractNextPageFromLink", () => {
-  const link = `<https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=4>; rel="next", <https://api.github.com/repositories/1300192/issues?page=515>; rel="last", <https://api.github.com/repositories/1300192/issues?page=1>; rel="first"`;
-  expect(extractNextPageFromLink(link)).toBe(4);
-
-  const noNextLink = `<https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=1>; rel="first"`;
-  expect(extractNextPageFromLink(noNextLink)).toBeNull();
-
-  const extra = `<https://api.github.com/user/repos?sort=pushed&page=2&per_page=3>; rel="next", <https://api.github.com/user/repos?sort=pushed&page=22&per_page=3>; rel="last"`;
-  expect(extractNextPageFromLink(extra)).toBe(2);
-});

frontend/__tests__/utils/extractModelAndProvider.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { extractModelAndProvider } from "../../src/utils/extract-model-and-provider";
+import { extractModelAndProvider } from "../../src/utils/extractModelAndProvider";
 
 describe("extractModelAndProvider", () => {
   it("should work", () => {

frontend/__tests__/utils/formatMs.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "vitest";
-import { formatMs } from "../../src/utils/format-ms";
+import { formatMs } from "../../src/utils/formatMs";
 
 test("formatMs", () => {
   expect(formatMs(1000)).toBe("00:01");

frontend/__tests__/utils/handle-capture-consent.test.ts

@@ -1,44 +0,0 @@
-import posthog from "posthog-js";
-import { afterEach, describe, expect, it, vi } from "vitest";
-import { handleCaptureConsent } from "#/utils/handle-capture-consent";
-
-describe("handleCaptureConsent", () => {
-  const optInSpy = vi.spyOn(posthog, "opt_in_capturing");
-  const optOutSpy = vi.spyOn(posthog, "opt_out_capturing");
-  const hasOptedInSpy = vi.spyOn(posthog, "has_opted_in_capturing");
-  const hasOptedOutSpy = vi.spyOn(posthog, "has_opted_out_capturing");
-
-  afterEach(() => {
-    vi.clearAllMocks();
-  });
-
-  it("should opt out of of capturing", () => {
-    handleCaptureConsent(false);
-
-    expect(optOutSpy).toHaveBeenCalled();
-    expect(optInSpy).not.toHaveBeenCalled();
-  });
-
-  it("should opt in to capturing if the user consents", () => {
-    handleCaptureConsent(true);
-
-    expect(optInSpy).toHaveBeenCalled();
-    expect(optOutSpy).not.toHaveBeenCalled();
-  });
-
-  it("should not opt in to capturing if the user is already opted in", () => {
-    hasOptedInSpy.mockReturnValueOnce(true);
-    handleCaptureConsent(true);
-
-    expect(optInSpy).not.toHaveBeenCalled();
-    expect(optOutSpy).not.toHaveBeenCalled();
-  });
-
-  it("should not opt out of capturing if the user is already opted out", () => {
-    hasOptedOutSpy.mockReturnValueOnce(true);
-    handleCaptureConsent(false);
-
-    expect(optOutSpy).not.toHaveBeenCalled();
-    expect(optInSpy).not.toHaveBeenCalled();
-  });
-});

frontend/__tests__/utils/isNumber.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "vitest";
-import { isNumber } from "../../src/utils/is-number";
+import { isNumber } from "../../src/utils/isNumber";
 
 test("isNumber", () => {
   expect(isNumber(1)).toBe(true);

frontend/__tests__/utils/mapProvider.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "vitest";
-import { mapProvider } from "../../src/utils/map-provider";
+import { mapProvider } from "../../src/utils/mapProvider";
 
 test("mapProvider", () => {
   expect(mapProvider("azure")).toBe("Azure");

frontend/__tests__/utils/organizeModelsAndProviders.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { organizeModelsAndProviders } from "../../src/utils/organize-models-and-providers";
+import { organizeModelsAndProviders } from "../../src/utils/organizeModelsAndProviders";
 
 test("organizeModelsAndProviders", () => {
   const models = [

frontend/__tests__/utils/parseGithubUrl.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { parseGithubUrl } from "../../src/utils/parse-github-url";
+import { parseGithubUrl } from "../../src/utils/parseGithubUrl";
 
 test("parseGithubUrl", () => {
   expect(

frontend/__tests__/utils/parseTerminalOutput.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { parseTerminalOutput } from "../../src/utils/parse-terminal-output";
+import { parseTerminalOutput } from "../../src/utils/parseTerminalOutput";
 
 describe("parseTerminalOutput", () => {
   it("should parse the command, env, and symbol", () => {

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openhands-frontend",
-  "version": "0.14.2",
+  "version": "0.13.1",
   "private": true,
   "type": "module",
   "engines": {
@@ -14,7 +14,6 @@
     "@remix-run/node": "^2.11.2",
     "@remix-run/react": "^2.11.2",
     "@remix-run/serve": "^2.11.2",
-    "@tanstack/react-query": "^5.60.5",
     "@vitejs/plugin-react": "^4.3.2",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.4.0",
@@ -40,6 +39,7 @@
     "react-textarea-autosize": "^8.5.4",
     "remark-gfm": "^4.0.0",
     "sirv-cli": "^3.0.0",
+    "socket.io-client": "^4.8.1",
     "tailwind-merge": "^2.5.4",
     "vite": "^5.4.9",
     "web-vitals": "^3.5.2",
@@ -77,7 +77,6 @@
     "@remix-run/dev": "^2.11.2",
     "@remix-run/testing": "^2.11.2",
     "@tailwindcss/typography": "^0.5.15",
-    "@tanstack/eslint-plugin-query": "^5.60.1",
     "@testing-library/jest-dom": "^6.6.1",
     "@testing-library/react": "^16.0.1",
     "@testing-library/user-event": "^14.5.2",
@@ -97,9 +96,9 @@
     "eslint-config-airbnb-typescript": "^18.0.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jsx-a11y": "^6.10.2",
+    "eslint-plugin-jsx-a11y": "^6.9.0",
     "eslint-plugin-prettier": "^5.2.1",
-    "eslint-plugin-react": "^7.37.2",
+    "eslint-plugin-react": "^7.35.0",
     "eslint-plugin-react-hooks": "^4.6.2",
     "husky": "^9.1.6",
     "jsdom": "^25.0.1",

frontend/playwright.config.ts

@@ -26,7 +26,7 @@ export default defineConfig({
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */
-    baseURL: "http://localhost:3001/",
+    baseURL: "http://127.0.0.1:3000",
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: "on-first-retry",
@@ -72,8 +72,8 @@ export default defineConfig({
 
   /* Run your local dev server before starting the tests */
   webServer: {
-    command: "npm run dev:mock -- --port 3001",
-    url: "http://localhost:3001/",
+    command: "npm run dev:mock -- --port 3000",
+    url: "http://127.0.0.1:3000",
     reuseExistingServer: !process.env.CI,
   },
 });

frontend/public/mockServiceWorker.js

@@ -8,8 +8,8 @@
  * - Please do NOT serve this file on production.
  */
 
-const PACKAGE_VERSION = '2.6.6'
-const INTEGRITY_CHECKSUM = 'ca7800994cc8bfb5eb961e037c877074'
+const PACKAGE_VERSION = '2.3.0-ws.rc-12'
+const INTEGRITY_CHECKSUM = '26357c79639bfa20d64c0efca2a87423'
 const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
 const activeClientIds = new Set()
 
@@ -62,12 +62,7 @@ self.addEventListener('message', async function (event) {
 
       sendToClient(client, {
         type: 'MOCKING_ENABLED',
-        payload: {
-          client: {
-            id: client.id,
-            frameType: client.frameType,
-          },
-        },
+        payload: true,
       })
       break
     }
@@ -160,10 +155,6 @@ async function handleRequest(event, requestId) {
 async function resolveMainClient(event) {
   const client = await self.clients.get(event.clientId)
 
-  if (activeClientIds.has(event.clientId)) {
-    return client
-  }
-
   if (client?.frameType === 'top-level') {
     return client
   }
@@ -192,14 +183,12 @@ async function getResponse(event, client, requestId) {
   const requestClone = request.clone()
 
   function passthrough() {
-    // Cast the request headers to a new Headers instance
-    // so the headers can be manipulated with.
-    const headers = new Headers(requestClone.headers)
+    const headers = Object.fromEntries(requestClone.headers.entries())
 
-    // Remove the "accept" header value that marked this request as passthrough.
-    // This prevents request alteration and also keeps it compliant with the
-    // user-defined CORS policies.
-    headers.delete('accept', 'msw/passthrough')
+    // Remove internal MSW request header so the passthrough request
+    // complies with any potential CORS preflight checks on the server.
+    // Some servers forbid unknown request headers.
+    delete headers['x-msw-intention']
 
     return fetch(requestClone, { headers })
   }

frontend/src/api/github.ts

@@ -27,19 +27,82 @@ export const isGitHubErrorReponse = <T extends object | Array<unknown>>(
  */
 export const retrieveGitHubUserRepositories = async (
   token: string,
-  page = 1,
   per_page = 30,
+  page = 1,
 ): Promise<Response> => {
   const url = new URL("https://api.github.com/user/repos");
   url.searchParams.append("sort", "pushed"); // sort by most recently pushed
-  url.searchParams.append("page", page.toString());
   url.searchParams.append("per_page", per_page.toString());
+  url.searchParams.append("page", page.toString());
 
   return fetch(url.toString(), {
     headers: generateGitHubAPIHeaders(token),
   });
 };
 
+/**
+ * Given a GitHub token, retrieves all repositories of the authenticated user
+ * @param token The GitHub token
+ * @returns A list of repositories or an error response
+ */
+export const retrieveAllGitHubUserRepositories = async (
+  token: string,
+): Promise<GitHubRepository[] | GitHubErrorReponse> => {
+  const repositories: GitHubRepository[] = [];
+
+  // Fetch the first page to extract the last page number and get the first batch of data
+  const firstPageResponse = await retrieveGitHubUserRepositories(token, 100, 1);
+
+  if (!firstPageResponse.ok) {
+    return {
+      message: "Failed to fetch repositories",
+      documentation_url:
+        "https://docs.github.com/rest/reference/repos#list-repositories-for-the-authenticated-user",
+      status: firstPageResponse.status,
+    };
+  }
+
+  const firstPageData = await firstPageResponse.json();
+  repositories.push(...firstPageData);
+
+  // Check for pagination and extract the last page number
+  const link = firstPageResponse.headers.get("link");
+  const lastPageMatch = link?.match(/page=(\d+)>; rel="last"/);
+  const lastPage = lastPageMatch ? parseInt(lastPageMatch[1], 10) : 1;
+
+  // If there is only one page, return the fetched repositories
+  if (lastPage === 1) {
+    return repositories;
+  }
+
+  // Create an array of promises for the remaining pages
+  const promises = [];
+  for (let page = 2; page <= lastPage; page += 1) {
+    promises.push(retrieveGitHubUserRepositories(token, 100, page));
+  }
+
+  // Fetch all pages in parallel
+  const responses = await Promise.all(promises);
+
+  for (const response of responses) {
+    if (response.ok) {
+      // TODO: Is there a way to avoid using await within a loop?
+      // eslint-disable-next-line no-await-in-loop
+      const data = await response.json();
+      repositories.push(...data);
+    } else {
+      return {
+        message: "Failed to fetch repositories",
+        documentation_url:
+          "https://docs.github.com/rest/reference/repos#list-repositories-for-the-authenticated-user",
+        status: response.status,
+      };
+    }
+  }
+
+  return repositories;
+};
+
 /**
  * Given a GitHub token, retrieves the authenticated user
  * @param token The GitHub token
@@ -51,11 +114,6 @@ export const retrieveGitHubUser = async (
   const response = await fetch("https://api.github.com/user", {
     headers: generateGitHubAPIHeaders(token),
   });
-
-  if (!response.ok) {
-    throw new Error("Failed to retrieve user data");
-  }
-
   const data = await response.json();
 
   if (!isGitHubErrorReponse(data)) {
@@ -91,9 +149,5 @@ export const retrieveLatestGitHubCommit = async (
     headers: generateGitHubAPIHeaders(token),
   });
 
-  if (!response.ok) {
-    throw new Error("Failed to retrieve latest commit");
-  }
-
   return response.json();
 };

frontend/src/api/open-hands.ts

@@ -1,4 +1,5 @@
 import { request } from "#/services/api";
+import { cache } from "#/utils/cache";
 import {
   SaveFileSuccessResponse,
   FileUploadSuccessResponse,
@@ -16,13 +17,13 @@ class OpenHands {
    * @returns List of models available
    */
   static async getModels(): Promise<string[]> {
-    const response = await fetch("/api/options/models");
+    const cachedData = cache.get<string[]>("models");
+    if (cachedData) return cachedData;
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch models");
-    }
+    const data = await request("/api/options/models");
+    cache.set("models", data);
 
-    return response.json();
+    return data;
   }
 
   /**
@@ -30,13 +31,13 @@ class OpenHands {
    * @returns List of agents available
    */
   static async getAgents(): Promise<string[]> {
-    const response = await fetch("/api/options/agents");
+    const cachedData = cache.get<string[]>("agents");
+    if (cachedData) return cachedData;
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch agents");
-    }
+    const data = await request(`/api/options/agents`);
+    cache.set("agents", data);
 
-    return response.json();
+    return data;
   }
 
   /**
@@ -44,23 +45,23 @@ class OpenHands {
    * @returns List of security analyzers available
    */
   static async getSecurityAnalyzers(): Promise<string[]> {
-    const response = await fetch("/api/options/security-analyzers");
+    const cachedData = cache.get<string[]>("agents");
+    if (cachedData) return cachedData;
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch security analyzers");
-    }
+    const data = await request(`/api/options/security-analyzers`);
+    cache.set("security-analyzers", data);
 
-    return response.json();
+    return data;
   }
 
   static async getConfig(): Promise<GetConfigResponse> {
-    const response = await fetch("/config.json");
+    const cachedData = cache.get<GetConfigResponse>("config");
+    if (cachedData) return cachedData;
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch config");
-    }
+    const data = await request("/config.json");
+    cache.set("config", data);
 
-    return response.json();
+    return data;
   }
 
   /**
@@ -68,21 +69,10 @@ class OpenHands {
    * @param path Path to list files from
    * @returns List of files available in the given path. If path is not provided, it lists all the files in the workspace
    */
-  static async getFiles(token: string, path?: string): Promise<string[]> {
-    const url = new URL("/api/list-files", window.location.origin);
-    if (path) url.searchParams.append("path", path);
-
-    const response = await fetch(url.toString(), {
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
-    if (!response.ok) {
-      throw new Error("Failed to fetch files");
-    }
-
-    return response.json();
+  static async getFiles(path?: string): Promise<string[]> {
+    let url = "/api/list-files";
+    if (path) url += `?path=${encodeURIComponent(path)}`;
+    return request(url);
   }
 
   /**
@@ -90,21 +80,9 @@ class OpenHands {
    * @param path Full path of the file to retrieve
    * @returns Content of the file
    */
-  static async getFile(token: string, path: string): Promise<string> {
-    const url = new URL("/api/select-file", window.location.origin);
-    url.searchParams.append("file", path);
-
-    const response = await fetch(url.toString(), {
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
-    if (!response.ok) {
-      throw new Error("Failed to fetch file");
-    }
-
-    const data = await response.json();
+  static async getFile(path: string): Promise<string> {
+    const url = `/api/select-file?file=${encodeURIComponent(path)}`;
+    const data = await request(url);
     return data.code;
   }
 
@@ -115,32 +93,16 @@ class OpenHands {
    * @returns Success message or error message
    */
   static async saveFile(
-    token: string,
     path: string,
     content: string,
-  ): Promise<SaveFileSuccessResponse> {
-    const response = await fetch("/api/save-file", {
+  ): Promise<SaveFileSuccessResponse | ErrorResponse> {
+    return request(`/api/save-file`, {
       method: "POST",
       body: JSON.stringify({ filePath: path, content }),
       headers: {
         "Content-Type": "application/json",
-        Authorization: `Bearer ${token}`,
       },
     });
-
-    if (!response.ok) {
-      throw new Error("Failed to save file");
-    }
-
-    const data = (await response.json()) as
-      | SaveFileSuccessResponse
-      | ErrorResponse;
-
-    if ("error" in data) {
-      throw new Error(data.error);
-    }
-
-    return data;
   }
 
   /**
@@ -149,78 +111,15 @@ class OpenHands {
    * @returns Success message or error message
    */
   static async uploadFiles(
-    token: string,
-    files: File[],
-  ): Promise<FileUploadSuccessResponse> {
+    file: File[],
+  ): Promise<FileUploadSuccessResponse | ErrorResponse> {
     const formData = new FormData();
-    files.forEach((file) => formData.append("files", file));
+    file.forEach((f) => formData.append("files", f));
 
-    const response = await fetch("/api/upload-files", {
+    return request(`/api/upload-files`, {
       method: "POST",
       body: formData,
-      headers: {
-        Authorization: `Bearer ${token}`,
-      },
     });
-
-    if (!response.ok) {
-      throw new Error("Failed to upload files");
-    }
-
-    const data = (await response.json()) as
-      | FileUploadSuccessResponse
-      | ErrorResponse;
-
-    if ("error" in data) {
-      throw new Error(data.error);
-    }
-
-    return data;
-  }
-
-  /**
-   * Send feedback to the server
-   * @param data Feedback data
-   * @returns The stored feedback data
-   */
-  static async submitFeedback(
-    token: string,
-    feedback: Feedback,
-  ): Promise<FeedbackResponse> {
-    const response = await fetch("/api/submit-feedback", {
-      method: "POST",
-      body: JSON.stringify(feedback),
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${token}`,
-      },
-    });
-
-    if (!response.ok) {
-      throw new Error("Failed to submit feedback");
-    }
-
-    return response.json();
-  }
-
-  /**
-   * Authenticate with GitHub token
-   * @returns Response with authentication status and user info if successful
-   */
-  static async authenticate(
-    gitHubToken: string,
-    appMode: GetConfigResponse["APP_MODE"],
-  ): Promise<boolean> {
-    if (appMode === "oss") return true;
-
-    const response = await fetch("/api/authenticate", {
-      method: "POST",
-      headers: {
-        "X-GitHub-Token": gitHubToken,
-      },
-    });
-
-    return response.ok;
   }
 
   /**
@@ -232,6 +131,21 @@ class OpenHands {
     return response.blob();
   }
 
+  /**
+   * Send feedback to the server
+   * @param data Feedback data
+   * @returns The stored feedback data
+   */
+  static async submitFeedback(data: Feedback): Promise<FeedbackResponse> {
+    return request(`/api/submit-feedback`, {
+      method: "POST",
+      body: JSON.stringify(data),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+  }
+
   /**
    * @param code Code provided by GitHub
    * @returns GitHub access token
@@ -239,19 +153,27 @@ class OpenHands {
   static async getGitHubAccessToken(
     code: string,
   ): Promise<GitHubAccessTokenResponse> {
-    const response = await fetch("/api/github/callback", {
+    return request(`/api/github/callback`, {
       method: "POST",
       body: JSON.stringify({ code }),
       headers: {
         "Content-Type": "application/json",
       },
     });
+  }
 
-    if (!response.ok) {
-      throw new Error("Failed to get GitHub access token");
-    }
-
-    return response.json();
+  /**
+   * Authenticate with GitHub token
+   * @returns Response with authentication status and user info if successful
+   */
+  static async authenticate(): Promise<Response> {
+    return request(
+      `/api/authenticate`,
+      {
+        method: "POST",
+      },
+      true,
+    );
   }
 
   /**
@@ -261,12 +183,6 @@ class OpenHands {
   static async getVSCodeUrl(): Promise<GetVSCodeUrlResponse> {
     return request(`/api/vscode-url`, {}, false, false, 1);
   }
-
-  static async getRuntimeId(): Promise<{ runtime_id: string }> {
-    const data = await request("/api/conversation");
-
-    return data;
-  }
 }
 
 export default OpenHands;

frontend/src/components/AgentControlBar.tsx

@@ -3,9 +3,9 @@ import React from "react";
 import { useSelector } from "react-redux";
 import PauseIcon from "#/assets/pause";
 import PlayIcon from "#/assets/play";
-import { generateAgentStateChangeEvent } from "#/services/agent-state-service";
+import { generateAgentStateChangeEvent } from "#/services/agentStateService";
 import { RootState } from "#/store";
-import AgentState from "#/types/agent-state";
+import AgentState from "#/types/AgentState";
 import { useWsClient } from "#/context/ws-client-provider";
 
 const IgnoreTaskStateMap: Record<string, AgentState[]> = {

frontend/src/components/AgentStatusBar.tsx

@@ -4,7 +4,7 @@ import { useSelector } from "react-redux";
 import toast from "react-hot-toast";
 import { I18nKey } from "#/i18n/declaration";
 import { RootState } from "#/store";
-import AgentState from "#/types/agent-state";
+import AgentState from "#/types/AgentState";
 import beep from "#/utils/beep";
 
 enum IndicatorColor {

frontend/src/components/Jupyter.tsx

@@ -6,8 +6,8 @@ import { VscArrowDown } from "react-icons/vsc";
 import { useTranslation } from "react-i18next";
 import { atomOneDark } from "react-syntax-highlighter/dist/esm/styles/hljs";
 import { RootState } from "#/store";
-import { Cell } from "#/state/jupyter-slice";
-import { useScrollToBottom } from "#/hooks/use-scroll-to-bottom";
+import { Cell } from "#/state/jupyterSlice";
+import { useScrollToBottom } from "#/hooks/useScrollToBottom";
 import { I18nKey } from "#/i18n/declaration";
 
 interface IJupyterCell {

frontend/src/components/analytics-consent-form-modal.tsx

@@ -1,35 +1,20 @@
+import { useFetcher } from "@remix-run/react";
 import { ModalBackdrop } from "./modals/modal-backdrop";
-import ModalBody from "./modals/modal-body";
-import ModalButton from "./buttons/modal-button";
+import ModalBody from "./modals/ModalBody";
+import ModalButton from "./buttons/ModalButton";
 import {
   BaseModalTitle,
   BaseModalDescription,
-} from "./modals/confirmation-modals/base-modal";
-import { handleCaptureConsent } from "#/utils/handle-capture-consent";
+} from "./modals/confirmation-modals/BaseModal";
 
-interface AnalyticsConsentFormModalProps {
-  onClose: () => void;
-}
-
-export function AnalyticsConsentFormModal({
-  onClose,
-}: AnalyticsConsentFormModalProps) {
-  const handleSubmit = (e: React.FormEvent<HTMLFormElement>) => {
-    e.preventDefault();
-    const formData = new FormData(e.currentTarget);
-    const analytics = formData.get("analytics") === "on";
-
-    handleCaptureConsent(analytics);
-    localStorage.setItem("analytics-consent", analytics.toString());
-
-    onClose();
-  };
+export function AnalyticsConsentFormModal() {
+  const fetcher = useFetcher({ key: "set-consent" });
 
   return (
     <ModalBackdrop>
-      <form
-        data-testid="user-capture-consent-form"
-        onSubmit={handleSubmit}
+      <fetcher.Form
+        method="POST"
+        action="/set-consent"
         className="flex flex-col gap-2"
       >
         <ModalBody>
@@ -51,7 +36,7 @@ export function AnalyticsConsentFormModal({
             className="bg-primary text-white w-full hover:opacity-80"
           />
         </ModalBody>
-      </form>
+      </fetcher.Form>
     </ModalBackdrop>
   );
 }