chore: add back accidentally removed repo info (#5532)

Co-authored-by: openhands <openhands@all-hands.dev>

.github/ISSUE_TEMPLATE/bug_template.yml

@@ -31,6 +31,8 @@ body:
       options:
         - Docker command in README
         - Development workflow
+        - app.all-hands.dev
+        - Other
       default: 0
 
   - type: input

.github/dependabot.yml

@@ -16,6 +16,9 @@ updates:
       chromadb:
         patterns:
           - "chromadb"
+      browsergym:
+        patterns:
+          - "browsergym*"
       security-all:
         applies-to: "security-updates"
         patterns:

.github/workflows/eval-runner.yml

@@ -1,10 +1,8 @@
-name: Run Evaluation
+name: Run SWE-Bench Evaluation
 
 on:
   pull_request:
     types: [labeled]
-  schedule:
-    - cron: "0 1 * * *" # Run daily at 1 AM UTC
   workflow_dispatch:
     inputs:
       reason:
@@ -60,24 +58,6 @@ jobs:
           echo "api_key = \"$DEEPSEEK_API_KEY\"" >> config.toml
           echo "temperature = 0.0" >> config.toml
 
-      - name: Run integration test evaluation
-        env:
-          ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
-          RUNTIME: remote
-          SANDBOX_REMOTE_RUNTIME_API_URL: https://runtime.eval.all-hands.dev
-          EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
-
-        run: |
-          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES
-
-          # get evaluation report
-          REPORT_FILE=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek-chat_maxiter_10_N* -name "report.md" -type f | head -n 1)
-          echo "REPORT_FILE: $REPORT_FILE"
-          echo "INTEGRATION_TEST_REPORT<<EOF" >> $GITHUB_ENV
-          cat $REPORT_FILE >> $GITHUB_ENV
-          echo >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-
       - name: Run SWE-Bench evaluation
         env:
           ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
@@ -86,12 +66,12 @@ jobs:
           EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
 
         run: |
-          poetry run ./evaluation/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
+          poetry run ./evaluation/benchmarks/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
           OUTPUT_FOLDER=$(find evaluation/evaluation_outputs/outputs/princeton-nlp__SWE-bench_Lite-test/CodeActAgent -name "deepseek-chat_maxiter_50_N_*-no-hint-run_1" -type d | head -n 1)
           echo "OUTPUT_FOLDER for SWE-bench evaluation: $OUTPUT_FOLDER"
-          poetry run ./evaluation/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
+          poetry run ./evaluation/benchmarks/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
 
-          poetry run ./evaluation/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
+          poetry run ./evaluation/benchmarks/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
           echo "SWEBENCH_REPORT<<EOF" >> $GITHUB_ENV
           cat summarize_outputs.log >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
@@ -145,9 +125,6 @@ jobs:
               **SWE-Bench Evaluation Report**
               ${{ env.SWEBENCH_REPORT }}
               ---
-              **Integration Tests Evaluation Report**
-              ${{ env.INTEGRATION_TEST_REPORT }}
-              ---
               You can download the full evaluation outputs [here](${{ env.ARTIFACT_URL }}).
 
       - name: Post to a Slack channel

.github/workflows/fe-unit-tests.yml

@@ -24,7 +24,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [20]
+        node-version: [20, 22]
+      fail-fast: true
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -35,6 +36,9 @@ jobs:
       - name: Install dependencies
         working-directory: ./frontend
         run: npm ci
+      - name: Run TypeScript compilation
+        working-directory: ./frontend
+        run: npm run make-i18n && tsc
       - name: Run tests and collect coverage
         working-directory: ./frontend
         run: npm run test:coverage

.github/workflows/ghcr-build.yml

@@ -68,6 +68,9 @@ jobs:
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
+      - name: "Set up docker layer caching"
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
       - name: Build and push app image
         if: "!github.event.pull_request.head.repo.fork"
         run: |
@@ -286,13 +289,12 @@ jobs:
           image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
-          SKIP_CONTAINER_LOGS=true \
           TEST_RUNTIME=eventstream \
           SANDBOX_USER_ID=$(id -u) \
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
@@ -364,13 +366,12 @@ jobs:
           image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image }}
           image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]')
 
-          SKIP_CONTAINER_LOGS=true \
           TEST_RUNTIME=eventstream \
           SANDBOX_USER_ID=$(id -u) \
           SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
           TEST_IN_CI=true \
           RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime
+          poetry run pytest -n 3 -raRs --reruns 2 --reruns-delay 5 --cov=openhands --cov-report=xml -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
@@ -399,3 +400,49 @@ jobs:
         run: |
           echo "Some runtime tests failed or were cancelled"
           exit 1
+  update_pr_description:
+    name: Update PR Description
+    if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'
+    needs: [ghcr_build_runtime]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: short_sha
+        run: echo "SHORT_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
+
+      - name: Update PR Description
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+          SHORT_SHA: ${{ steps.short_sha.outputs.SHORT_SHA }}
+        run: |
+          echo "updating PR description"
+          DOCKER_RUN_COMMAND="docker run -it --rm \
+            -p 3000:3000 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            --add-host host.docker.internal:host-gateway \
+            -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:$SHORT_SHA-nikolaik \
+            --name openhands-app-$SHORT_SHA \
+            docker.all-hands.dev/all-hands-ai/openhands:$SHORT_SHA"
+
+          PR_BODY=$(gh pr view $PR_NUMBER --json body --jq .body)
+
+          if echo "$PR_BODY" | grep -q "To run this PR locally, use the following command:"; then
+            UPDATED_PR_BODY=$(echo "${PR_BODY}" | sed -E "s|docker run -it --rm.*|$DOCKER_RUN_COMMAND|")
+          else
+            UPDATED_PR_BODY="${PR_BODY}
+
+          ---
+
+          To run this PR locally, use the following command:
+          \`\`\`
+          $DOCKER_RUN_COMMAND
+          \`\`\`"
+          fi
+
+          echo "updated body: $UPDATED_PR_BODY"
+          gh pr edit $PR_NUMBER --body "$UPDATED_PR_BODY"

.github/workflows/integration-runner.yml

@@ -0,0 +1,158 @@
+name: Run Integration Tests
+
+on:
+  pull_request:
+    types: [labeled]
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: 'Reason for manual trigger'
+        required: true
+        default: ''
+  schedule:
+    - cron: '30 22 * * *'  # Runs at 10:30pm UTC every day
+
+env:
+  N_PROCESSES: 10 # Global configuration for number of parallel processes for evaluation
+
+jobs:
+  run-integration-tests:
+    if: github.event.label.name == 'integration-test' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "read"
+      id-token: "write"
+      pull-requests: "write"
+      issues: "write"
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry via pipx
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "poetry"
+
+      - name: Comment on PR if 'integration-test' label is present
+        if: github.event_name == 'pull_request' && github.event.label.name == 'integration-test'
+        uses: KeisukeYamashita/create-comment@v1
+        with:
+          unique: false
+          comment: |
+            Hi! I started running the integration tests on your PR. You will receive a comment with the results shortly.
+
+      - name: Install Python dependencies using Poetry
+        run: poetry install --without evaluation,llama-index
+
+      - name: Configure config.toml for testing with Haiku
+        env:
+          LLM_MODEL: "litellm_proxy/claude-3-5-haiku-20241022"
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+        run: |
+          echo "[llm.eval]" > config.toml
+          echo "model = \"$LLM_MODEL\"" >> config.toml
+          echo "api_key = \"$LLM_API_KEY\"" >> config.toml
+          echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
+          echo "temperature = 0.0" >> config.toml
+
+      - name: Build environment
+        run: make build
+
+      - name: Run integration test evaluation for Haiku
+        env:
+          SANDBOX_FORCE_REBUILD_RUNTIME: True
+        run: |
+          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'haiku_run'
+
+          # get integration tests report
+          REPORT_FILE_HAIKU=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/*haiku*_maxiter_10_N* -name "report.md" -type f | head -n 1)
+          echo "REPORT_FILE: $REPORT_FILE_HAIKU"
+          echo "INTEGRATION_TEST_REPORT_HAIKU<<EOF" >> $GITHUB_ENV
+          cat $REPORT_FILE_HAIKU >> $GITHUB_ENV
+          echo >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Wait a little bit
+        run: sleep 10
+
+      - name: Configure config.toml for testing with DeepSeek
+        env:
+          LLM_MODEL: "litellm_proxy/deepseek-chat"
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+        run: |
+          echo "[llm.eval]" > config.toml
+          echo "model = \"$LLM_MODEL\"" >> config.toml
+          echo "api_key = \"$LLM_API_KEY\"" >> config.toml
+          echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
+          echo "temperature = 0.0" >> config.toml
+
+      - name: Run integration test evaluation for DeepSeek
+        env:
+          SANDBOX_FORCE_REBUILD_RUNTIME: True
+        run: |
+          poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'deepseek_run'
+
+          # get integration tests report
+          REPORT_FILE_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek*_maxiter_10_N* -name "report.md" -type f | head -n 1)
+          echo "REPORT_FILE: $REPORT_FILE_DEEPSEEK"
+          echo "INTEGRATION_TEST_REPORT_DEEPSEEK<<EOF" >> $GITHUB_ENV
+          cat $REPORT_FILE_DEEPSEEK >> $GITHUB_ENV
+          echo >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Create archive of evaluation outputs
+        run: |
+          TIMESTAMP=$(date +'%y-%m-%d-%H-%M')
+          cd evaluation/evaluation_outputs/outputs  # Change to the outputs directory
+          tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/*  # Only include the actual result directories
+
+      - name: Upload evaluation results as artifact
+        uses: actions/upload-artifact@v4
+        id: upload_results_artifact
+        with:
+          name: integration-test-outputs-${{ github.run_id }}-${{ github.run_attempt }}
+          path: integration_tests_*.tar.gz
+
+      - name: Get artifact URLs
+        run: |
+          echo "ARTIFACT_URL=${{ steps.upload_results_artifact.outputs.artifact-url }}" >> $GITHUB_ENV
+
+      - name: Set timestamp and trigger reason
+        run: |
+          echo "TIMESTAMP=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "TRIGGER_REASON=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "TRIGGER_REASON=manual-${{ github.event.inputs.reason }}" >> $GITHUB_ENV
+          else
+            echo "TRIGGER_REASON=nightly-scheduled" >> $GITHUB_ENV
+          fi
+
+      - name: Comment with results and artifact link
+        id: create_comment
+        uses: KeisukeYamashita/create-comment@v1
+        with:
+          # if triggered by PR, use PR number, otherwise use 5318 as fallback issue number for manual triggers
+          number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || 5318 }}
+          unique: false
+          comment: |
+              Trigger by: ${{ github.event_name == 'pull_request' && format('Pull Request (integration-test label on PR #{0})', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && format('Manual Trigger: {0}', github.event.inputs.reason)) || 'Nightly Scheduled Run' }}
+              Commit: ${{ github.sha }}
+              **Integration Tests Report (Haiku)**
+              Haiku LLM Test Results:
+              ${{ env.INTEGRATION_TEST_REPORT_HAIKU }}
+              ---
+              **Integration Tests Report (DeepSeek)**
+              DeepSeek LLM Test Results:
+              ${{ env.INTEGRATION_TEST_REPORT_DEEPSEEK }}
+              ---
+              Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }})

.github/workflows/lint-fix.yml

@@ -0,0 +1,91 @@
+name: Lint Fix
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  # Frontend lint fixes
+  lint-fix-frontend:
+    if: github.event.label.name == 'lint-fix'
+    name: Fix frontend linting issues
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install frontend dependencies
+        run: |
+          cd frontend
+          npm install --frozen-lockfile
+      - name: Fix frontend lint issues
+        run: |
+          cd frontend
+          npm run lint:fix
+
+      # Commit and push changes if any
+      - name: Check for changes
+        id: git-check
+        run: |
+          git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT
+      - name: Commit and push if there are changes
+        if: steps.git-check.outputs.changes == 'true'
+        run: |
+          git config --local user.email "openhands@all-hands.dev"
+          git config --local user.name "OpenHands Bot"
+          git add -A
+          git commit -m "🤖 Auto-fix frontend linting issues"
+          git push
+
+  # Python lint fixes
+  lint-fix-python:
+    if: github.event.label.name == 'lint-fix'
+    name: Fix Python linting issues
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          cache: 'pip'
+      - name: Install pre-commit
+        run: pip install pre-commit==3.7.0
+      - name: Fix python lint issues
+        run: |
+          # Run all pre-commit hooks and continue even if they modify files (exit code 1)
+          pre-commit run --config ./dev_config/python/.pre-commit-config.yaml --files openhands/**/* evaluation/**/* tests/**/* || true
+
+      # Commit and push changes if any
+      - name: Check for changes
+        id: git-check
+        run: |
+          git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT
+      - name: Commit and push if there are changes
+        if: steps.git-check.outputs.changes == 'true'
+        run: |
+          git config --local user.email "openhands@all-hands.dev"
+          git config --local user.name "OpenHands Bot"
+          git add -A
+          git commit -m "🤖 Auto-fix Python linting issues"
+          git push

.github/workflows/lint.yml

@@ -30,10 +30,11 @@ jobs:
         run: |
           cd frontend
           npm install --frozen-lockfile
-      - name: Lint
+      - name: Lint and TypeScript compilation
         run: |
           cd frontend
           npm run lint
+          npm run make-i18n && tsc
 
   # Run lint on the python code
   lint-python:

.github/workflows/openhands-resolver.yml

@@ -1,13 +1,316 @@
-name: Resolve Issues with OpenHands
+name: Auto-Fix Tagged Issue with OpenHands
 
 on:
+  workflow_call:
+    inputs:
+      max_iterations:
+        required: false
+        type: number
+        default: 50
+      macro:
+        required: false
+        type: string
+        default: "@openhands-agent"
+      target_branch:
+        required: false
+        type: string
+        default: "main"
+        description: "Target branch to pull and create PR against"
+      LLM_MODEL:
+        required: false
+        type: string
+        default: "anthropic/claude-3-5-sonnet-20241022"
+      base_container_image:
+        required: false
+        type: string
+        default: ""
+        description: "Custom sandbox env"
+    secrets:
+      LLM_MODEL:
+        required: false
+      LLM_API_KEY:
+        required: true
+      LLM_BASE_URL:
+        required: false
+      PAT_TOKEN:
+        required: false
+      PAT_USERNAME:
+        required: false
+
   issues:
     types: [labeled]
+  pull_request:
+    types: [labeled]
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  pull_request_review:
+    types: [submitted]
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
 
 jobs:
-  call-openhands-resolver:
-    uses: All-Hands-AI/openhands-resolver/.github/workflows/openhands-resolver.yml@main
-    if: github.event.label.name == 'fix-me'
-    with:
-      issue_number: ${{ github.event.issue.number }}
-    secrets: inherit
+  auto-fix:
+    if: |
+      github.event_name == 'workflow_call' ||
+      github.event.label.name == 'fix-me' ||
+      github.event.label.name == 'fix-me-experimental' ||
+
+      (
+        ((github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
+        contains(github.event.comment.body, inputs.macro || '@openhands-agent') &&
+        (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER')
+        ) ||
+
+        (github.event_name == 'pull_request_review' &&
+        contains(github.event.review.body, inputs.macro || '@openhands-agent') &&
+        (github.event.review.author_association == 'OWNER' || github.event.review.author_association == 'COLLABORATOR' || github.event.review.author_association == 'MEMBER')
+        )
+      )
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Get latest versions and create requirements.txt
+        run: |
+          python -m pip index versions openhands-ai > openhands_versions.txt
+          OPENHANDS_VERSION=$(head -n 1 openhands_versions.txt | awk '{print $2}' | tr -d '()')
+          echo "openhands-ai==${OPENHANDS_VERSION}" >> requirements.txt
+          cat requirements.txt
+
+      - name: Cache pip dependencies
+        if: |
+          !(
+            github.event.label.name == 'fix-me-experimental' ||
+            (
+              (github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
+              contains(github.event.comment.body, '@openhands-agent-exp')
+            ) ||
+            (
+              github.event_name == 'pull_request_review' &&
+              contains(github.event.review.body, '@openhands-agent-exp')
+            )
+          )
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/*
+          key: ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-openhands-resolver-${{ hashFiles('requirements.txt') }}
+
+      - name: Check required environment variables
+        env:
+          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
+          PAT_USERNAME: ${{ secrets.PAT_USERNAME }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          required_vars=("LLM_MODEL" "LLM_API_KEY")
+          for var in "${required_vars[@]}"; do
+            if [ -z "${!var}" ]; then
+              echo "Error: Required environment variable $var is not set."
+              exit 1
+            fi
+          done
+
+          # Check optional variables and warn about fallbacks
+          if [ -z "$PAT_TOKEN" ]; then
+            echo "Warning: PAT_TOKEN is not set, falling back to GITHUB_TOKEN"
+          fi
+
+          if [ -z "$LLM_BASE_URL" ]; then
+            echo "Warning: LLM_BASE_URL is not set, will use default API endpoint"
+          fi
+
+          if [ -z "$PAT_USERNAME" ]; then
+            echo "Warning: PAT_USERNAME is not set, will use openhands-agent"
+          fi
+
+      - name: Set environment variables
+        run: |
+          if [ -n "${{ github.event.review.body }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          elif [ -n "${{ github.event.issue.pull_request }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          elif [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          else
+            echo "ISSUE_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
+            echo "ISSUE_TYPE=issue" >> $GITHUB_ENV
+          fi
+
+          if [ -n "${{ github.event.review.body }}" ]; then
+            echo "COMMENT_ID=${{ github.event.review.id || 'None' }}" >> $GITHUB_ENV
+          else
+            echo "COMMENT_ID=${{ github.event.comment.id || 'None' }}" >> $GITHUB_ENV
+          fi
+
+          echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV
+          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.PAT_TOKEN || github.token }}" >> $GITHUB_ENV
+          echo "SANDBOX_ENV_BASE_CONTAINER_IMAGE=${{ inputs.base_container_image }}" >> $GITHUB_ENV
+
+          # Set branch variables
+          echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+
+      - name: Comment on issue with start message
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.PAT_TOKEN || github.token }}
+          script: |
+            const issueType = process.env.ISSUE_TYPE;
+            github.rest.issues.createComment({
+              issue_number: ${{ env.ISSUE_NUMBER }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started fixing the ${issueType}! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
+            });
+
+      - name: Install OpenHands
+        run: |
+          if [[ "${{ github.event.label.name }}" == "fix-me-experimental" ]] ||
+             ([[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]] &&
+              [[ "${{ github.event.comment.body }}" == "@openhands-agent-exp"* ]]) ||
+             ([[ "${{ github.event_name }}" == "pull_request_review" ]] &&
+              [[ "${{ github.event.review.body }}" == "@openhands-agent-exp"* ]]); then
+            python -m pip install --upgrade pip
+            pip install git+https://github.com/all-hands-ai/openhands.git
+          else
+            python -m pip install --upgrade -r requirements.txt
+          fi
+
+      - name: Attempt to resolve issue
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PYTHONPATH: ""
+        run: |
+          cd /tmp && python -m openhands.resolver.resolve_issue \
+            --repo ${{ github.repository }} \
+            --issue-number ${{ env.ISSUE_NUMBER }} \
+            --issue-type ${{ env.ISSUE_TYPE }} \
+            --max-iterations ${{ env.MAX_ITERATIONS }} \
+            --comment-id ${{ env.COMMENT_ID }}
+
+      - name: Check resolution result
+        id: check_result
+        run: |
+          if cd /tmp && grep -q '"success":true' output/output.jsonl; then
+            echo "RESOLUTION_SUCCESS=true" >> $GITHUB_OUTPUT
+          else
+            echo "RESOLUTION_SUCCESS=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload output.jsonl as artifact
+        uses: actions/upload-artifact@v4
+        if: always() # Upload even if the previous steps fail
+        with:
+          name: resolver-output
+          path: /tmp/output/output.jsonl
+          retention-days: 30 # Keep the artifact for 30 days
+
+      - name: Create draft PR or push branch
+        if: always() # Create PR or branch even if the previous steps fail
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN || github.token }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME || 'openhands-agent' }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL || inputs.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PYTHONPATH: ""
+        run: |
+          if [ "${{ steps.check_result.outputs.RESOLUTION_SUCCESS }}" == "true" ]; then
+            cd /tmp && python -m openhands.resolver.send_pull_request \
+              --issue-number ${{ env.ISSUE_NUMBER }} \
+              --pr-type draft \
+              --reviewer ${{ github.actor }} | tee pr_result.txt && \
+              grep "draft created" pr_result.txt | sed 's/.*\///g' > pr_number.txt
+          else
+            cd /tmp && python -m openhands.resolver.send_pull_request \
+              --issue-number ${{ env.ISSUE_NUMBER }} \
+              --pr-type branch \
+              --send-on-failure | tee branch_result.txt && \
+              grep "branch created" branch_result.txt | sed 's/.*\///g; s/.expand=1//g' > branch_name.txt
+          fi
+
+      - name: Comment on issue
+        uses: actions/github-script@v7
+        if: always() # Comment on issue even if the previous steps fail
+        with:
+          github-token: ${{ secrets.PAT_TOKEN || github.token }}
+          script: |
+            const fs = require('fs');
+            const issueNumber = ${{ env.ISSUE_NUMBER }};
+            const success = ${{ steps.check_result.outputs.RESOLUTION_SUCCESS }};
+
+            let prNumber = '';
+            let branchName = '';
+            let logContent = '';
+            const noChangesMessage = `No changes to commit for issue #${issueNumber}. Skipping commit.`;
+
+            try {
+              if (success){
+                logContent = fs.readFileSync('/tmp/pr_result.txt', 'utf8').trim();
+              } else {
+                logContent = fs.readFileSync('/tmp/branch_result.txt', 'utf8').trim();
+              }
+            } catch (error) {
+              console.error('Error reading results file:', error);
+            }
+
+            try {
+              if (success) {
+                prNumber = fs.readFileSync('/tmp/pr_number.txt', 'utf8').trim();
+              } else {
+                branchName = fs.readFileSync('/tmp/branch_name.txt', 'utf8').trim();
+              }
+            } catch (error) {
+              console.error('Error reading file:', error);
+            }
+
+            if (logContent.includes(noChangesMessage)) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Openhands failed to create any code changes.`
+              });
+            } else if (success && prNumber) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `A potential fix has been generated and a draft PR #${prNumber} has been created. Please review the changes.`
+              });
+            } else if (!success && branchName) {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named '${branchName}' has been created with the attempted changes. You can view the branch [here](https://github.com/${context.repo.owner}/${context.repo.repo}/tree/${branchName}). Manual intervention may be required.`
+              });
+            } else {
+              github.rest.issues.createComment({
+                issue_number: issueNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `The workflow to fix this issue encountered an error. Please check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for more information.`
+              });
+            }

.github/workflows/py-unit-tests.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Tests
-        run: poetry run pytest --forked --cov=openhands --cov-report=xml -svv ./tests/unit --ignore=tests/unit/test_memory.py
+        run: poetry run pytest --forked -n auto --cov=openhands --cov-report=xml -svv ./tests/unit --ignore=tests/unit/test_memory.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:

.github/workflows/review-pr.yml

@@ -1,81 +0,0 @@
-# Workflow that uses OpenHands to review a pull request. PR must be labeled 'review-this'
-name: Use OpenHands to Review Pull Request
-
-on:
-  pull_request:
-    types: [synchronize, labeled]
-
-permissions:
-  contents: write
-  pull-requests: write
-
-jobs:
-  dogfood:
-    if: contains(github.event.pull_request.labels.*.name, 'review-this')
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@v3
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-    - name: install git, github cli
-      run: |
-        sudo apt-get install -y git gh
-        git config --global --add safe.directory $PWD
-    - name: Checkout Repository
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.base.ref }} # check out the target branch
-    - name: Download Diff
-      run: |
-        curl -O "${{ github.event.pull_request.diff_url }}" -L
-    - name: Write Task File
-      run: |
-        echo "Your coworker wants to apply a pull request to this project." > task.txt
-        echo "Read and review ${{ github.event.pull_request.number }}.diff file. Create a review-${{ github.event.pull_request.number }}.txt and write your concise comments and suggestions there." >> task.txt
-        echo "Do not ask me for confirmation at any point." >> task.txt
-        echo "" >> task.txt
-        echo "Title" >> task.txt
-        echo "${{ github.event.pull_request.title }}" >> task.txt
-        echo "" >> task.txt
-        echo "Description" >> task.txt
-        echo "${{ github.event.pull_request.body }}" >> task.txt
-        echo "" >> task.txt
-        echo "Diff file is: ${{ github.event.pull_request.number }}.diff" >> task.txt
-    - name: Set up environment
-      run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        export PATH="/github/home/.local/bin:$PATH"
-        poetry install --without evaluation,llama-index
-        poetry run playwright install --with-deps chromium
-    - name: Run OpenHands
-      env:
-        LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
-        LLM_MODEL: ${{ vars.LLM_MODEL }}
-      run: |
-        # Append path to launch poetry
-        export PATH="/github/home/.local/bin:$PATH"
-        # Append path to correctly import package, note: must set pwd at first
-        export PYTHONPATH=$(pwd):$PYTHONPATH
-        export WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE
-        export WORKSPACE_BASE=$GITHUB_WORKSPACE
-        echo -e "/exit\n" | poetry run python openhands/core/main.py -i 50 -f task.txt
-        rm task.txt
-    - name: Check if review file is non-empty
-      id: check_file
-      run: |
-        ls -la
-        if [[ -s review-${{ github.event.pull_request.number }}.txt ]]; then
-          echo "non_empty=true" >> $GITHUB_OUTPUT
-        fi
-      shell: bash
-    - name: Create PR review if file is non-empty
-      env:
-        GH_TOKEN: ${{ github.token }}
-      if: steps.check_file.outputs.non_empty == 'true'
-      run: |
-        gh pr review ${{ github.event.pull_request.number }} --comment --body-file "review-${{ github.event.pull_request.number }}.txt"

.github/workflows/run-eval.yml

@@ -0,0 +1,53 @@
+# Run evaluation on a PR
+name: Run Eval
+
+# Runs when a PR is labeled with one of the "run-eval-" labels
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  trigger-job:
+    name: Trigger remote eval job
+    if: ${{ github.event.label.name == 'run-eval-xs' || github.event.label.name == 'run-eval-s' || github.event.label.name == 'run-eval-m' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Trigger remote job
+        run: |
+          REPO_URL="https://github.com/${{ github.repository }}"
+          PR_BRANCH="${{ github.head_ref }}"
+          echo "Repository URL: $REPO_URL"
+          echo "PR Branch: $PR_BRANCH"
+
+          if [[ "${{ github.event.label.name }}" == "run-eval-xs" ]]; then
+            EVAL_INSTANCES="1"
+          elif [[ "${{ github.event.label.name }}" == "run-eval-s" ]]; then
+            EVAL_INSTANCES="5"
+          elif [[ "${{ github.event.label.name }}" == "run-eval-m" ]]; then
+            EVAL_INSTANCES="30"
+          fi
+
+          curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
+            -H "Accept: application/vnd.github+json" \
+            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
+            https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
+
+          # Send Slack message
+          PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
+          slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
+          curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
+            https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
+
+      - name: Comment on PR
+        uses: KeisukeYamashita/create-comment@v1
+        with:
+          unique: false
+          comment: |
+            Running evaluation on the PR. Once eval is done, the results will be posted.

.gitignore

@@ -174,6 +174,11 @@ evaluation/bird/data
 evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
+evaluation/scienceagentbench/benchmark
+evaluation/commit0_bench/repos
+
+# openhands resolver
+output/
 
 # frontend
 

COMMUNITY.md

@@ -0,0 +1,43 @@
+# 🙌 The OpenHands Community
+
+The OpenHands community is built around the belief that (1) AI and AI agents are going to fundamentally change the way
+we build software, and (2) if this is true, we should do everything we can to make sure that the benefits provided by
+such powerful technology are accessible to everyone.
+
+If this resonates with you, we'd love to have you join us in our quest!
+
+## 🤝 How to Join
+
+Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
+
+## 💪 Becoming a Contributor
+
+We welcome contributions from everyone! Whether you're a developer, a researcher, or simply enthusiastic about advancing
+the field of software engineering with AI, there are many ways to get involved:
+
+- **Code Contributions:** Help us develop new core functionality, improve our agents, improve the frontend and other
+interfaces, or anything else that would help make OpenHands better.
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in
+evaluating the models, or suggest improvements.
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+
+For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
+
+## Code of Conduct
+
+We have a [Code of Conduct](./CODE_OF_CONDUCT.md) that we expect all contributors to adhere to.
+Long story short, we are aiming for an open, welcoming, diverse, inclusive, and healthy community.
+All contributors are expected to contribute to building this sort of community.
+
+## 🛠️ Becoming a Maintainer
+
+For contributors who have made significant and sustained contributions to the project, there is a possibility of joining
+the maintainer team. The process for this is as follows:
+
+1. Any contributor who has made sustained and high-quality contributions to the codebase can be nominated by any
+maintainer. If you feel that you may qualify you can reach out to any of the maintainers that have reviewed your PRs and ask if you can be nominated.
+2. Once a maintainer nominates a new maintainer, there will be a discussion period among the maintainers for at least 3 days.
+3. If no concerns are raised the nomination will be accepted by acclamation, and if concerns are raised there will be a discussion and possible vote.
+
+Note that just making many PRs does not immediately imply that you will become a maintainer. We will be looking
+at sustained high-quality contributions over a period of time, as well as good teamwork and adherence to our [Code of Conduct](./CODE_OF_CONDUCT.md).

CONTRIBUTING.md

@@ -21,14 +21,14 @@ There are many ways that you can contribute:
 
 1. **Download and use** OpenHands, and send [issues](https://github.com/All-Hands-AI/OpenHands/issues) when you encounter something that isn't working or a feature that you'd like to see.
 2. **Send feedback** after each session by [clicking the thumbs-up thumbs-down buttons](https://docs.all-hands.dev/modules/usage/feedback), so we can see where things are working and failing, and also build an open dataset for training code agents.
-3. **Improve the Codebase** by sending PRs (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
+3. **Improve the Codebase** by sending [PRs](#sending-pull-requests-to-openhands) (see details below). In particular, we have some [good first issues](https://github.com/All-Hands-AI/OpenHands/labels/good%20first%20issue) that may be ones to start on.
 
 ## What can I build?
 Here are a few ways you can help improve the codebase.
 
 #### UI/UX
 We're always looking to improve the look and feel of the application. If you've got a small fix
-for something that's bugging you, feel free to open up a PR that changes the `./frontend` directory.
+for something that's bugging you, feel free to open up a PR that changes the [`./frontend`](./frontend) directory.
 
 If you're looking to make a bigger change, add a new UI element, or significantly alter the style
 of the application, please open an issue first, or better, join the #frontend channel in our Slack
@@ -46,7 +46,7 @@ We use the [SWE-bench](https://www.swebench.com/) benchmark to test our agent. Y
 channel in Slack to learn more.
 
 #### Adding a new agent
-You may want to experiment with building new types of agents. You can add an agent to `openhands/agenthub`
+You may want to experiment with building new types of agents. You can add an agent to [`openhands/agenthub`](./openhands/agenthub)
 to help expand the capabilities of OpenHands.
 
 #### Adding a new runtime
@@ -54,11 +54,11 @@ The agent needs a place to run code and commands. When you run OpenHands on your
 to do this by default. But there are other ways of creating a sandbox for the agent.
 
 If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
-by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/runtime.py).
+by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
 
 #### Testing
-When you write code, it is also good to write tests. Please navigate to the `tests` folder to see existing test suites.
-At the moment, we have two kinds of tests: `unit` and `integration`. Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
+When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites.
+At the moment, we have two kinds of tests: [`unit`](./tests/unit) and [`integration`](./evaluation/integration_tests). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
 
 ## Sending Pull Requests to OpenHands
 
@@ -92,3 +92,32 @@ You may also check out previous PRs in the [PR list](https://github.com/All-Hand
 
 If your changes are user-facing (e.g. a new feature in the UI, a change in behavior, or a bugfix)
 please include a short message that we can add to our changelog.
+
+## How to Make Effective Contributions
+
+### Opening Issues
+
+If you notice any bugs or have any feature requests please open them via the [issues page](https://github.com/All-Hands-AI/OpenHands/issues). We will triage based on how critical the bug is or how potentially useful the improvement is, discuss, and implement the ones that the community has interest/effort for.
+
+Further, if you see an issue you like, please leave a "thumbs-up" or a comment, which will help us prioritize.
+
+### Making Pull Requests
+
+We're generally happy to consider all [PRs](https://github.com/All-Hands-AI/OpenHands/pulls), with the evaluation process varying based on the type of change:
+
+#### For Small Improvements
+
+Small improvements with few downsides are typically reviewed and approved quickly.
+One thing to check when making changes is to ensure that all continuous integration tests pass, which you can check before getting a review.
+
+#### For Core Agent Changes
+
+We need to be more careful with changes to the core agent, as it is imperative to maintain high quality. These PRs are evaluated based on three key metrics:
+
+1. **Accuracy**
+2. **Efficiency**
+3. **Code Complexity**
+
+If it improves accuracy, efficiency, or both with only a minimal change to code quality, that's great we're happy to merge it in!
+If there are bigger tradeoffs (e.g. helping efficiency a lot and hurting accuracy a little) we might want to put it behind a feature flag.
+Either way, please feel free to discuss on github issues or slack, and we will give guidance and preliminary feedback.

Development.md

@@ -38,7 +38,9 @@ make build
 ```
 
 ### 3. Configuring the Language Model
-OpenHands supports a diverse array of Language Models (LMs) through the powerful [litellm](https://docs.litellm.ai) library. By default, we've chosen the mighty GPT-4 from OpenAI as our go-to model, but the world is your oyster! You can unleash the potential of Anthropic's suave Claude, the enigmatic Llama, or any other LM that piques your interest.
+OpenHands supports a diverse array of Language Models (LMs) through the powerful [litellm](https://docs.litellm.ai) library.
+By default, we've chosen Claude Sonnet 3.5 as our go-to model, but the world is your oyster! You can unleash the
+potential of any other LM that piques your interest.
 
 To configure the LM of your choice, run:
 
@@ -52,10 +54,7 @@ To configure the LM of your choice, run:
    Environment variables > config.toml variables > default variables
 
 **Note on Alternative Models:**
-Some alternative models may prove more challenging to tame than others. Fear not, brave adventurer! We shall soon unveil LLM-specific documentation to guide you on your quest.
-And if you've already mastered the art of wielding a model other than OpenAI's GPT, we encourage you to share your setup instructions with us by creating instructions and adding it [to our documentation](https://github.com/All-Hands-AI/OpenHands/tree/main/docs/modules/usage/llms).
-
-For a full list of the LM providers and models available, please consult the [litellm documentation](https://docs.litellm.ai/docs/providers).
+See [our documentation](https://docs.all-hands.dev/modules/usage/llms) for recommended models.
 
 ### 4. Running the application
 #### Option A: Run the Full Application
@@ -98,9 +97,10 @@ poetry run pytest ./tests/unit/test_*.py
 2. Update the poetry.lock file via `poetry lock --no-update`
 
 ### 9. Use existing Docker image
-To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image. Follow these steps:
-1. Set the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
-2. Example: export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.9-nikolaik
+To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by
+setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image.
+
+Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/all-hands-ai/runtime:0.15-nikolaik`
 
 ## Develop inside Docker container
 

ISSUE_TRIAGE.md

@@ -6,9 +6,9 @@ These are the procedures and guidelines on how issues are triaged in this repo b
 * Issues may be tagged with what it relates to (**backend**, **frontend**, **agent quality**, etc.)
 
 ## Severity
-* **Low**: Minor issues, single user report
-* **Medium**: Affecting multiple users
-* **Critical**: Affecting all users or potential security issues
+* **Low**: Minor issues or affecting single user.
+* **Medium**: Affecting multiple users.
+* **Critical**: Affecting all users or potential security issues.
 
 ## Effort
 * Issues may be estimated with effort required (**small effort**, **medium effort**, **large effort**)
@@ -17,9 +17,9 @@ These are the procedures and guidelines on how issues are triaged in this repo b
 * Issues with low implementation difficulty may be tagged with **good first issue**
 
 ## Not Enough Information
-* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear
-* If an issue is unclear and the author does not provide more information or respond to a request, the issue may be closed as **not planned** (Usually after a week)
+* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear.
+* If an issue is unclear and the author does not provide more information or respond to a request, the issue may be closed as **not planned** (Usually after a week).
 
 ## Multiple Requests/Fixes in One Issue
-* These issues will be narrowed down to one request/fix so the issue is more easily tracked and fixed
-* Issues may be broken down into multiple issues if required
+* These issues will be narrowed down to one request/fix so the issue is more easily tracked and fixed.
+* Issues may be broken down into multiple issues if required.

README.md

@@ -12,7 +12,7 @@
   <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License"></a>
   <br/>
-  <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
+  <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
   <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits"></a>
   <br/>
@@ -38,15 +38,16 @@ See the [Installation](https://docs.all-hands.dev/modules/usage/installation) gu
 system requirements and more information.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik
 
-docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+docker run -it --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik \
+    -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11
+    docker.all-hands.dev/all-hands-ai/openhands:0.15
 ```
 
 You'll find OpenHands running at [http://localhost:3000](http://localhost:3000)!
@@ -59,7 +60,8 @@ works best, but you have [many options](https://docs.all-hands.dev/modules/usage
 
 You can also [connect OpenHands to your local filesystem](https://docs.all-hands.dev/modules/usage/runtimes),
 run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode),
-or interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode).
+interact with it via a [friendly CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode),
+or run it on tagged issues with [a github action](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
 
 Visit [Installation](https://docs.all-hands.dev/modules/usage/installation) for more information and setup instructions.
 
@@ -75,28 +77,21 @@ To learn more about the project, and for tips on using OpenHands,
 There you'll find resources on how to use different LLM providers,
 troubleshooting resources, and advanced configuration options.
 
-## 🤝 How to Contribute
+## 🤝 How to Join the Community
 
-OpenHands is a community-driven project, and we welcome contributions from everyone.
-Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of
-software engineering with AI, there are many ways to get involved:
+OpenHands is a community-driven project, and we welcome contributions from everyone. We do most of our communication
+through Slack, so this is the best place to start, but we also are happy to have you contact us on Discord or Github:
 
-- **Code Contributions:** Help us develop new agents, core functionality, the frontend and other interfaces, or sandboxing solutions.
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements.
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+- [Join our Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg) - Here we talk about research, architecture, and future development.
+- [Join our Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
+- [Read or post Github Issues](https://github.com/All-Hands-AI/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
 
-For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
-
-## 🤖 Join Our Community
-
-Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
-Let's make software engineering better together!
-
-- [Slack workspace](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) - Here we talk about research, architecture, and future development.
-- [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
+See more about the community in [COMMUNITY.md](./COMMUNITY.md) or find details on contributing in [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ## 📈 Progress
 
+See the monthly OpenHands roadmap [here](https://github.com/orgs/All-Hands-AI/projects/1) (updated at the maintainer's meeting at the end of each month).
+
 <p align="center">
   <a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
     <img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">

compose.yml

@@ -7,7 +7,7 @@ services:
     image: openhands:latest
     container_name: openhands-app-${DATE:-}
     environment:
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.15-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

config.template.toml

@@ -32,7 +32,8 @@ workspace_base = "./workspace"
 # Enable saving and restoring the session when run from CLI
 #enable_cli_session = false
 
-# Path to store trajectories
+# Path to store trajectories, can be a folder or a file
+# If it's a folder, the session id will be used as the file name
 #trajectories_path="./trajectories"
 
 # File store path
@@ -94,10 +95,10 @@ workspace_base = "./workspace"
 # AWS secret access key
 #aws_secret_access_key = ""
 
-# API key to use
+# API key to use (For Headless / CLI only -  In Web this is overridden by Session Init)
 api_key = "your-api-key"
 
-# API base URL
+# API base URL (For Headless / CLI only -  In Web this is overridden by Session Init)
 #base_url = ""
 
 # API version
@@ -130,7 +131,7 @@ embedding_model = "local"
 # Maximum number of output tokens
 #max_output_tokens = 0
 
-# Model to use
+# Model to use. (For Headless / CLI only -  In Web this is overridden by Session Init)
 model = "gpt-4o"
 
 # Number of retries to attempt when an operation fails with the LLM.
@@ -236,10 +237,10 @@ llm_config = 'gpt3'
 ##############################################################################
 [security]
 
-# Enable confirmation mode
+# Enable confirmation mode (For Headless / CLI only -  In Web this is overridden by Session Init)
 #confirmation_mode = false
 
-# The security analyzer to use
+# The security analyzer to use (For Headless / CLI only -  In Web this is overridden by Session Init)
 #security_analyzer = ""
 
 #################################### Eval ####################################

containers/dev/compose.yml

@@ -11,7 +11,7 @@ services:
       - BACKEND_HOST=${BACKEND_HOST:-"0.0.0.0"}
       - SANDBOX_API_HOSTNAME=host.docker.internal
       #
-      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.9-nikolaik}
+      - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/all-hands-ai/runtime:0.15-nikolaik}
       - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234}
       - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace}
     ports:

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/about.md

@@ -27,7 +27,7 @@ Pour plus de détails, veuillez consulter [ce document](https://github.com/All-H
 
 Nous avons à la fois un espace de travail Slack pour la collaboration sur la construction d'OpenHands et un serveur Discord pour discuter de tout ce qui est lié, par exemple, à ce projet, LLM, agent, etc.
 
-- [Espace de travail Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
+- [Espace de travail Slack](https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg)
 - [Serveur Discord](https://discord.gg/ESHStjSjD4)
 
 Si vous souhaitez contribuer, n'hésitez pas à rejoindre notre communauté. Simplifions ensemble l'ingénierie logicielle !

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -76,7 +76,7 @@ La fonction `run_controller()` est le cœur de l'exécution d'OpenHands. Elle g
 
 ## Le moyen le plus simple de commencer : Explorer les benchmarks existants
 
-Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) de notre dépôt.
+Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/benchmarks/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) de notre dépôt.
 
 Pour intégrer votre propre benchmark, nous vous suggérons de commencer par celui qui ressemble le plus à vos besoins. Cette approche peut considérablement rationaliser votre processus d'intégration, vous permettant de vous appuyer sur les structures existantes et de les adapter à vos exigences spécifiques.
 
@@ -161,7 +161,7 @@ Pour créer un workflow d'évaluation pour votre benchmark, suivez ces étapes :
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -260,7 +260,7 @@ def codeact_user_response(state: State | None) -> str:
         # vérifier si l'agent a essayé de parler à l'utilisateur 3 fois, si oui, faire savoir à l'agent qu'il peut abandonner
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:
@@ -279,4 +279,3 @@ Cette fonction fait ce qui suit :
 3. Si l'agent a fait plusieurs tentatives, il lui donne la possibilité d'abandonner
 
 En utilisant cette fonction, vous pouvez garantir un comportement cohérent sur plusieurs exécutions d'évaluation et empêcher l'agent de rester bloqué en attendant une entrée humaine.
-

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -14,4 +14,4 @@ Pour utiliser l'Action GitHub OpenHands dans le dépôt OpenHands, un mainteneur
 
 ## Installation de l'Action dans un nouveau dépôt
 
-Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).
+Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/about.md

@@ -27,7 +27,7 @@ OpenHands 是一个社区驱动的项目，我们欢迎每个人的贡献。无
 
 我们有 Slack 工作区用于协作构建 OpenHands，也有 Discord 服务器用于讨论任何相关的内容，例如此项目、大语言模型、代理等。
 
-- [Slack 工作区](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
+- [Slack 工作区](https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg)
 - [Discord 服务器](https://discord.gg/ESHStjSjD4)
 
 如果你想做出贡献，欢迎加入我们的社区。让我们一起简化软件工程！

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md

@@ -73,7 +73,7 @@ OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工
 
 ## 入门最简单的方法：探索现有基准
 
-我们鼓励您查看我们仓库的 [`evaluation/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation)中提供的各种评估基准。
+我们鼓励您查看我们仓库的 [`evaluation/benchmarks/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks)中提供的各种评估基准。
 
 要集成您自己的基准，我们建议从最接近您需求的基准开始。这种方法可以显著简化您的集成过程，允许您在现有结构的基础上进行构建并使其适应您的特定要求。
 
@@ -158,7 +158,7 @@ OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -257,7 +257,7 @@ def codeact_user_response(state: State | None) -> str:
         # 检查代理是否已尝试与用户对话 3 次，如果是，让代理知道它可以放弃
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -12,4 +12,4 @@
 
 ## 在新仓库中安装 Action
 
-要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow) 进行操作。
+要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md) 进行操作。

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/headless-mode.md

@@ -58,4 +58,3 @@ docker run -it \
     ghcr.io/all-hands-ai/openhands:0.11 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
-

docs/modules/usage/about.md

@@ -1,6 +1,6 @@
-# 📚 Misc
+# About OpenHands
 
-## ⭐️ Research Strategy
+## Research Strategy
 
 Achieving full replication of production-grade applications with LLMs is a complex endeavor. Our strategy involves:
 
@@ -9,34 +9,11 @@ Achieving full replication of production-grade applications with LLMs is a compl
 3. **Task Planning:** Developing capabilities for bug detection, codebase management, and optimization
 4. **Evaluation:** Establishing comprehensive evaluation metrics to better understand and improve our models
 
-## 🚧 Default Agent
+## Default Agent
 
 Our default Agent is currently the [CodeActAgent](agents), which is capable of generating code and handling files.
 
-## 🤝 How to Contribute
-
-OpenHands is a community-driven project, and we welcome contributions from everyone. Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of software engineering with AI, there are many ways to get involved:
-
-- **Code Contributions:** Help us develop the core functionalities, frontend interface, or sandboxing solutions
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability
-
-For details, please check [this document](https://github.com/All-Hands-AI/OpenHands/blob/main/CONTRIBUTING.md).
-
-## 🤖 Join Our Community
-
-We have both Slack workspace for the collaboration on building OpenHands and Discord server for discussion about anything related, e.g., this project, LLM, agent, etc.
-
-- [Slack workspace](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA)
-- [Discord server](https://discord.gg/ESHStjSjD4)
-
-If you would love to contribute, feel free to join our community. Let's simplify software engineering together!
-
-🐚 **Code less, make more with OpenHands.**
-
-[![Star History Chart](https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date)](https://star-history.com/#All-Hands-AI/OpenHands&Date)
-
-## 🛠️ Built With
+## Built With
 
 OpenHands is built using a combination of powerful frameworks and libraries, providing a robust foundation for its development. Here are the key technologies used in the project:
 
@@ -44,6 +21,6 @@ OpenHands is built using a combination of powerful frameworks and libraries, pro
 
 Please note that the selection of these technologies is in progress, and additional technologies may be added or existing ones may be removed as the project evolves. We strive to adopt the most suitable and efficient tools to enhance the capabilities of OpenHands.
 
-## 📜 License
+## License
 
-Distributed under the MIT License. See [our license](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE) for more information.
+Distributed under MIT [License](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE).

docs/modules/usage/configuration-options.md

@@ -0,0 +1,465 @@
+# Configuration Options
+
+This guide details all configuration options available for OpenHands, helping you customize its behavior and integrate it with other services.
+
+:::note
+If you are running in [GUI Mode](https://docs.all-hands.dev/modules/usage/how-to/gui-mode), the settings available in the Settings UI will always
+take precedence.
+:::
+
+---
+
+# Table of Contents
+
+1. [Core Configuration](#core-configuration)
+   - [API Keys](#api-keys)
+   - [Workspace](#workspace)
+   - [Debugging and Logging](#debugging-and-logging)
+   - [Session Management](#session-management)
+   - [Trajectories](#trajectories)
+   - [File Store](#file-store)
+   - [Task Management](#task-management)
+   - [Sandbox Configuration](#sandbox-configuration)
+   - [Miscellaneous](#miscellaneous)
+2. [LLM Configuration](#llm-configuration)
+   - [AWS Credentials](#aws-credentials)
+   - [API Configuration](#api-configuration)
+   - [Custom LLM Provider](#custom-llm-provider)
+   - [Embeddings](#embeddings)
+   - [Message Handling](#message-handling)
+   - [Model Selection](#model-selection)
+   - [Retrying](#retrying)
+   - [Advanced Options](#advanced-options)
+3. [Agent Configuration](#agent-configuration)
+   - [Microagent Configuration](#microagent-configuration)
+   - [Memory Configuration](#memory-configuration)
+   - [LLM Configuration](#llm-configuration-2)
+   - [ActionSpace Configuration](#actionspace-configuration)
+   - [Microagent Usage](#microagent-usage)
+4. [Sandbox Configuration](#sandbox-configuration-2)
+   - [Execution](#execution)
+   - [Container Image](#container-image)
+   - [Networking](#networking)
+   - [Linting and Plugins](#linting-and-plugins)
+   - [Dependencies and Environment](#dependencies-and-environment)
+   - [Evaluation](#evaluation)
+5. [Security Configuration](#security-configuration)
+   - [Confirmation Mode](#confirmation-mode)
+   - [Security Analyzer](#security-analyzer)
+
+---
+
+## Core Configuration
+
+The core configuration options are defined in the `[core]` section of the `config.toml` file.
+
+**API Keys**
+- `e2b_api_key`
+  - Type: `str`
+  - Default: `""`
+  - Description: API key for E2B
+
+- `modal_api_token_id`
+  - Type: `str`
+  - Default: `""`
+  - Description: API token ID for Modal
+
+- `modal_api_token_secret`
+  - Type: `str`
+  - Default: `""`
+  - Description: API token secret for Modal
+
+**Workspace**
+- `workspace_base`
+  - Type: `str`
+  - Default: `"./workspace"`
+  - Description: Base path for the workspace
+
+- `cache_dir`
+  - Type: `str`
+  - Default: `"/tmp/cache"`
+  - Description: Cache directory path
+
+**Debugging and Logging**
+- `debug`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Enable debugging
+
+- `disable_color`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Disable color in terminal output
+
+**Trajectories**
+- `trajectories_path`
+  - Type: `str`
+  - Default: `"./trajectories"`
+  - Description: Path to store trajectories (can be a folder or a file). If it's a folder, the trajectories will be saved in a file named with the session id name and .json extension, in that folder.
+
+**File Store**
+- `file_store_path`
+  - Type: `str`
+  - Default: `"/tmp/file_store"`
+  - Description: File store path
+
+- `file_store`
+  - Type: `str`
+  - Default: `"memory"`
+  - Description: File store type
+
+- `file_uploads_allowed_extensions`
+  - Type: `list of str`
+  - Default: `[".*"]`
+  - Description: List of allowed file extensions for uploads
+
+- `file_uploads_max_file_size_mb`
+  - Type: `int`
+  - Default: `0`
+  - Description: Maximum file size for uploads, in megabytes
+
+- `file_uploads_restrict_file_types`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Restrict file types for file uploads
+
+- `file_uploads_allowed_extensions`
+  - Type: `list of str`
+  - Default: `[".*"]`
+  - Description: List of allowed file extensions for uploads
+
+**Task Management**
+- `max_budget_per_task`
+  - Type: `float`
+  - Default: `0.0`
+  - Description: Maximum budget per task (0.0 means no limit)
+
+- `max_iterations`
+  - Type: `int`
+  - Default: `100`
+  - Description: Maximum number of iterations
+
+**Sandbox Configuration**
+- `workspace_mount_path_in_sandbox`
+  - Type: `str`
+  - Default: `"/workspace"`
+  - Description: Path to mount the workspace in the sandbox
+
+- `workspace_mount_path`
+  - Type: `str`
+  - Default: `""`
+  - Description: Path to mount the workspace
+
+- `workspace_mount_rewrite`
+  - Type: `str`
+  - Default: `""`
+  - Description: Path to rewrite the workspace mount path to. You can usually ignore this, it refers to special cases of running inside another container.
+
+**Miscellaneous**
+- `run_as_openhands`
+  - Type: `bool`
+  - Default: `true`
+  - Description: Run as OpenHands
+
+- `runtime`
+  - Type: `str`
+  - Default: `"eventstream"`
+  - Description: Runtime environment
+
+- `default_agent`
+  - Type: `str`
+  - Default: `"CodeActAgent"`
+  - Description: Name of the default agent
+
+- `jwt_secret`
+  - Type: `str`
+  - Default: `uuid.uuid4().hex`
+  - Description: JWT secret for authentication. Please set it to your own value.
+
+## LLM Configuration
+
+The LLM (Large Language Model) configuration options are defined in the `[llm]` section of the `config.toml` file.
+
+To use these with the docker command, pass in `-e LLM_<option>`. Example: `-e LLM_NUM_RETRIES`.
+
+**AWS Credentials**
+- `aws_access_key_id`
+  - Type: `str`
+  - Default: `""`
+  - Description: AWS access key ID
+
+- `aws_region_name`
+  - Type: `str`
+  - Default: `""`
+  - Description: AWS region name
+
+- `aws_secret_access_key`
+  - Type: `str`
+  - Default: `""`
+  - Description: AWS secret access key
+
+**API Configuration**
+- `api_key`
+  - Type: `str`
+  - Default: `None`
+  - Description: API key to use
+
+- `base_url`
+  - Type: `str`
+  - Default: `""`
+  - Description: API base URL
+
+- `api_version`
+  - Type: `str`
+  - Default: `""`
+  - Description: API version
+
+- `input_cost_per_token`
+  - Type: `float`
+  - Default: `0.0`
+  - Description: Cost per input token
+
+- `output_cost_per_token`
+  - Type: `float`
+  - Default: `0.0`
+  - Description: Cost per output token
+
+**Custom LLM Provider**
+- `custom_llm_provider`
+  - Type: `str`
+  - Default: `""`
+  - Description: Custom LLM provider
+
+**Embeddings**
+- `embedding_base_url`
+  - Type: `str`
+  - Default: `""`
+  - Description: Embedding API base URL
+
+- `embedding_deployment_name`
+  - Type: `str`
+  - Default: `""`
+  - Description: Embedding deployment name
+
+- `embedding_model`
+  - Type: `str`
+  - Default: `"local"`
+  - Description: Embedding model to use
+
+**Message Handling**
+- `max_message_chars`
+  - Type: `int`
+  - Default: `30000`
+  - Description: The approximate maximum number of characters in the content of an event included in the prompt to the LLM. Larger observations are truncated.
+
+- `max_input_tokens`
+  - Type: `int`
+  - Default: `0`
+  - Description: Maximum number of input tokens
+
+- `max_output_tokens`
+  - Type: `int`
+  - Default: `0`
+  - Description: Maximum number of output tokens
+
+**Model Selection**
+- `model`
+  - Type: `str`
+  - Default: `"claude-3-5-sonnet-20241022"`
+  - Description: Model to use
+
+**Retrying**
+- `num_retries`
+  - Type: `int`
+  - Default: `8`
+  - Description: Number of retries to attempt
+
+- `retry_max_wait`
+  - Type: `int`
+  - Default: `120`
+  - Description: Maximum wait time (in seconds) between retry attempts
+
+- `retry_min_wait`
+  - Type: `int`
+  - Default: `15`
+  - Description: Minimum wait time (in seconds) between retry attempts
+
+- `retry_multiplier`
+  - Type: `float`
+  - Default: `2.0`
+  - Description: Multiplier for exponential backoff calculation
+
+**Advanced Options**
+- `drop_params`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Drop any unmapped (unsupported) params without causing an exception
+
+- `caching_prompt`
+  - Type: `bool`
+  - Default: `true`
+  - Description: Using the prompt caching feature if provided by the LLM and supported
+
+- `ollama_base_url`
+  - Type: `str`
+  - Default: `""`
+  - Description: Base URL for the OLLAMA API
+
+- `temperature`
+  - Type: `float`
+  - Default: `0.0`
+  - Description: Temperature for the API
+
+- `timeout`
+  - Type: `int`
+  - Default: `0`
+  - Description: Timeout for the API
+
+- `top_p`
+  - Type: `float`
+  - Default: `1.0`
+  - Description: Top p for the API
+
+- `disable_vision`
+  - Type: `bool`
+  - Default: `None`
+  - Description: If model is vision capable, this option allows to disable image processing (useful for cost reduction)
+
+## Agent Configuration
+
+The agent configuration options are defined in the `[agent]` and `[agent.<agent_name>]` sections of the `config.toml` file.
+
+**Microagent Configuration**
+- `micro_agent_name`
+  - Type: `str`
+  - Default: `""`
+  - Description: Name of the micro agent to use for this agent
+
+**Memory Configuration**
+- `memory_enabled`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Whether long-term memory (embeddings) is enabled
+
+- `memory_max_threads`
+  - Type: `int`
+  - Default: `3`
+  - Description: The maximum number of threads indexing at the same time for embeddings
+
+**LLM Configuration**
+- `llm_config`
+  - Type: `str`
+  - Default: `'your-llm-config-group'`
+  - Description: The name of the LLM config to use
+
+**ActionSpace Configuration**
+- `function_calling`
+  - Type: `bool`
+  - Default: `true`
+  - Description: Whether function calling is enabled
+
+- `codeact_enable_browsing`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Whether browsing delegate is enabled in the action space (only works with function calling)
+
+- `codeact_enable_llm_editor`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Whether LLM editor is enabled in the action space (only works with function calling)
+
+- `codeact_enable_jupyter`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Whether Jupyter is enabled in the action space
+
+**Microagent Usage**
+- `use_microagents`
+  - Type: `bool`
+  - Default: `true`
+  - Description: Whether to use microagents at all
+
+- `disabled_microagents`
+  - Type: `list of str`
+  - Default: `None`
+  - Description: A list of microagents to disable
+
+## Sandbox Configuration
+
+The sandbox configuration options are defined in the `[sandbox]` section of the `config.toml` file.
+
+To use these with the docker command, pass in `-e SANDBOX_<option>`. Example: `-e SANDBOX_TIMEOUT`.
+
+**Execution**
+- `timeout`
+  - Type: `int`
+  - Default: `120`
+  - Description: Sandbox timeout in seconds
+
+- `user_id`
+  - Type: `int`
+  - Default: `1000`
+  - Description: Sandbox user ID
+
+**Container Image**
+- `base_container_image`
+  - Type: `str`
+  - Default: `"nikolaik/python-nodejs:python3.12-nodejs22"`
+  - Description: Container image to use for the sandbox
+
+**Networking**
+- `use_host_network`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Use host network
+
+**Linting and Plugins**
+- `enable_auto_lint`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Enable auto linting after editing
+
+- `initialize_plugins`
+  - Type: `bool`
+  - Default: `true`
+  - Description: Whether to initialize plugins
+
+**Dependencies and Environment**
+- `runtime_extra_deps`
+  - Type: `str`
+  - Default: `""`
+  - Description: Extra dependencies to install in the runtime image
+
+- `runtime_startup_env_vars`
+  - Type: `dict`
+  - Default: `{}`
+  - Description: Environment variables to set at the launch of the runtime
+
+**Evaluation**
+- `browsergym_eval_env`
+  - Type: `str`
+  - Default: `""`
+  - Description: BrowserGym environment to use for evaluation
+
+## Security Configuration
+
+The security configuration options are defined in the `[security]` section of the `config.toml` file.
+
+To use these with the docker command, pass in `-e SECURITY_<option>`. Example: `-e SECURITY_CONFIRMATION_MODE`.
+
+**Confirmation Mode**
+- `confirmation_mode`
+  - Type: `bool`
+  - Default: `false`
+  - Description: Enable confirmation mode
+
+**Security Analyzer**
+- `security_analyzer`
+  - Type: `str`
+  - Default: `""`
+  - Description: The security analyzer to use
+
+---
+
+> **Note**: Adjust configurations carefully, especially for memory, security, and network-related settings to ensure optimal performance and security.
+Please note that the configuration options may be subject to change in future versions of OpenHands. It's recommended to refer to the official documentation for the most up-to-date information.

docs/modules/usage/how-to/cli-mode.md

@@ -50,6 +50,7 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
@@ -58,7 +59,7 @@ docker run -it \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.15 \
     python -m openhands.core.cli
 ```
 
@@ -107,4 +108,3 @@ Expected Output:
 ```bash
 🤖 An error occurred. Please try again.
 ```
-

docs/modules/usage/how-to/custom-sandbox-guide.md

@@ -62,25 +62,3 @@ Run OpenHands by running ```make run``` in the top level directory.
 ## Technical Explanation
 
 Please refer to [custom docker image section of the runtime documentation](https://docs.all-hands.dev/modules/usage/architecture/runtime#advanced-how-openhands-builds-and-maintains-od-runtime-images) for more details.
-
-## Troubleshooting / Errors
-
-### Error: ```useradd: UID 1000 is not unique```
-
-If you see this error in the console output it is because OpenHands is trying to create the openhands user in the sandbox with a UID of 1000, however this UID is already being used in the image (for some reason). To fix this change the sandbox_user_id field in the config.toml file to a different value:
-
-```toml
-[core]
-workspace_base="./workspace"
-run_as_openhands=true
-sandbox_base_container_image="custom_image"
-sandbox_user_id="1001"
-```
-
-### Port use errors
-
-If you see an error about a port being in use or unavailable, try deleting all running Docker Containers (run `docker ps` and `docker rm` relevant containers) and then re-running ```make run``` .
-
-## Discuss
-
-For other issues or questions join the [Slack](https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA) or [Discord](https://discord.gg/ESHStjSjD4) and ask!

docs/modules/usage/how-to/evaluation-harness.md

@@ -73,7 +73,7 @@ The `run_controller()` function is the core of OpenHands's execution. It manages
 
 ## Easiest way to get started: Exploring Existing Benchmarks
 
-We encourage you to review the various evaluation benchmarks available in the [`evaluation/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) of our repository.
+We encourage you to review the various evaluation benchmarks available in the [`evaluation/benchmarks/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) of our repository.
 
 To integrate your own benchmark, we suggest starting with the one that most closely resembles your needs. This approach can significantly streamline your integration process, allowing you to build upon existing structures and adapt them to your specific requirements.
 
@@ -158,7 +158,7 @@ To create an evaluation workflow for your benchmark, follow these steps:
            instruction=instruction,
            test_result=evaluation_result,
            metadata=metadata,
-           history=state.history.compatibility_for_eval_history_pairs(),
+           history=compatibility_for_eval_history_pairs(state.history),
            metrics=state.metrics.get() if state.metrics else None,
            error=state.last_error if state and state.last_error else None,
        )
@@ -257,7 +257,7 @@ def codeact_user_response(state: State | None) -> str:
         # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) >= 2:

docs/modules/usage/how-to/github-action.md

@@ -4,12 +4,101 @@ This guide explains how to use the OpenHands GitHub Action, both within the Open
 
 ## Using the Action in the OpenHands Repository
 
-To use the OpenHands GitHub Action in the OpenHands repository, an OpenHands maintainer can:
+To use the OpenHands GitHub Action in a repository, you can:
 
 1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue.
-3. The action will automatically trigger and attempt to resolve the issue.
+2. Add the `fix-me` label to the issue or leave a comment on the issue starting with `@openhands-agent`.
+
+The action will automatically trigger and attempt to resolve the issue.
 
 ## Installing the Action in a New Repository
 
-To install the OpenHands GitHub Action in your own repository, follow the [directions in the OpenHands Resolver repo](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).
+To install the OpenHands GitHub Action in your own repository, follow
+the [README for the OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
+
+## Usage Tips
+
+### Iterative resolution
+
+1. Create an issue in the repository.
+2. Add the `fix-me` label to the issue, or leave a comment starting with `@openhands-agent`
+3. Review the attempt to resolve the issue by checking the pull request
+4. Follow up with feedback through general comments, review comments, or inline thread comments
+5. Add the `fix-me` label to the pull request, or address a specific comment by starting with `@openhands-agent`
+
+### Label versus Macro
+
+- Label (`fix-me`): Requests OpenHands to address the **entire** issue or pull request.
+- Macro (`@openhands-agent`): Requests OpenHands to consider only the issue/pull request description and **the specific comment**.
+
+## Advanced Settings
+
+### Add custom repository settings
+
+You can provide custom directions for OpenHands by following the [README for the resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md#providing-custom-instructions).
+
+### Custom configurations
+
+Github resolver will automatically check for valid [repository secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions?tool=webui#creating-secrets-for-a-repository) or [repository variables](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#creating-configuration-variables-for-a-repository) to customize its behavior. The customization options you can set are:
+
+| **Attribute name**               | **Type** | **Purpose**                                                                                         | **Example**                                     |
+| -------------------------------- | -------- | --------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
+| `OPENHANDS_MAX_ITER`             | Variable | Set max limit for agent iterations                                                                  | `OPENHANDS_MAX_ITER=10`                         |
+| `OPENHANDS_MACRO`                | Variable | Customize default macro for invoking the resolver                                                   | `OPENHANDS_MACRO=@resolveit`                    |
+| `OPENHANDS_BASE_CONTAINER_IMAGE` | Variable | Custom Sandbox ([learn more](https://docs.all-hands.dev/modules/usage/how-to/custom-sandbox-guide)) | `OPENHANDS_BASE_CONTAINER_IMAGE="custom_image"` |
+
+## Writing Effective .openhands_instructions Files
+
+The `.openhands_instructions` file is a file that you can put in the root directory of your repository to guide OpenHands in understanding and working with your repository effectively. Here are key tips for writing high-quality instructions:
+
+### Core Principles
+
+1. **Concise but Informative**: Provide a clear, focused overview of the repository that emphasizes the most common actions OpenHands will need to perform.
+
+2. **Repository Structure**: Explain the key directories and their purposes, especially highlighting where different types of code (e.g., frontend, backend) are located.
+
+3. **Development Workflows**: Document the essential commands for:
+
+   - Building and setting up the project
+   - Running tests
+   - Linting and code quality checks
+   - Any environment-specific requirements
+
+4. **Testing Guidelines**: Specify:
+   - Where tests are located
+   - How to run specific test suites
+   - Any testing conventions or requirements
+
+### Example Structure
+
+```markdown
+# Repository Overview
+
+[Brief description of the project]
+
+## General Setup
+
+- Main build command
+- Development environment setup
+- Pre-commit checks
+
+## Backend
+
+- Location and structure
+- Testing instructions
+- Environment requirements
+
+## Frontend
+
+- Setup prerequisites
+- Build and test commands
+- Environment variables
+
+## Additional Guidelines
+
+- Code style requirements
+- Special considerations
+- Common workflows
+```
+
+For a real-world example, refer to the [OpenHands repository's .openhands_instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/.openhands_instructions).

docs/modules/usage/how-to/gui-mode.md

@@ -19,6 +19,80 @@ OpenHands provides a user-friendly Graphical User Interface (GUI) mode for inter
 3. Enter the corresponding `API Key` for your chosen provider.
 4. Click "Save" to apply the settings.
 
+### GitHub Token Setup
+
+OpenHands automatically exports a `GITHUB_TOKEN` to the shell environment if it is available. This can happen in two ways:
+
+1. **Locally (OSS)**: The user directly inputs their GitHub token
+2. **Online (SaaS)**: The token is obtained through GitHub OAuth authentication
+
+#### Setting Up a Local GitHub Token
+
+1. **Generate a Personal Access Token (PAT)**:
+   - Go to GitHub Settings > Developer Settings > Personal Access Tokens > Tokens (classic)
+   - Click "Generate new token (classic)"
+   - Required scopes:
+     - `repo` (Full control of private repositories)
+     - `workflow` (Update GitHub Action workflows)
+     - `read:org` (Read organization data)
+
+2. **Enter Token in OpenHands**:
+   - Click the Settings button (gear icon) in the top right
+   - Navigate to the "GitHub" section
+   - Paste your token in the "GitHub Token" field
+   - Click "Save" to apply the changes
+
+#### Organizational Token Policies
+
+If you're working with organizational repositories, additional setup may be required:
+
+1. **Check Organization Requirements**:
+   - Organization admins may enforce specific token policies
+   - Some organizations require tokens to be created with SSO enabled
+   - Review your organization's [token policy settings](https://docs.github.com/en/organizations/managing-programmatic-access-to-your-organization/setting-a-personal-access-token-policy-for-your-organization)
+
+2. **Verify Organization Access**:
+   - Go to your token settings on GitHub
+   - Look for the organization under "Organization access"
+   - If required, click "Enable SSO" next to your organization
+   - Complete the SSO authorization process
+
+#### OAuth Authentication (Online Mode)
+
+When using OpenHands in online mode, the GitHub OAuth flow:
+
+1. Requests the following permissions:
+   - Repository access (read/write)
+   - Workflow management
+   - Organization read access
+
+2. Authentication steps:
+   - Click "Sign in with GitHub" when prompted
+   - Review the requested permissions
+   - Authorize OpenHands to access your GitHub account
+   - If using an organization, authorize organization access if prompted
+
+#### Troubleshooting
+
+Common issues and solutions:
+
+1. **Token Not Recognized**:
+   - Ensure the token is properly saved in settings
+   - Check that the token hasn't expired
+   - Verify the token has the required scopes
+   - Try regenerating the token
+
+2. **Organization Access Denied**:
+   - Check if SSO is required but not enabled
+   - Verify organization membership
+   - Contact organization admin if token policies are blocking access
+
+3. **Verifying Token Works**:
+   - The app will show a green checkmark if the token is valid
+   - Try accessing a repository to confirm permissions
+   - Check the browser console for any error messages
+   - Use the "Test Connection" button in settings if available
+
 ### Advanced Settings
 
 1. Toggle `Advanced Options` to access additional settings.

docs/modules/usage/how-to/headless-mode.md

@@ -44,15 +44,16 @@ LLM_API_KEY="sk_test_12345"
 ```bash
 docker run -it \
     --pull=always \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik \
     -e SANDBOX_USER_ID=$(id -u) \
     -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \
     -e LLM_API_KEY=$LLM_API_KEY \
     -e LLM_MODEL=$LLM_MODEL \
+    -e LOG_ALL_EVENTS=true \
     -v $WORKSPACE_BASE:/opt/workspace_base \
     -v /var/run/docker.sock:/var/run/docker.sock \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app-$(date +%Y%m%d%H%M%S) \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11 \
+    docker.all-hands.dev/all-hands-ai/openhands:0.15 \
     python -m openhands.core.main -t "write a bash script that prints hi"
 ```
-

docs/modules/usage/installation.mdx

@@ -11,15 +11,16 @@
 The easiest way to run OpenHands is in Docker.
 
 ```bash
-docker pull docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik
+docker pull docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik
 
 docker run -it --rm --pull=always \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik \
+    -e LOG_ALL_EVENTS=true \
     -v /var/run/docker.sock:/var/run/docker.sock \
     -p 3000:3000 \
     --add-host host.docker.internal:host-gateway \
     --name openhands-app \
-    docker.all-hands.dev/all-hands-ai/openhands:0.11
+    docker.all-hands.dev/all-hands-ai/openhands:0.15
 ```
 
 You can also run OpenHands in a scriptable [headless mode](https://docs.all-hands.dev/modules/usage/how-to/headless-mode), as an [interactive CLI](https://docs.all-hands.dev/modules/usage/how-to/cli-mode), or using the [OpenHands GitHub Action](https://docs.all-hands.dev/modules/usage/how-to/github-action).

docs/modules/usage/llms/litellm-proxy.md

@@ -0,0 +1,20 @@
+# LiteLLM Proxy
+
+OpenHands supports using the [LiteLLM proxy](https://docs.litellm.ai/docs/proxy/quick_start) to access various LLM providers.
+
+## Configuration
+
+To use LiteLLM proxy with OpenHands, you need to:
+
+1. Set up a LiteLLM proxy server (see [LiteLLM documentation](https://docs.litellm.ai/docs/proxy/quick_start))
+2. When running OpenHands, you'll need to set the following in the OpenHands UI through the Settings:
+  * Enable `Advanced Options`
+  * `Custom Model` to the prefix `litellm_proxy/` + the model you will be using (e.g. `litellm_proxy/anthropic.claude-3-5-sonnet-20241022-v2:0`)
+  * `Base URL` to your LiteLLM proxy URL (e.g. `https://your-litellm-proxy.com`)
+  * `API Key` to your LiteLLM proxy API key
+
+## Supported Models
+
+The supported models depend on your LiteLLM proxy configuration. OpenHands supports any model that your LiteLLM proxy is configured to handle.
+
+Refer to your LiteLLM proxy configuration for the list of available models and their names.

docs/modules/usage/llms/llms.md

@@ -4,11 +4,11 @@ OpenHands can connect to any LLM supported by LiteLLM. However, it requires a po
 
 ## Model Recommendations
 
-Based on a recent evaluation of language models for coding tasks (using the SWE-bench dataset), we can provide some recommendations for model selection. The full analysis can be found in [this blog article](https://www.all-hands.dev/blog/evaluation-of-llms-as-coding-agents-on-swe-bench-at-30x-speed).
+Based on our evaluations of language models for coding tasks (using the SWE-bench dataset), we can provide some recommendations for model selection. Some analyses can be found in [this blog article comparing LLMs](https://www.all-hands.dev/blog/evaluation-of-llms-as-coding-agents-on-swe-bench-at-30x-speed) and [this blog article with some more recent results](https://www.all-hands.dev/blog/openhands-codeact-21-an-open-state-of-the-art-software-development-agent).
 
 When choosing a model, consider both the quality of outputs and the associated costs. Here's a summary of the findings:
 
-- Claude 3.5 Sonnet is the best by a fair amount, achieving a 27% resolve rate with the default agent in OpenHands.
+- Claude 3.5 Sonnet is the best by a fair amount, achieving a 53% resolve rate on SWE-Bench Verified with the default agent in OpenHands.
 - GPT-4o lags behind, and o1-mini actually performed somewhat worse than GPT-4o. We went in and analyzed the results a little, and briefly it seemed like o1 was sometimes "overthinking" things, performing extra environment configuration tasks when it could just go ahead and finish the task.
 - Finally, the strongest open models were Llama 3.1 405 B and deepseek-v2.5, and they performed reasonably, even besting some of the closed models.
 
@@ -63,6 +63,7 @@ We have a few guides for running OpenHands with specific model providers:
 - [Azure](llms/azure-llms)
 - [Google](llms/google-llms)
 - [Groq](llms/groq)
+- [LiteLLM Proxy](llms/litellm-proxy)
 - [OpenAI](llms/openai-llms)
 - [OpenRouter](llms/openrouter)
 

docs/modules/usage/runtimes.md

@@ -16,7 +16,7 @@ some flags being passed to `docker run` that make this possible:
 
 ```
 docker run # ...
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.15-nikolaik \
     -v /var/run/docker.sock:/var/run/docker.sock \
     # ...
 ```
@@ -49,7 +49,7 @@ but seems to work well on most systems.
 
 ## All Hands Runtime
 The All Hands Runtime is currently in beta. You can request access by joining
-the #remote-runtime-limited-beta channel on Slack (see the README for an invite).
+the #remote-runtime-limited-beta channel on Slack ([see the README](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-join-our-community) for an invite).
 
 To use the All Hands Runtime, set the following environment variables when
 starting OpenHands:
@@ -59,15 +59,14 @@ docker run # ...
     -e RUNTIME=remote \
     -e SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.app.all-hands.dev" \
     -e SANDBOX_API_KEY="your-all-hands-api-key" \
-    -e SANDBOX_KEEP_REMOTE_RUNTIME_ALIVE="true" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
+    -e SANDBOX_KEEP_RUNTIME_ALIVE="true" \
     # ...
 ```
 
 ## Modal Runtime
 Our partners at [Modal](https://modal.com/) have also provided a runtime for OpenHands.
 
-To use the Modal Runtime, create an account, and then [create an API key](https://modal.com/settings)
+To use the Modal Runtime, create an account, and then [create an API key.](https://modal.com/settings)
 
 You'll then need to set the following environment variables when starting OpenHands:
 ```bash
@@ -75,5 +74,4 @@ docker run # ...
     -e RUNTIME=modal \
     -e MODAL_API_TOKEN_ID="your-id" \
     -e MODAL_API_TOKEN_SECRET="your-secret" \
-    -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.all-hands.dev/all-hands-ai/runtime:0.11-nikolaik \
 ```

docs/modules/usage/troubleshooting/troubleshooting.md

@@ -17,6 +17,7 @@ Check out [Notes for WSL on Windows Users](troubleshooting/windows) for some tro
 * [`make build` getting stuck on package installations](#make-build-getting-stuck-on-package-installations)
 * [Sessions are not restored](#sessions-are-not-restored)
 * [Connection to host.docker.internal timed out](#connection-to-host-docker-internal-timed-out)
+* [Error building runtime docker image](#error-building-runtime-docker-image)
 
 ### Unable to connect to Docker
 
@@ -178,3 +179,21 @@ which OpenHands makes use of when the main server is running inside a docker con
 * [Install Docker Desktop](https://www.docker.com/products/docker-desktop/)
 * Run OpenHands in [Development Mode](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md),
   So that the main server is not run inside a container, but still creates dockerized runtime sandboxes.
+
+---
+### Error building runtime docker image
+
+**Symptoms**
+Attempts to start a new session fail, and an errors with terms like the following appear in the logs:
+* `debian-security bookworm-security`
+* `InRelease At least one invalid signature was encountered.`
+
+This seems to happen when the hash of an existing external library changes and your local docker instance has
+cached a previous version. To work around this, please try the following:
+
+* Stop any containers where the name has the prefix `openhands-runtime-` :
+  `docker ps --filter name=openhands-runtime- --filter status=running -aq | xargs docker stop`
+* Remove any containers where the name has the prefix `openhands-runtime-` :
+  `docker rmi $(docker images --filter name=openhands-runtime- -q --no-trunc)`
+* Stop and Remove any containers / images where the name has the prefix `openhands-runtime-`
+* Prune containers / images : `docker container prune -f && docker image prune -f`

docs/package.json

@@ -15,23 +15,23 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.5.2",
-    "@docusaurus/plugin-content-pages": "^3.5.2",
-    "@docusaurus/preset-classic": "^3.5.2",
-    "@docusaurus/theme-mermaid": "^3.5.2",
+    "@docusaurus/core": "^3.6.3",
+    "@docusaurus/plugin-content-pages": "^3.6.3",
+    "@docusaurus/preset-classic": "^3.6.3",
+    "@docusaurus/theme-mermaid": "^3.6.3",
     "@mdx-js/react": "^3.1.0",
     "clsx": "^2.0.0",
     "prism-react-renderer": "^2.4.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
-    "react-icons": "^5.3.0",
-    "react-use": "^17.5.1"
+    "react-icons": "^5.4.0",
+    "react-use": "^17.6.0"
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.1",
-    "@docusaurus/tsconfig": "^3.5.2",
+    "@docusaurus/tsconfig": "^3.6.3",
     "@docusaurus/types": "^3.5.1",
-    "typescript": "~5.6.3"
+    "typescript": "~5.7.2"
   },
   "browserslist": {
     "production": [

docs/sidebars.ts

@@ -76,6 +76,11 @@ const sidebars: SidebarsConfig = {
                   label: 'Groq',
                   id: 'usage/llms/groq',
                 },
+                {
+                  type: 'doc',
+                  label: 'LiteLLM Proxy',
+                  id: 'usage/llms/litellm-proxy',
+                },
                 {
                   type: 'doc',
                   label: 'OpenAI',
@@ -95,6 +100,11 @@ const sidebars: SidebarsConfig = {
           label: 'Runtime Configuration',
           id: 'usage/runtimes',
         },
+        {
+          type: 'doc',
+          label: 'Configuration Options',
+          id: 'usage/configuration-options',
+        },
         {
           type: 'doc',
           label: 'Custom Sandbox',

docs/src/components/CustomFooter.tsx

@@ -8,7 +8,7 @@ function CustomFooter() {
     <footer className="custom-footer">
       <div className="footer-content">
         <div className="footer-icons">
-          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA" target="_blank" rel="noopener noreferrer">
+          <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg" target="_blank" rel="noopener noreferrer">
             <FaSlack />
           </a>
           <a href="https://discord.gg/ESHStjSjD4" target="_blank" rel="noopener noreferrer">

docs/src/components/HomepageHeader/HomepageHeader.tsx

@@ -23,7 +23,7 @@ export function HomepageHeader() {
           <a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" /></a>
           <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License" /></a>
           <br/>
-          <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
+          <a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2vbfigwev-G03twSpXaErwzYVD4CFiBg"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
           <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community" /></a>
           <a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits" /></a>
           <br/>

evaluation/README.md

@@ -6,9 +6,9 @@ This folder contains code and resources to run experiments and evaluations.
 
 ### Setup
 
-Before starting evaluation, follow the instructions here [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
+Before starting evaluation, follow the instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
 
-Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the evaluation directory.
+Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the [evaluation directory](#supported-benchmarks).
 Generally these will involve running `run_infer.py` to perform inference with the agents.
 
 ### Implementing and Evaluating an Agent
@@ -42,32 +42,36 @@ temperature = 0.0
 
 ## Supported Benchmarks
 
-The OpenHands evaluation harness supports a wide variety of benchmarks across software engineering, web browsing, and miscellaneous assistance tasks.
+The OpenHands evaluation harness supports a wide variety of benchmarks across [software engineering](#software-engineering), [web browsing](#web-browsing), and [miscellaneous assistance](#misc-assistance) tasks.
 
 ### Software Engineering
 
-- SWE-Bench: [`evaluation/swe_bench`](./swe_bench)
-- HumanEvalFix: [`evaluation/humanevalfix`](./humanevalfix)
-- BIRD: [`evaluation/bird`](./bird)
-- BioCoder: [`evaluation/ml_bench`](./ml_bench)
-- ML-Bench: [`evaluation/ml_bench`](./ml_bench)
-- APIBench: [`evaluation/gorilla`](./gorilla/)
-- ToolQA: [`evaluation/toolqa`](./toolqa/)
-- AiderBench: [`evaluation/aider_bench`](./aider_bench/)
+- SWE-Bench: [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench)
+- HumanEvalFix: [`evaluation/benchmarks/humanevalfix`](./benchmarks/humanevalfix)
+- BIRD: [`evaluation/benchmarks/bird`](./benchmarks/bird)
+- BioCoder: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
+- ML-Bench: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
+- APIBench: [`evaluation/benchmarks/gorilla`](./benchmarks/gorilla/)
+- ToolQA: [`evaluation/benchmarks/toolqa`](./benchmarks/toolqa/)
+- AiderBench: [`evaluation/benchmarks/aider_bench`](./benchmarks/aider_bench/)
+- Commit0: [`evaluation/benchmarks/commit0_bench`](./benchmarks/commit0_bench/)
+- DiscoveryBench: [`evaluation/benchmarks/discoverybench`](./benchmarks/discoverybench/)
 
 ### Web Browsing
 
-- WebArena: [`evaluation/webarena`](./webarena/)
-- MiniWob++: [`evaluation/miniwob`](./miniwob/)
+- WebArena: [`evaluation/benchmarks/webarena`](./benchmarks/webarena/)
+- MiniWob++: [`evaluation/benchmarks/miniwob`](./benchmarks/miniwob/)
+- Browsing Delegation: [`evaluation/benchmarks/browsing_delegation`](./benchmarks/browsing_delegation/)
 
 ### Misc. Assistance
 
-- GAIA: [`evaluation/gaia`](./gaia)
-- GPQA: [`evaluation/gpqa`](./gpqa)
-- AgentBench: [`evaluation/agent_bench`](./agent_bench)
-- MINT: [`evaluation/mint`](./mint)
-- Entity deduction Arena (EDA): [`evaluation/EDA`](./EDA)
-- ProofWriter: [`evaluation/logic_reasoning`](./logic_reasoning)
+- GAIA: [`evaluation/benchmarks/gaia`](./benchmarks/gaia)
+- GPQA: [`evaluation/benchmarks/gpqa`](./benchmarks/gpqa)
+- AgentBench: [`evaluation/benchmarks/agent_bench`](./benchmarks/agent_bench)
+- MINT: [`evaluation/benchmarks/mint`](./benchmarks/mint)
+- Entity deduction Arena (EDA): [`evaluation/benchmarks/EDA`](./benchmarks/EDA)
+- ProofWriter: [`evaluation/benchmarks/logic_reasoning`](./benchmarks/logic_reasoning)
+- ScienceAgentBench: [`evaluation/benchmarks/scienceagentbench`](./benchmarks/scienceagentbench)
 
 ## Result Visualization
 
@@ -79,9 +83,8 @@ You can start your own fork of [our huggingface evaluation outputs](https://hugg
 
 To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/how-to/evaluation-harness). Briefly,
 
-- Each subfolder contains a specific benchmark or experiment. For example, `evaluation/swe_bench` should contain
+- Each subfolder contains a specific benchmark or experiment. For example, [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench) should contain
 all the preprocessing/evaluation/analysis scripts.
 - Raw data and experimental records should not be stored within this repo.
 - For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.
 - Important data files of manageable size and analysis scripts (e.g., jupyter notebooks) can be directly uploaded to this repo.
-

evaluation/benchmarks/EDA/README.md

@@ -4,15 +4,13 @@ This folder contains evaluation harness for evaluating agents on the Entity-dedu
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
-
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## Start the evaluation
 
-
 ```bash
 export OPENAI_API_KEY="sk-XXX"; # This is required for evaluation (to simulate another party of conversation)
-./evaluation/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
+./evaluation/benchmarks/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
 ```
 
 where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@@ -33,11 +31,12 @@ to `CodeActAgent`.
 For example,
 
 ```bash
-./evaluation/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
+./evaluation/benchmarks/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
 ```
 
 ## Reference
-```
+
+```bibtex
 @inproceedings{zhang2023entity,
   title={Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games},
   author={Zhang, Yizhe and Lu, Jiarui and Jaitly, Navdeep},

evaluation/benchmarks/EDA/game.py

@@ -87,9 +87,7 @@ class Q20Game:
         # others
         bingo, anwser_reply = self.judge_winner(response)
         if bingo:
-            return (
-                'You are bingo! quit now, run: <execute_bash> exit </execute_bash>.\n'
-            )
+            return 'You are bingo! Use the "finish" tool to finish the interaction.\n'
         if self.curr_turn == self.num_turns - 2:
             anwser_reply += " You must guess now, what's it?"
         return anwser_reply

evaluation/benchmarks/EDA/run_infer.py

@@ -4,10 +4,11 @@ import os
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.EDA.game import Q20Game, Q20GameCelebrity
+from evaluation.benchmarks.EDA.game import Q20Game, Q20GameCelebrity
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -34,7 +35,8 @@ def codeact_user_response_eda(state: State) -> str:
 
     # retrieve the latest model message from history
     if state.history:
-        model_guess = state.history.get_last_agent_message()
+        last_agent_message = state.get_last_agent_message()
+        model_guess = last_agent_message.content if last_agent_message else ''
 
     assert game is not None, 'Game is not initialized.'
     msg = game.generate_user_response(model_guess)
@@ -139,7 +141,8 @@ def process_instance(
     if state is None:
         raise ValueError('State should not be None.')
 
-    final_message = state.history.get_last_agent_message()
+    last_agent_message = state.get_last_agent_message()
+    final_message = last_agent_message.content if last_agent_message else ''
 
     logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
     test_result = game.reward()
@@ -148,7 +151,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/EDA/scripts/run_infer.sh

@@ -21,7 +21,7 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 if [ -z "$DATASET" ]; then
   echo "Dataset not specified, use default 'things'"
@@ -34,16 +34,13 @@ if [ -z "$OPENAI_API_KEY" ]; then
   exit 1
 fi
 
-# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenHands
-# We need to track the version of Agent in the evaluation to make sure results are comparable
-AGENT_VERSION=v$(poetry run python -c "import openhands.agenthub; from openhands.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 echo "DATASET: $DATASET"
 
-COMMAND="poetry run python evaluation/EDA/run_infer.py \
+COMMAND="poetry run python evaluation/benchmarks/EDA/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --dataset $DATASET \
@@ -51,7 +48,7 @@ COMMAND="poetry run python evaluation/EDA/run_infer.py \
   --max-iterations 20 \
   --OPENAI_API_KEY $OPENAI_API_KEY \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note ${AGENT_VERSION}_${DATASET}"
+  --eval-note ${OPENHANDS_VERSION}_${DATASET}"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/benchmarks/agent_bench/README.md

@@ -4,12 +4,12 @@ This folder contains evaluation harness for evaluating agents on the [AgentBench
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## Start the evaluation
 
 ```bash
-./evaluation/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
 ```
 
 - `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
@@ -25,7 +25,7 @@ in order to use `eval_limit`, you must also set `agent`.
 
 Following is the basic command to start the evaluation.
 
-You can update the arguments in the script `evaluation/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
+You can update the arguments in the script `evaluation/benchmarks/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
 
 - `--agent-cls`, the agent to use. For example, `CodeActAgent`.
 - `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
@@ -34,5 +34,23 @@ You can update the arguments in the script `evaluation/agent_bench/scripts/run_i
 - `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
 
 ```bash
-./evaluation/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
 ```
+
+## Run with Remote Runtime (experimental)
+
+You can run the evaluation using a remote runtime instead of a local Docker container. This is useful when you want to run the evaluation in a cloud environment or when you don't have Docker installed locally.
+
+To use the remote runtime, set the following environment variables:
+
+```bash
+# Required environment variables
+export ALLHANDS_API_KEY="your-api-key"  # Contact the team to get an API key
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+
+# Run the evaluation
+./evaluation/benchmarks/agent_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 1
+```
+
+The remote runtime will build a container image and run the evaluation in a cloud environment. The results will be saved locally in the same way as when running with a local runtime.

evaluation/benchmarks/agent_bench/run_infer.py

@@ -7,7 +7,7 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.agent_bench.helper import (
+from evaluation.benchmarks.agent_bench.helper import (
     FAKE_RESPONSES,
     INST_SUFFIXES,
     compare_results,
@@ -16,6 +16,7 @@ from evaluation.agent_bench.helper import (
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -42,12 +43,16 @@ def get_config(
     config = AppConfig(
         default_agent=metadata.agent_class,
         run_as_openhands=False,
-        runtime='eventstream',
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
         max_iterations=metadata.max_iterations,
         sandbox=SandboxConfig(
-            base_container_image='python:3.12-bookworm',
+            base_container_image='python:3.12-slim',
             enable_auto_lint=True,
             use_host_network=False,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_runtime_alive=False,
+            remote_runtime_init_timeout=3600,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -242,7 +247,7 @@ def process_instance(
         raw_ans = ''
 
         # retrieve the last agent message or thought
-        for event in state.history.get_events(reverse=True):
+        for event in reversed(state.history):
             if event.source == 'agent':
                 if isinstance(event, AgentFinishAction):
                     raw_ans = event.thought
@@ -271,7 +276,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     metrics = state.metrics.get() if state.metrics else None
 

evaluation/benchmarks/agent_bench/scripts/run_infer.sh

@@ -20,18 +20,18 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-COMMAND="export PYTHONPATH=evaluation/agent_bench:\$PYTHONPATH && poetry run python evaluation/agent_bench/run_infer.py \
+COMMAND="export PYTHONPATH=evaluation/benchmarks/agent_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/agent_bench/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 30 \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note $AGENT_VERSION"
+  --eval-note $OPENHANDS_VERSION"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/benchmarks/aider_bench/README.md

@@ -10,13 +10,13 @@ Hugging Face dataset based on the
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local
+Please follow instruction [here](../../README.md#setup) to setup your local
 development environment and LLM.
 
 ## Start the evaluation
 
 ```bash
-./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
 ```
 
 - `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
@@ -42,7 +42,7 @@ export SKIP_NUM=12 # skip the first 12 instances from the dataset
 Following is the basic command to start the evaluation.
 
 You can update the arguments in the script
-`evaluation/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
+`evaluation/benchmarks/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
 `--eval-num-workers` and so on:
 
 - `--agent-cls`, the agent to use. For example, `CodeActAgent`.
@@ -53,19 +53,33 @@ You can update the arguments in the script
 - `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`.
 
 ```bash
-./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
+```
+
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
+
+# Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
+export ALLHANDS_API_KEY="YOUR-API-KEY"
+export RUNTIME=remote
+export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
+./evaluation/benchmarks/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
 ```
 
 ## Summarize Results
 
 ```bash
-poetry run python ./evaluation/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
+poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
 ```
 
 Full example:
 
 ```bash
-poetry run python ./evaluation/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
+poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
 ```
 
 This will list the instances that passed and the instances that failed. For each

evaluation/benchmarks/aider_bench/run_infer.py

@@ -7,7 +7,7 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.aider_bench.helper import (
+from evaluation.benchmarks.aider_bench.helper import (
     FAKE_RESPONSES,
     INST_SUFFIXES,
     INSTRUCTIONS_ADDENDUM,
@@ -15,6 +15,7 @@ from evaluation.aider_bench.helper import (
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -57,6 +58,9 @@ def get_config(
             use_host_network=False,
             timeout=100,
             api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_runtime_alive=False,
+            remote_runtime_init_timeout=1800,
         ),
         # do not mount workspace
         workspace_base=None,
@@ -250,7 +254,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
     metrics = state.metrics.get() if state.metrics else None
 
     # Save the output

evaluation/benchmarks/aider_bench/scripts/run_infer.sh

@@ -21,13 +21,13 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-EVAL_NOTE=$AGENT_VERSION
+EVAL_NOTE=$OPENHANDS_VERSION
 
 # Default to NOT use unit tests.
 if [ -z "$USE_UNIT_TESTS" ]; then
@@ -39,7 +39,7 @@ if [ "$USE_UNIT_TESTS" = true ]; then
   EVAL_NOTE=$EVAL_NOTE-w-test
 fi
 
-COMMAND="export PYTHONPATH=evaluation/aider_bench:\$PYTHONPATH && poetry run python evaluation/aider_bench/run_infer.py \
+COMMAND="export PYTHONPATH=evaluation/benchmarks/aider_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/aider_bench/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 30 \

evaluation/benchmarks/biocoder/README.md

@@ -4,13 +4,14 @@ Implements evaluation of agents on BioCoder from the BioCoder benchmark introduc
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## BioCoder Docker Image
 
 In the openhands branch of the Biocoder repository, we have slightly modified our original Docker image to work with the OpenHands environment. In the Docker image are testing scripts (`/testing/start_test_openhands.py` and aux files in `/testing_files/`) to assist with evaluation. Additionally, we have installed all dependencies, including OpenJDK, mamba (with Python 3.6), and many system libraries. Notably, we have **not** packaged all repositories into the image, so they are downloaded at runtime.
 
 **Before first execution, pull our Docker image with the following command**
+
 ```bash
 docker pull public.ecr.aws/i5g0m1f6/eval_biocoder:v1.0
 ```
@@ -19,9 +20,8 @@ To reproduce this image, please see the Dockerfile_Openopenhands in the `biocode
 
 ## Start the evaluation
 
-
 ```bash
-./evaluation/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/benchmarks/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
 ```
 
 where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@@ -43,11 +43,12 @@ with current OpenHands version, then your command would be:
 ## Examples
 
 ```bash
-./evaluation/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
+./evaluation/benchmarks/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
 ```
 
 ## Reference
-```
+
+```bibtex
 @misc{tang2024biocoder,
       title={BioCoder: A Benchmark for Bioinformatics Code Generation with Large Language Models},
       author={Xiangru Tang and Bill Qian and Rick Gao and Jiakang Chen and Xinyun Chen and Mark Gerstein},

evaluation/benchmarks/biocoder/run_infer.py

@@ -8,11 +8,12 @@ from typing import Any
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.biocoder.utils import BiocoderData
+from evaluation.benchmarks.biocoder.utils import BiocoderData
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -39,7 +40,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 FILE_EXT_MAP = {
@@ -299,7 +300,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     test_result['generated'] = test_result['metadata']['1_copy_change_code']
 

evaluation/benchmarks/biocoder/scripts/run_infer.sh

@@ -21,19 +21,19 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 echo "DATASET: $DATASET"
 
-COMMAND="poetry run python evaluation/biocoder/run_infer.py \
+COMMAND="poetry run python evaluation/benchmarks/biocoder/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 10 \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note ${AGENT_VERSION}_${DATASET}"
+  --eval-note ${OPENHANDS_VERSION}_${DATASET}"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/benchmarks/bird/run_infer.py

@@ -16,6 +16,7 @@ from tqdm import tqdm
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -39,21 +40,21 @@ from openhands.utils.async_utils import call_async_from_sync
 def codeact_user_response(state: State) -> str:
     msg = (
         'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have completed the SQL, please run the following command: <execute_bash> exit </execute_bash>.\n'
+        'If you think you have completed the SQL, please finish the interaction using the "finish" tool.\n'
         'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
     )
     if state.history:
         # check if the agent has tried to talk to the user 3 times, if so, let the agent know it can give up
         user_msgs = [
             event
-            for event in state.history.get_events()
+            for event in state.history
             if isinstance(event, MessageAction) and event.source == 'user'
         ]
         if len(user_msgs) > 2:
             # let the agent know that it can give up when it has tried 3 times
             return (
                 msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
+                + 'If you want to give up, use the "finish" tool to finish the interaction.\n'
             )
     return msg
 
@@ -63,7 +64,7 @@ AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
 }
 
 
@@ -431,7 +432,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/bird/scripts/run_infer.sh

@@ -20,18 +20,18 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-COMMAND="poetry run python evaluation/bird/run_infer.py \
+COMMAND="poetry run python evaluation/benchmarks/bird/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 5 \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note $AGENT_VERSION" \
+  --eval-note $OPENHANDS_VERSION" \
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/benchmarks/browsing_delegation/README.md

@@ -7,12 +7,12 @@ If so, the browsing performance upper-bound of CodeActAgent will be the performa
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## Run Inference
 
 ```bash
-./evaluation/browsing_delegation/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
+./evaluation/benchmarks/browsing_delegation/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
 # e.g., ./evaluation/swe_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview_llm HEAD CodeActAgent 300
 ```
 

evaluation/benchmarks/browsing_delegation/run_infer.py

@@ -9,6 +9,7 @@ from datasets import load_dataset
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -89,7 +90,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # find the last delegate action
     last_delegate_action = None

evaluation/benchmarks/browsing_delegation/scripts/run_infer.sh

@@ -20,15 +20,15 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 
-EVAL_NOTE="$AGENT_VERSION"
+EVAL_NOTE="$OPENHANDS_VERSION"
 
-COMMAND="poetry run python evaluation/browsing_delegation/run_infer.py \
+COMMAND="poetry run python evaluation/benchmarks/browsing_delegation/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 1 \

evaluation/benchmarks/commit0_bench/README.md

@@ -0,0 +1,81 @@
+# Commit0 Evaluation with OpenHands
+
+This folder contains the evaluation harness that we built on top of the original [Commit0](https://commit-0.github.io/) ([paper](TBD)).
+
+The evaluation consists of three steps:
+
+1. Environment setup: [install python environment](../../README.md#development-environment), [configure LLM config](../../README.md#configure-openhands-and-your-llm).
+2. [Run Evaluation](#run-inference-on-commit0-instances): Generate a edit patch for each Commit0 Repo, and get the evaluation results
+
+## Setup Environment and LLM Configuration
+
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
+
+## OpenHands Commit0 Instance-level Docker Support
+
+OpenHands supports using the Commit0 Docker for **[inference](#run-inference-on-commit0-instances).
+This is now the default behavior.
+
+## Run Inference on Commit0 Instances
+
+Make sure your Docker daemon is running, and you have ample disk space (at least 200-500GB, depends on the Commit0 set you are running on) for the [instance-level docker image](#openhands-commit0-instance-level-docker-support).
+
+When the `run_infer.sh` script is started, it will automatically pull the `lite` split in Commit0. For example, for instance ID `commit-0/minitorch`, it will try to pull our pre-build docker image `wentingzhao/minitorch` from DockerHub. This image will be used create an OpenHands runtime image where the agent will operate on.
+
+```bash
+./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
+
+# Example
+./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 16 100 8 wentingzhao/commit0_combined test
+```
+
+where `model_config` is mandatory, and the rest are optional.
+
+- `repo_split`, e.g. `lite`, is the split of the Commit0 dataset you would like to evaluate on. Available options are `lite`, `all` and each individual repo.
+- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
+LLM settings, as defined in your `config.toml`.
+- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would
+like to evaluate. It could also be a release tag like `0.6.2`.
+- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
+to `CodeActAgent`.
+- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
+default, the script evaluates the `lite` split of the Commit0 dataset (16 repos). Note:
+in order to use `eval_limit`, you must also set `agent`.
+- `max_iter`, e.g. `20`, is the maximum number of iterations for the agent to run. By
+default, it is set to 30.
+- `num_workers`, e.g. `3`, is the number of parallel workers to run the evaluation. By
+default, it is set to 1.
+- `dataset`, a huggingface dataset name. e.g. `wentingzhao/commit0_combined`, specifies which dataset to evaluate on.
+- `dataset_split`, split for the huggingface dataset. Notice only `test` is supported for Commit0.
+
+Note that the `USE_INSTANCE_IMAGE` environment variable is always set to `true` for Commit0.
+
+Let's say you'd like to run 10 instances using `llm.eval_sonnet` and CodeActAgent,
+
+then your command would be:
+
+```bash
+./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
+```
+
+### Run Inference on `RemoteRuntime` (experimental)
+
+This is in limited beta. Contact Xingyao over slack if you want to try this out!
+
+```bash
+./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
+
+# Example - This runs evaluation on CodeActAgent for 10 instances on "wentingzhao/commit0_combined"'s test set, with max 30 iteration per instances, with 1 number of workers running in parallel
+ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev" EVAL_DOCKER_IMAGE_PREFIX="docker.io/wentingzhao" \
+./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
+```
+
+To clean-up all existing runtime you've already started, run:
+
+```bash
+ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/benchmarks/commit0_bench/scripts/cleanup_remote_runtime.sh
+```
+
+### Specify a subset of tasks to run infer
+
+If you would like to specify a list of tasks you'd like to benchmark on, you just need to pass selected repo through `repo_split` option.

evaluation/benchmarks/commit0_bench/run_infer.py

@@ -0,0 +1,606 @@
+import asyncio
+import json
+import os
+from collections import Counter
+from typing import Any
+
+import pandas as pd
+from commit0.harness.constants import SPLIT
+from datasets import load_dataset
+
+import openhands.agenthub
+from evaluation.utils.shared import (
+    EvalException,
+    EvalMetadata,
+    EvalOutput,
+    assert_and_raise,
+    codeact_user_response,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+    update_llm_config_for_completions_logging,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    get_parser,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation, ErrorObservation
+from openhands.events.serialization.event import event_to_dict
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+from openhands.utils.shutdown_listener import sleep_if_should_continue
+
+USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false').lower() == 'true'
+USE_INSTANCE_IMAGE = os.environ.get('USE_INSTANCE_IMAGE', 'false').lower() == 'true'
+RUN_WITH_BROWSING = os.environ.get('RUN_WITH_BROWSING', 'false').lower() == 'true'
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+    'CodeActCommit0Agent': codeact_user_response,
+}
+
+
+def _get_commit0_workspace_dir_name(instance: pd.Series) -> str:
+    return instance['repo'].split('/')[1]
+
+
+def get_instruction(instance: pd.Series, metadata: EvalMetadata):
+    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
+    # Prepare instruction
+    test_cmd = instance['test']['test_cmd']
+    test_dir = instance['test']['test_dir']
+    # Instruction based on Anthropic's official trajectory
+    # https://github.com/eschluntz/swe-bench-experiments/tree/main/evaluation/verified/20241022_tools_claude-3-5-sonnet-updated/trajs
+    instruction = (
+        '<uploaded_files>\n'
+        f'/workspace/{workspace_dir_name}\n'
+        '</uploaded_files>\n'
+        f"I've uploaded a python code repository in the directory {workspace_dir_name}. Here is your task:\n\n"
+        'Here is your task:\n\n'
+        '  You need to complete the implementations for all functions (i.e., those with pass\n'
+        '  statements) and pass the unit tests.\n\n'
+        '  Do not change the names of existing functions or classes, as they may be referenced\n'
+        '  from other code like unit tests, etc.\n\n'
+        '  When you generate code, you must maintain the original formatting of the function\n'
+        '  stubs (such as whitespaces), otherwise we will not able to search/replace blocks\n'
+        '  for code modifications, and therefore you will receive a score of 0 for your generated\n'
+        '  code.'
+        '\n\n'
+        'Here is the command to run the unit tests:\n'
+        '<test_command>\n'
+        f'{test_cmd} {test_dir}\n'
+        '</test_command>\n\n'
+        'Make a local git commit for each agent step for all code changes. If there is not change in current step, do not make a commit.'
+    )
+
+    if RUN_WITH_BROWSING:
+        instruction += (
+            '<IMPORTANT!>\n'
+            'You SHOULD NEVER attempt to browse the web. '
+            '</IMPORTANT!>\n'
+        )
+    return instruction
+
+
+# TODO: migrate all swe-bench docker to ghcr.io/openhands
+DOCKER_IMAGE_PREFIX = os.environ.get(
+    'EVAL_DOCKER_IMAGE_PREFIX', 'docker.io/wentingzhao/'
+)
+logger.info(f'Using docker image prefix: {DOCKER_IMAGE_PREFIX}')
+
+
+def get_instance_docker_image(repo_name: str) -> str:
+    return (DOCKER_IMAGE_PREFIX.rstrip('/') + '/' + repo_name).lower() + ':v0'
+
+
+def get_config(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+) -> AppConfig:
+    # COMMIT0_CONTAINER_IMAGE = 'wentingzhao/'
+    assert USE_INSTANCE_IMAGE
+    # We use a different instance image for the each instance of commit0 eval
+    repo_name = instance['repo'].split('/')[1]
+    base_container_image = get_instance_docker_image(repo_name)
+    logger.info(
+        f'Using instance container image: {base_container_image}. '
+        f'Please make sure this image exists. '
+        f'Submit an issue on https://github.com/All-Hands-AI/OpenHands if you run into any issues.'
+    )
+    # else:
+    #     raise
+    # base_container_image = SWE_BENCH_CONTAINER_IMAGE
+    # logger.info(f'Using swe-bench container image: {base_container_image}')
+
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        max_iterations=metadata.max_iterations,
+        runtime=os.environ.get('RUNTIME', 'eventstream'),
+        sandbox=SandboxConfig(
+            base_container_image=base_container_image,
+            enable_auto_lint=True,
+            use_host_network=False,
+            # large enough timeout, since some testcases take very long to run
+            timeout=300,
+            api_key=os.environ.get('ALLHANDS_API_KEY', None),
+            remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
+            keep_runtime_alive=False,
+            remote_runtime_init_timeout=3600,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(
+        update_llm_config_for_completions_logging(
+            metadata.llm_config, metadata.eval_output_dir, instance['instance_id']
+        )
+    )
+    agent_config = AgentConfig(
+        codeact_enable_jupyter=False,
+        codeact_enable_browsing=RUN_WITH_BROWSING,
+        codeact_enable_llm_editor=False,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required
+):
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Initialization Fn')
+    logger.info('-' * 30)
+    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
+    obs: CmdOutputObservation
+
+    action = CmdRunAction(
+        command=f'git clone -b commit0_combined https://github.com/{instance["repo"]}.git'
+    )
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to git clone -b commit0_combined https://github.com/{instance["repo"]}.git: {str(obs)}',
+    )
+
+    action = CmdRunAction(command=f'cd /workspace/{workspace_dir_name}')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to cd to /workspace/{workspace_dir_name}: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git checkout -b openhands')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0, f'Failed to git checkout new branch openhands: {str(obs)}'
+    )
+
+    # Install commit0
+    action = CmdRunAction(command='/root/.cargo/bin/uv pip install commit0')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        obs.exit_code == 0,
+        f'Failed to install commit0: {str(obs)}',
+    )
+    logger.info('-' * 30)
+    logger.info('END Runtime Initialization Fn')
+    logger.info('-' * 30)
+
+
+def complete_runtime(
+    runtime: Runtime,
+    instance: pd.Series,  # this argument is not required, but it is used to get the workspace_dir_name
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info('-' * 30)
+    logger.info('BEGIN Runtime Completion Fn')
+    logger.info('-' * 30)
+    obs: CmdOutputObservation
+    workspace_dir_name = _get_commit0_workspace_dir_name(instance)
+
+    action = CmdRunAction(command='git add .')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to git add -A: {str(obs)}',
+    )
+
+    action = CmdRunAction(command='git commit -m "openhands edits"')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation)
+        and (obs.exit_code == 0 or obs.exit_code == 1),
+        f'Failed to git commit -m "openhands": {str(obs)}',
+    )
+
+    # Generate diff patch compared to base commit, excluding spec.pdf.bz2 files
+    n_retries = 0
+    git_patch = None
+    while n_retries < 5:
+        action = CmdRunAction(
+            command=f"git diff {instance['base_commit']} HEAD -- . ':(exclude)spec.pdf.bz2'"
+        )
+        action.timeout = 600 + 100 * n_retries
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+        n_retries += 1
+        if isinstance(obs, CmdOutputObservation):
+            if obs.exit_code == 0:
+                git_patch = obs.content.strip()
+                break
+            else:
+                logger.info('Failed to get git diff, retrying...')
+                sleep_if_should_continue(10)
+        elif isinstance(obs, ErrorObservation):
+            logger.error(f'Error occurred: {obs.content}. Retrying...')
+            sleep_if_should_continue(10)
+        else:
+            assert_and_raise(False, f'Unexpected observation type: {str(obs)}')
+
+    assert_and_raise(git_patch is not None, 'Failed to get git diff (None)')
+
+    test_dir = instance['test']['test_dir']
+    action = CmdRunAction(
+        command=f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors {test_dir} > test_output.txt 2>&1"
+    )
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation),
+        f'Failed to run test command: {str(obs)}',
+    )
+    # Read test output
+    action = CmdRunAction(command='cat test_output.txt')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation),
+        f'Failed to read test output: {str(obs)}',
+    )
+    test_output = obs.content.strip()
+    # logger.info(f'Test output: {test_output}')
+
+    # Save pytest exit code
+    action = CmdRunAction(command='echo $?')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation) and obs.exit_code == 0,
+        f'Failed to save pytest exit code: {str(obs)}',
+    )
+    pytest_exit_code = obs.content.strip()
+    # logger.info(f'Pytest exit code: {pytest_exit_code}')
+
+    # Read the test report
+    action = CmdRunAction(command='cat report.json')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    assert_and_raise(
+        isinstance(obs, CmdOutputObservation),
+        f'Failed to read test report: {str(obs)}',
+    )
+    # Get test IDs from instance
+    repo_name = instance['repo'].split('/')[1]
+    repo_name = repo_name.replace('.', '-')
+    action = CmdRunAction(command=f'commit0 get-tests {repo_name}')
+    action.timeout = 600
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    # logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    test_ids = obs.content.strip().split('\n')
+
+    try:
+        report = json.loads(obs.content)
+        tests = {x['nodeid']: x['call'] for x in report['tests'] if 'call' in x}
+
+        # Calculate test statistics
+        status = []
+        runtimes = []
+        no_runs = 0
+
+        for test_id in test_ids:
+            if test_id in tests and tests[test_id] is not None:
+                status.append(tests[test_id]['outcome'])
+                runtimes.append(tests[test_id]['duration'])
+                no_runs += 1
+            else:
+                status.append('failed')
+                runtimes.append(0)
+
+        status_counts = Counter(status)
+        total_runtime = sum(runtimes) if no_runs > 0 else 0
+        num_passed = status_counts.get('passed', 0) + status_counts.get('xfail', 0)
+        passed_ratio = num_passed / len(status) if status else 0
+
+        eval_result = {
+            'name': workspace_dir_name,
+            'sum': total_runtime,
+            'passed': passed_ratio,
+            'num_passed': num_passed,
+            'num_tests': len(test_ids),
+        }
+
+    except json.JSONDecodeError:
+        logger.error('Failed to parse test report JSON')
+        eval_result = {
+            'name': workspace_dir_name,
+            'sum': 0,
+            'passed': 0,
+            'num_passed': 0,
+            'num_tests': len(test_ids),
+        }
+
+    # Create tarball of workspace
+    temp_zip = runtime.copy_from(f'/workspace/{workspace_dir_name}')
+
+    commit0_dir = os.path.dirname(__file__)
+    persistent_zip = os.path.join(commit0_dir, f'{workspace_dir_name}.zip')
+    with open(temp_zip, 'rb') as src, open(persistent_zip, 'wb') as dst:
+        dst.write(src.read())
+    zip_file = persistent_zip
+    return {
+        'eval_result': eval_result,
+        'git_patch': git_patch,
+        'test_output': test_output,
+        'pytest_exit_code': pytest_exit_code,
+        'zip_file': zip_file,
+    }
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+) -> EvalOutput:
+    config = get_config(instance, metadata)
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+    try:
+        initialize_runtime(runtime, instance)
+
+        instruction = get_instruction(instance, metadata)
+
+        # Here's how you can run the agent (similar to the `main` function) and get the final task state
+        state: State | None = asyncio.run(
+            run_controller(
+                config=config,
+                initial_user_action=MessageAction(content=instruction),
+                runtime=runtime,
+                fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN[
+                    metadata.agent_class
+                ],
+            )
+        )
+
+        # if fatal error, throw EvalError to trigger re-run
+        if (
+            state.last_error
+            and 'fatal error during agent execution' in state.last_error
+            and 'stuck in a loop' not in state.last_error
+        ):
+            raise EvalException('Fatal error detected: ' + state.last_error)
+
+        # ======= THIS IS Commit0 specific =======
+        # Get git patch
+        return_val = complete_runtime(runtime, instance)
+        eval_result = return_val['eval_result']
+        git_patch = return_val['git_patch']
+        test_output = return_val['test_output']
+        pytest_exit_code = return_val['pytest_exit_code']
+        zip_file = return_val['zip_file']
+
+        repo_name = instance['repo'].split('/')[1]
+        zip_dest = os.path.join(
+            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}.zip'
+        )
+        patch_file = os.path.join(
+            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_patch.diff'
+        )
+        test_output_file = os.path.join(
+            metadata.eval_output_dir, 'repos', repo_name, f'{repo_name}_test_output.txt'
+        )
+        pytest_exit_code_file = os.path.join(
+            metadata.eval_output_dir,
+            'repos',
+            repo_name,
+            f'{repo_name}_pytest_exit_code.txt',
+        )
+
+        os.makedirs(os.path.dirname(zip_dest), exist_ok=True)
+        os.rename(zip_file, zip_dest)
+
+        write_targets = [
+            (patch_file, git_patch),
+            (test_output_file, test_output),
+            (pytest_exit_code_file, pytest_exit_code),
+        ]
+
+        for write_target in write_targets:
+            with open(write_target[0], 'w') as f:
+                f.write(write_target[1])
+
+        logger.info(
+            f'Got evaluation result for repo {instance.instance_id}:\n--------\n{eval_result}\n--------'
+        )
+    finally:
+        runtime.close()
+    # ==========================================
+
+    # ======= Attempt to evaluate the agent's edits =======
+    # we use eval_infer.sh to evaluate the agent's edits, not here
+    # because the agent may alter the environment / testcases
+    test_result = {
+        'eval_result': eval_result,
+    }
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    # NOTE: this is NO LONGER the event stream, but an agent history that includes delegate agent's events
+    histories = [event_to_dict(event) for event in state.history]
+    metrics = state.metrics.get() if state.metrics else None
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=instance.instance_id,
+        instruction=instruction,
+        instance=instance.to_dict(),
+        test_result=test_result,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+    )
+    return output
+
+
+def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
+    """Setup Commit0 dataset based on split type.
+
+    Args:
+        dataset: Full Commit0 dataset
+        repo_split: Split type ('all', 'lite' or specific repo name)
+
+    Returns:
+        Filtered dataset based on split type
+    """
+
+    filtered_dataset = pd.concat(
+        [
+            dataset[dataset['repo'].str.split('/').str[1] == repo]
+            for repo in SPLIT.get(repo_split, [])
+        ]
+    )
+
+    # Drop setup column if it exists
+    if 'setup' in filtered_dataset.columns:
+        filtered_dataset = filtered_dataset.drop('setup', axis=1)
+
+    # Replace all forward slashes in instance_id with hyphens
+    filtered_dataset['instance_id'] = filtered_dataset['repo'].str.split('/').str[1]
+
+    return filtered_dataset
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        default='wentingzhao/commit0_combined',
+        help='dataset to evaluate on, only test split exists for this HF dataset',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='test',
+        help='this is the HF dataset split',
+    )
+    parser.add_argument(
+        '--repo-split',
+        type=str,
+        default='lite',
+        help='all, lite, or each repo name',
+    )
+    args, _ = parser.parse_known_args()
+
+    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
+    # so we don't need to manage file uploading to OpenHands's repo
+    dataset = load_dataset(args.dataset, split=args.split)
+
+    commit0_datasets = commit0_setup(dataset.to_pandas(), args.repo_split)
+
+    logger.info(f'Loaded dataset {args.dataset} with reposplit {args.repo_split}')
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+        llm_config.log_completions = True
+
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    details = {}
+    _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls)
+
+    dataset_descrption = (
+        args.dataset.replace('/', '__') + '-' + args.repo_split.replace('/', '__')
+    )
+    metadata = make_metadata(
+        llm_config,
+        dataset_descrption,
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+        details=details,
+    )
+
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+
+    instances = prepare_dataset(commit0_datasets, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+        timeout_seconds=120 * 60,  # 2 hour PER instance should be more than enough
+    )

evaluation/benchmarks/commit0_bench/scripts/run_infer.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+REPO_SPLIT=$1
+MODEL_CONFIG=$2
+COMMIT_HASH=$3
+AGENT=$4
+EVAL_LIMIT=$5
+MAX_ITER=$6
+NUM_WORKERS=$7
+DATASET=$8
+SPLIT=$9
+N_RUNS=${10}
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+if [ -z "$MAX_ITER" ]; then
+  echo "MAX_ITER not specified, use default 100"
+  MAX_ITER=100
+fi
+
+if [ -z "$USE_INSTANCE_IMAGE" ]; then
+  echo "USE_INSTANCE_IMAGE not specified, use default true"
+  USE_INSTANCE_IMAGE=true
+fi
+
+if [ -z "$RUN_WITH_BROWSING" ]; then
+  echo "RUN_WITH_BROWSING not specified, use default false"
+  RUN_WITH_BROWSING=false
+fi
+
+
+if [ -z "$DATASET" ]; then
+  echo "DATASET not specified, use default wentingzhao/commit0_combined"
+  DATASET="wentingzhao/commit0_combined"
+fi
+
+if [ -z "$REPO_SPLIT" ]; then
+  echo "REPO_SPLIT not specified, use default lite"
+  REPO_SPLIT=0
+fi
+
+if [ -z "$SPLIT" ]; then
+  echo "HF SPLIT not specified, use default test"
+  SPLIT="test"
+fi
+
+export USE_INSTANCE_IMAGE=$USE_INSTANCE_IMAGE
+echo "USE_INSTANCE_IMAGE: $USE_INSTANCE_IMAGE"
+export RUN_WITH_BROWSING=$RUN_WITH_BROWSING
+echo "RUN_WITH_BROWSING: $RUN_WITH_BROWSING"
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+echo "DATASET: $DATASET"
+echo "HF SPLIT: $SPLIT"
+echo "REPO SPLIT: $REPO_SPLIT"
+
+# Default to NOT use Hint
+if [ -z "$USE_HINT_TEXT" ]; then
+  export USE_HINT_TEXT=false
+fi
+echo "USE_HINT_TEXT: $USE_HINT_TEXT"
+EVAL_NOTE="$OPENHANDS_VERSION"
+# if not using Hint, add -no-hint to the eval note
+if [ "$USE_HINT_TEXT" = false ]; then
+  EVAL_NOTE="$EVAL_NOTE-no-hint"
+fi
+
+if [ "$RUN_WITH_BROWSING" = true ]; then
+  EVAL_NOTE="$EVAL_NOTE-with-browsing"
+fi
+
+if [ -n "$EXP_NAME" ]; then
+  EVAL_NOTE="$EVAL_NOTE-$EXP_NAME"
+fi
+
+function run_eval() {
+  local eval_note=$1
+  COMMAND="poetry run python evaluation/benchmarks/commit0_bench/run_infer.py \
+    --agent-cls $AGENT \
+    --llm-config $MODEL_CONFIG \
+    --max-iterations $MAX_ITER \
+    --eval-num-workers $NUM_WORKERS \
+    --eval-note $eval_note \
+    --dataset $DATASET \
+    --split $SPLIT \
+    --repo-split $REPO_SPLIT"
+
+  if [ -n "$EVAL_LIMIT" ]; then
+    echo "EVAL_LIMIT: $EVAL_LIMIT"
+    COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+  fi
+
+  # Run the command
+  eval $COMMAND
+}
+
+unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
+if [ -z "$N_RUNS" ]; then
+  N_RUNS=1
+  echo "N_RUNS not specified, use default $N_RUNS"
+fi
+
+for i in $(seq 1 $N_RUNS); do
+  current_eval_note="$EVAL_NOTE-run_$i"
+  echo "EVAL_NOTE: $current_eval_note"
+  run_eval $current_eval_note
+done
+
+checkout_original_branch

evaluation/benchmarks/discoverybench/README.md

@@ -0,0 +1,37 @@
+# DiscoveryBench with OpenHands
+
+[DiscoveryBench](https://github.com/allenai/discoverybench/) [(Paper)](https://arxiv.org/abs/2407.01725v1) contains 264 tasks collected across 6 diverse domains, such as biology, economics, and sociology. It incorporates discovery workflows from published papers to approximate the real-world challenges faced by researchers.
+
+<p align="center">
+  <a href="[https://github.com/allenai/discoverybench](https://github.com/allenai/discoverybench)">
+    <img src="https://raw.githubusercontent.com/allenai/discoverybench/refs/heads/main/assets/discoverybench-openhands-teaser.png" width="100%" alt="DiscoveryBench Background" />
+  </a>
+</p>
+
+
+## Setup Environment and LLM Configuration
+
+1. Please follow instructions mentioned [here](https://github.com/openlocus/OpenHands/blob/discoverybench-openhands-integration/evaluation/README.md#setup) to setup OpenHands development environment and LLMs locally
+
+2. Execute the bash script to start DiscoveryBench Evaluation
+
+```
+./evaluation/benchmarks/discoverybench/scripts/run_infer.sh [YOUR MODEL CONFIG]
+```
+Replace `[YOUR MODEL CONFIG]` with any model the model that you have set up in `config.toml`
+
+
+## Run Inference on DiscoveryBench Instances
+
+When the `run_infer.sh` script is started, it will automatically pull the latest DiscoveryBench instances & set up the agent environment. The OpenHands agent is invoked to process the task within this environment, producing a hypothesis. We then evaluate it against the “gold” hypothesis provided by DiscoveryBench. The evaluation result, along with the agent chat history is logged to `output.jsonl` under `evaluation_outputs`.
+
+
+```
+./evaluation/benchmarks/discoverybench/scripts/run_infer.sh [MODEL_CONFIG] [GIT_COMMIT] [AGENT] [EVAL_LIMIT] [NUM_WORKERS]
+```
+
+- `MODEL_CONFIG`: Name of the model you want to evaluate with
+- `GIT_COMMIT`: This should be the git commit hash or release tag for OpenHands, e.g., HEAD or a specific tag like 0.6.2.
+- `AGENT`: Use CoderActAgent, right now it only supports that.
+- `EVAL_LIMIT`: Number of samples to evaluate.
+- `NUM_WORKERS`: Number of workers to parallelize the evaluation process.

evaluation/benchmarks/discoverybench/eval_utils/README.md

@@ -0,0 +1,7 @@
+## DiscoveryBench Evaluation Utils
+
+- **`eval_w_subhypo_gen.py`**: Implements the DiscoveryBench logic for evaluating agent-generated hypotheses.
+- **`lm_utils.py`**: Provides utility functions necessary for the evaluation process.
+- **`openai_helpers.py`**: Includes helper functions for OpenAI-related tasks.
+- **`openai_semantic_gen_prompts.py`**: Contains prompts used for semantic generation.
+- **`response_parser.py`**: Handles the parsing of agent-generated hypotheses.

evaluation/benchmarks/discoverybench/eval_utils/eval_w_subhypo_gen.py

@@ -0,0 +1,538 @@
+import json
+import logging
+
+from openai import OpenAI
+
+from .lm_utils import run_chatgpt_query_multi_turn
+from .openai_helpers import get_response
+
+logging.basicConfig(
+    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+    datefmt='%m/%d/%Y %H:%M:%S',
+    level=logging.INFO,
+)
+logger = logging.getLogger(__name__)
+
+
+def get_score_from_answer(type, answer):
+    if type == 'context':
+        answer = answer.replace('Answer:', '').strip()
+        if answer.startswith('A)'):
+            return 1.0
+        elif answer.startswith('B)'):
+            return 0.0
+        return -1.0
+
+    elif type == 'var':
+        try:
+            var_json = json.loads(answer)
+            # print(f"var_json:{var_json}")
+            p = 0.0
+            r = 0.0
+            f1 = 0.0
+            if var_json['sizeB']:
+                p = var_json['intersection'] / var_json['sizeB']
+            if var_json['sizeA']:
+                r = var_json['intersection'] / var_json['sizeA']
+            if p > 0.0 and r > 0.0:
+                f1 = (2 * p * r) / (p + r)
+            else:
+                f1 = 0.0
+            eval_rec = {
+                'p': p,
+                'r': r,
+                'f1': f1,
+                'sizeA': var_json['sizeA'],
+                'sizeB': var_json['sizeB'],
+                'intersection': var_json['intersection'],
+                'explanation': var_json['explanation'],
+            }
+            print(f'var_eval: {eval_rec}')
+            return eval_rec
+        except Exception:  # COMMENT: added Exception
+            return {'p': -1.0, 'r': -1.0, 'f1': -1.0}
+    elif type == 'rel':
+        print(answer)
+        rel_json = json.loads(answer)
+        answer_str = rel_json['answer'].strip()
+        if answer_str.startswith('A') or 'very similar' in answer_str:
+            return 1.0
+        elif (
+            answer_str.startswith('B') or 'similar but general than HypoA' in answer_str
+        ):
+            return 0.5
+        elif answer_str.startswith('C') or 'different' in answer_str:
+            return 0.0
+        return -1.0
+    return -1.0
+
+
+def ask_dimension_question(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    dimension,
+    dataset_type,
+    use_column_metadata=True,
+):
+    dimension_question = ''
+    answer = ''
+    score = 0.0
+    if dimension == 'var':
+        score = {'p': -1.0, 'r': -1.0, 'f1': -1.0}
+    num_tokens = 256
+    num_retries = 1
+    json_response = False
+
+    messages = [
+        {
+            'role': 'system',
+            'content': 'You are an AI assistant that helps evaluate a data-driven hypothesis. You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+    ]
+    if dimension == 'context':
+        dimension_question = """\
+        Question: Is HypoB defined in the same context as HypoA?
+        (Context refers to assumptions/stratification under which the hypotheses are defined.)
+        Options: A) same   B) different
+        What is your answer?"""
+    elif dimension == 'var':
+        dimension_question = """\
+        Question: For both HypoA and HypoB, what are the different variables found in the hypotheses? \
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+        "sizeA": num of variables used in HypoA
+        "sizeB": num of variables used in HypoB
+        "intersection": num of variables common in HypoA and HypoB. Use *fuzzy matching* to determine intersection, accounting for paraphrases or slightly different surface forms
+        "explanation": a short text explanation about the variables
+        }}```
+        Answer:"""
+        num_tokens = 512
+        num_retries = 1
+        json_response = True
+    elif dimension == 'rel':
+        dimension_question = """\
+        Question: Does HypoB exhibit the same relation as HypoA?
+        Compare using following example hierarchy of relationships (based on specificity): \
+        "there exists a relationship" > "positive relationship" > "positive AND (linear OR quadratic)" > "positive AND linear".
+        Options: A) very similar B) similar but general than HypoA C) different
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+        "answer": one of the options from A) very similar B) similar but general than HypoA C) different
+        "explanation": a short text explanation about the relationship comparison
+        }}```
+        Answer:"""
+        num_tokens = 512
+        num_retries = 1
+        json_response = True
+
+    datasets_json = prepare_dataset_metadata_json(
+        dataset_meta, dataset_type=dataset_type, use_column_metadata=use_column_metadata
+    )
+
+    dimension_question_str = f"""\
+        You are going to compare two natural-language hypotheses HypoA and HypoB accompanied with optional workflows: WorkflowA for HypoA and WorkflowB for HypoB. \
+        Both the hypotheses answer the natural language query "QUERY" over the dataset(s) described by dataset description(s) and column description(s) below. \
+        Compare HypoA and HypoB in terms of three aspects: Contexts, Variables, and Relations. \
+        E.g., for the hypothesis "From 1995 to 2009, the number of sandhill cranes around the tundra (Indigilka River) surged by an astounding ~10X":
+        * Contexts refer to stratification of the data under which the given hypothesis is True. E.g., "For all women", "From 1995 to 2009".
+        * Variables refer to the set of variables (either dependent or independent) that are mentioned in the hypothesis. E.g., number of sandhill cranes, location.
+        * Relations refer to the form of relation between the variables. E.g., "surged by ~10x".
+
+        Answer following questions for a given pair of hypotheses, HypoA and HypoB, along with an explanation grounded on the QUERY and the DATASET(S).
+
+        Here is the metadata for the task:
+        ```json
+        {{
+        "datasets": {datasets_json},
+        "query": {query},
+        "HypoA": {gold_hypo},
+        "WorkflowA": {gold_workflow},
+        "HypoB": {gen_hypo},
+        "WorkflowB": {gen_workflow}
+        }}
+        ```
+
+        {dimension_question}"""
+
+    messages.append({'role': 'user', 'content': dimension_question_str})
+    for retry in range(num_retries):
+        response = run_chatgpt_query_multi_turn(
+            messages=messages,
+            model_name=llm_used,
+            max_tokens=num_tokens,
+            temperature=0,  # 0 for greedy best decoding
+            json_response=json_response,
+        )
+        if response is not None:  # COMMENT: changed from != to is not
+            break
+
+    if response is not None:  # COMMENT: changed from != to is not
+        answer = response.choices[0].message.content.strip()
+        score = get_score_from_answer(type=dimension, answer=answer)
+
+    return dimension_question, answer, score
+
+
+def prepare_dataset_metadata_json(dataset_meta, dataset_type, use_column_metadata=True):
+    if dataset_meta is None:  # COMMENT: changed from == to is None
+        return [
+            {
+                'dataset_description': '',
+                'columns': [],
+            }
+        ]
+    datasets_json = []
+    if dataset_type == 'real':
+        for d in dataset_meta['datasets']:
+            datasets_json.append(
+                {
+                    'dataset_description': d['description'],
+                    'columns': [
+                        {'name': col['name'], 'description': col['description']}
+                        for col in d['columns']['raw']
+                    ]
+                    if use_column_metadata
+                    else [],
+                }
+            )
+    else:
+        for d in dataset_meta['datasets']:
+            datasets_json.append(
+                {
+                    'dataset_description': d['description'],
+                    'columns': [
+                        {'name': col['name'], 'description': col['description']}
+                        for col in d['columns']
+                    ]
+                    if use_column_metadata
+                    else [],
+                }
+            )
+    return datasets_json
+
+
+def get_sub_hypotheses(
+    query,
+    hypo,
+    workflow,
+    dataset_meta,
+    llm_used,
+    dataset_type,
+    use_column_metadata=True,
+):
+    client = OpenAI()
+    extraction_prompt = """\
+        Given a set of dataset columns, a ground-truth hypothesis, and the analysis workflow used, your task is to extract three dimensions that define the hypothesis: Context, Variables, and Relations. \
+        Here are the definitions for these dimensions:
+        - Contexts: Boundary conditions that limit the scope of a hypothesis. E.g., “for men over \
+        the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then extract the context from the dataset_descrption.
+        - Variables: Known concepts that interact in a meaningful way under a given context to \
+        produce the hypothesis. E.g., gender, age, income, or "None" if there is no interacting variable.
+        - Relations: Interactions between a given set of variables under a given context to produce \
+        the hypothesis. E.g., “quadratic relationship”, “inversely proportional”, piecewise conditionals, \
+        or "None" if there is no interacting relationship.
+        Make sure to only use the information present in the hypothesis and the workflow. Do not add any new information. \
+        For each dimension, be specific, and do not omit any important details.
+
+        Here is the metadata for the task:
+        ```json
+        {
+        "datasets": %s,
+        "hypothesis": "%s",
+        "workflow": "%s"
+        }
+        ```
+
+        Return your answer as a JSON object in the following format:
+        ```json
+        {
+        "sub_hypo": [
+            {
+                "text": the hypothesis in natural language,
+                "context": a short text description of the context of the hypothesis,
+                "variables": a list of columns involved in the hypothesis,
+                "relations": a short text description of the relationship between the variables of the hypothesis
+            },
+            ...
+        ]
+        }```
+        """
+    datasets_json = prepare_dataset_metadata_json(
+        dataset_meta, dataset_type, use_column_metadata=use_column_metadata
+    )
+    _prompt = extraction_prompt % (datasets_json, hypo, workflow)
+    sub_hypo_json = get_response(client, _prompt, model=llm_used, max_retry=1)
+
+    if sub_hypo_json is not None:  # COMMENT: changed from != to is not
+        # print(f"full hypothesis: {hypo}")
+        print(f'sub_hypo_json: {sub_hypo_json}')
+    else:
+        sub_hypo_json = {
+            'sub_hypo': [],
+        }
+
+    sub_hypo_json['full_hypo'] = hypo
+
+    return sub_hypo_json
+
+
+def match_context_with_gpt(
+    gold_hyp, gold_context, pred_hyp, pred_context, model='gpt-3.5-turbo'
+):
+    prompt = f"""\
+        Given a gold hypothesis, a gold context, a predicted hypothesis, and a predicted context, your task is \
+        to determine if the predicted context semantically matches the ground-truth context. \
+        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
+        Here is the definition for Context: Boundary conditions that limit the scope of a sub-hypothesis. E.g., “for men over the age of 30”, “in Asia and Europe”. If the context applies to the full dataset, then the context is derived from the dataset_descrption. \
+        If the predicted context matches the gold context, return true, otherwise return false.
+        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
+        If both gold and predicted hypotheses are defined over the context of the full dataset, then also return true.
+
+        Here is the metadata for the task:
+        ```json
+        {{
+            "gold_hypothesis": "{gold_hyp}",
+            "gold_context": "{gold_context}",
+            "predicted_hypothesis": "{pred_hyp}",
+            "predicted_context": "{pred_context}"
+        }}
+        ```
+
+        Return your answer as a JSON object in the following format:
+        ```json
+        {{
+            "match": true or false
+        }}
+        ```"""
+
+    client = OpenAI()
+    output = get_response(client, prompt, model=model)
+    return output.get('match', False)
+
+
+def is_matching_context(gold_hyp, gold_context, pred_hyp, pred_context, llm_used):
+    if gold_context == pred_context:
+        return True
+    if 'None' in [gold_context, pred_context]:
+        return False
+    return match_context_with_gpt(
+        gold_hyp, gold_context, pred_hyp, pred_context, model=llm_used
+    )
+
+
+def run_eval_gold_vs_gen_NL_subhypo(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    context_score,
+    dataset_type,
+    use_column_metadata=True,
+):
+    # GPT-4 based evaluation to evaluate generated hypothesis in terms of context, variables, relation
+
+    eval_rec = {
+        'query': query,
+        'HypoA': gold_hypo,
+        'WorkflowA': gold_workflow,
+        'HypoB': gen_hypo,
+        'WorkflowB': gen_workflow,
+    }
+
+    for dimension in ['var', 'rel']:
+        question, answer, score = ask_dimension_question(
+            query,
+            gold_hypo,
+            gold_workflow,
+            gen_hypo,
+            gen_workflow,
+            dataset_meta,
+            llm_used,
+            dimension=dimension,
+            dataset_type=dataset_type,
+            use_column_metadata=use_column_metadata,
+        )
+
+        eval_rec[dimension] = {'question': question, 'answer': answer, 'score': score}
+
+    eval_rec['context'] = context_score
+    eval_rec['accuracy_score'] = (
+        1.0
+        * eval_rec['context']['score']
+        * eval_rec['var']['score']['f1']
+        * eval_rec['rel']['score']
+    )
+
+    return eval_rec
+
+
+def run_eval_gold_vs_gen_NL_hypo_workflow(
+    query,
+    gold_hypo,
+    gold_workflow,
+    gen_hypo,
+    gen_workflow,
+    dataset_meta,
+    llm_used,
+    dataset_type,
+    use_column_metadata=True,
+):
+    # Input: Dataset Metadata, Query, Gold {Hg, Wg}, Predicted {Hp, Wp}
+    # Output: eval_rec json includes final_score
+
+    # Procedure:
+    # Dataset Metadata, Query, Gold {Hg, Wg}, Pred {Hg, Wg}
+    # Gold: [Hg1, Hg2] (compute on the fly) Hg1 is a NL form of subhypothesis
+    # Predicted: [Hp1, Hp2] (compute on the fly)
+
+    # Compute Intersection: [(Hg_i, Hp_j), …]  # tuples of (gold,pred) that matched with context (do this w/o explicit extraction)
+    # # filter so that a gold context and a predicted context are only attached to one tuple
+    # Compute recall_context (programmatically)
+
+    # r_v_list = []
+    # For (Hg_i, Hp_j) in the intersection:
+    #             With Hg_i, Hp_j in NL, ask GPT4 → #variables and #intersection and a paragraph explanation and programmatically calculate f1_v
+    # Hg_i, Hp_j in NL, ask GPT4 → matching score (0, 0.5 or 1) : A) very similar B) similar but general than HypoA C) different + explanation
+    # 	r_v_list ← f1_v * score_r
+    # accuracy_score = mean(r_v_list)
+    # score =   [ recall_context * mean over predicted context(context_score * var_score *rel_score )]
+
+    # recall_context = 1.0  # COMMENT: never used
+    eval_rec = {
+        'query': query,
+        'HypoA': gold_hypo,
+        'WorkflowA': gold_workflow,
+        'HypoB': gen_hypo,
+        'WorkflowB': gen_workflow,
+    }
+
+    gold_sub_hypo_json = get_sub_hypotheses(
+        query=query,
+        hypo=gold_hypo,
+        workflow=gold_workflow,
+        dataset_meta=dataset_meta,
+        llm_used=llm_used,
+        dataset_type=dataset_type,
+        use_column_metadata=use_column_metadata,
+    )
+    if len(gold_sub_hypo_json['sub_hypo']) == 0:
+        gold_sub_hypo_json['sub_hypo'] = [
+            {
+                'text': gold_hypo,
+                'context': 'None',
+                'variables': [],
+                'relations': '',
+                'explanation': 'unable to segment',
+            }
+        ]
+    print(f'gold_sub_hypo_json: {gold_sub_hypo_json}')
+
+    gen_sub_hypo_json = get_sub_hypotheses(
+        query=query,
+        hypo=gen_hypo,
+        workflow=gen_workflow,
+        dataset_meta=dataset_meta,
+        llm_used=llm_used,
+        dataset_type=dataset_type,
+        use_column_metadata=use_column_metadata,
+    )
+    if len(gen_sub_hypo_json['sub_hypo']) == 0:
+        gen_sub_hypo_json['sub_hypo'] = [
+            {
+                'text': gen_hypo,
+                'context': 'None',
+                'variables': [],
+                'relations': '',
+                'explanation': 'unable to segment',
+            }
+        ]
+    print(f'gen_sub_hypo_json: {gen_sub_hypo_json}')
+
+    eval_rec['gold_sub_hypo'] = gold_sub_hypo_json
+    eval_rec['gen_sub_hypo'] = gen_sub_hypo_json
+
+    gold_subh_covered = []
+    gen_subh_to_gold_subh = dict()
+    gen_gold_subh_to_context = dict()
+
+    for p_id, gen_subh in enumerate(gen_sub_hypo_json['sub_hypo']):
+        gen_subh_to_gold_subh[p_id] = -1
+
+        for g_id, gold_subh in enumerate(gold_sub_hypo_json['sub_hypo']):
+            if g_id in gold_subh_covered:
+                continue
+
+            # match context
+            context_bool = is_matching_context(
+                gold_subh['text'],
+                gold_subh.get('context', ''),
+                gen_subh['text'],
+                gen_subh.get('context', ''),
+                llm_used,
+            )
+            if context_bool:
+                context_score = 1.0
+            else:
+                context_score = 0.0
+
+            if context_score == 1.0:  # match only when context_score = 1.0
+                gen_subh_to_gold_subh[p_id] = g_id
+                gold_subh_covered.append(g_id)
+                gen_gold_subh_to_context[f'P{p_id}||G{g_id}'] = {
+                    'question': f"""Comapring: GoldH: {gold_subh["text"]}, GoldC: {gold_subh['context']}\nGenH: {gen_subh['text']}, GenC: {gen_subh['context']}""",
+                    'answer': context_bool,
+                    'score': context_score,
+                }
+                break
+
+    print(f'gen_subh_to_gold_subh: {gen_subh_to_gold_subh}')
+    eval_rec['gen_subh_to_gold_subh'] = gen_subh_to_gold_subh
+    eval_rec['gold_subh_covered'] = gold_subh_covered
+    matched_gold_gen_subh_evals = dict()
+    sum_accuracy_score = 0.0
+    for p_id, g_id in gen_subh_to_gold_subh.items():
+        if g_id >= 0:
+            key = f'P{p_id}||G{g_id}'
+            context_score = gen_gold_subh_to_context[key]
+            subh_eval_rec = run_eval_gold_vs_gen_NL_subhypo(
+                query,
+                gold_hypo,
+                gold_workflow,
+                gen_hypo,
+                gen_workflow,
+                dataset_meta,
+                llm_used,
+                context_score,
+                dataset_type=dataset_type,
+                use_column_metadata=use_column_metadata,
+            )
+            sum_accuracy_score += subh_eval_rec['accuracy_score']
+            matched_gold_gen_subh_evals[key] = subh_eval_rec
+
+    eval_rec['matched_gold_gen_subh_evals'] = matched_gold_gen_subh_evals
+    eval_rec['recall_context'] = (
+        len(gold_subh_covered) / len(gold_sub_hypo_json['sub_hypo'])
+        if len(gold_sub_hypo_json['sub_hypo'])
+        else 0.0
+    )
+    mean_accuracy_score = (
+        sum_accuracy_score / len(gen_subh_to_gold_subh)
+        if len(gen_subh_to_gold_subh)
+        else 0.0
+    )
+    eval_rec['mean_accuracy_score'] = mean_accuracy_score
+    final_score = eval_rec['recall_context'] * mean_accuracy_score
+    eval_rec['final_score'] = final_score
+    print(f'eval_rec: {json.dumps(eval_rec, indent=2)}')
+
+    return eval_rec

evaluation/benchmarks/discoverybench/eval_utils/lm_utils.py

@@ -0,0 +1,64 @@
+import os
+import sys
+import time
+
+from openai import OpenAI
+from tenacity import (
+    retry,
+    stop_after_attempt,  # type: ignore
+    wait_random_exponential,  # type: ignore
+)
+
+if sys.version_info >= (3, 8):
+    from typing import Literal
+else:
+    from typing_extensions import Literal
+
+
+Model = Literal['gpt-4', 'gpt-3.5-turbo', 'text-davinci-003']
+
+OpenAI.api_key = os.getenv('OPENAI_API_KEY')
+OPENAI_GEN_HYP = {
+    'temperature': 0,
+    'max_tokens': 250,
+    'top_p': 1.0,
+    'frequency_penalty': 0,
+    'presence_penalty': 0,
+}
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+def run_chatgpt_query_multi_turn(
+    messages,
+    model_name='gpt-4-turbo',  # pass "gpt4" for more recent model output
+    max_tokens=256,
+    temperature=0.0,
+    json_response=False,
+):
+    response = None
+    num_retries = 3
+    retry = 0
+    while retry < num_retries:
+        retry += 1
+        try:
+            client = OpenAI()
+
+            if json_response:
+                response = client.chat.completions.create(
+                    model=model_name,
+                    response_format={'type': 'json_object'},
+                    messages=messages,
+                    **OPENAI_GEN_HYP,
+                )
+            else:
+                response = client.chat.completions.create(
+                    model=model_name, messages=messages, **OPENAI_GEN_HYP
+                )
+            break
+
+        except Exception as e:
+            print(e)
+            print('GPT error. Retrying in 2 seconds...')
+            time.sleep(2)
+
+    return response

evaluation/benchmarks/discoverybench/eval_utils/openai_helpers.py

@@ -0,0 +1,190 @@
+import json
+
+
+def OPENAI_TOPIC_GEN_MESSAGES(n=10):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given `n`, come up with a list of `n` distinct topics and their descriptions. The topics can be absolutely anything. Be as creative as possible. Return your answer as a JSON object. \n\nFor example, for `n`=3, a valid answer might be:\n```json\n{{"topics": [\n  {{"id": 1, "topic": "cooking", "description": "Related to recipes, ingredients, chefs, etc."}},\n  {{"id": 2, "topic": "sports", "description": "Related to players, stadiums, trophies, etc."}},\n  {{"id": 3, "topic": "antiquing", "description": "Related to unique items, history, etc."}}\n]}}```\n\nNow, give me a list for `n`={n}. Remember, pick diverse topics from everything possible. No consecutive topics should be broadly similar. Directly respond with the answer JSON object.',
+        },
+    ]
+
+
+OPENAI_GEN_HYP = {
+    'temperature': 1.0,
+    'max_tokens': 4096,
+    'top_p': 1.0,
+    'frequency_penalty': 0,
+    'presence_penalty': 0,
+}
+
+
+def OPENAI_SEMANTICS_GEN_MESSAGES(dependent, relationship, domain, domain_desc):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given the true relationship in a dataset and a given domain, your task is to come up with an interpretation of some real-world concepts that the relationship could be modeling from the provided domain. It\'s okay to be wrong, but suggest something reasonable. Try as much as possible to make sure that the TARGET is actually derivable from the other variables. Give your answer as a JSON object. Here\'s an example:\n\nRelationship for x2 = "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)"\nDomain="Sales"\nDomain description="Related to product distribution, revenues, marketing, etc."\n\nBased on this, the following real-world concepts might be applicable:\n```json\n{{\n  "dependent": "x2",\n  "relationship": "(96.4 * x1 ** 3) + (88.72 * x5 ** 2) + (81.96 * x6 ** -2) + (28.13 * x3)  + (97.0) + (0 * x4)",\n  "domain": "Sales",\n  "trends": {{\n    "x1": "Positive, cubic factor",\n    "x2": "TARGET",\n    "x3": "Positive, linear factor",\n    "x4": "No relation",\n    "x5": "Positive quadratic factor",\n    "x6": "Positive, inverse quadratic factor"\n  }},\n  "interpretation": {{\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area", "is_target": true}},\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n}}```\n\nHere\'s a new test question:\nRelationship for {dependent} = "{relationship}"\nDomain = "{domain}"\nDomain description="{domain_desc}"\n\nRespond only with the answer JSON. Make sure that you do not forget to include the TARGET variable in the interpretation object.',
+        },
+    ]
+
+
+def OPENAI_SEMANTICS_GEN_W_MAP_MESSAGES(
+    dependent, relationship, domain, domain_desc, mapping
+):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given a partial mapping from variables to real-world concepts and a true relationship in a dataset, your task is to come up with an interpretation of real-world concepts for the variables without any assigned mapping (those starting with x). Suggest something reasonable. The dependent variable must be derivable only from the other variables in the dependent relationship. Give your answer as a JSON object. Here\'s an example:\n\nExample partial mapping and relationship:\n```json\n{{\n  "domain": "Sales",\n  "domain_description": "Related to product distribution, revenues, marketing, etc.",\n  "variable_mapping": {{\n    "x1": {{"description": "Population by area", "name": "pop_area"}},\n    "x2": {{"description": "Volume of product sales by area", "name": "sales_area"}},\n    "x4": {{"description": "Gender ratio of marketing team", "name": "gdr_ratio_mkt_team"}},\n    "x6": {{"description": "Distance to distribution center", "name": "dist_to_distr_ctr"}}\n  }},\n  "dependent_variable": "sales_area",\n  "dependent_relationship": "(96.4 * pop_area ** 3) + (88.72 * x5 ** 2) + (81.96 * dist_to_distr_ctr ** -2) + (28.13 * x3)  + (97.0)"\n}}```\nBased on this, an example answer would be:\n```json\n{{\n  "dependent_variable": "sales_area",\n  "missing_mapping": ["x3", "x5"],\n  "trends": {{\n    "x3": "Positive, linear factor",\n    "x5": "Positive quadratic factor"\n  }},\n  "interpretation": {{\n    "x3": {{"description": "Advertising spending", "name": "ad_spend"}},\n    "x5": {{"description": "Intensity of marketing campaign", "name": "mkt_intensity"}}\n  }}\n}}```\n\nHere\'s a new test question:\n```json\n{{\n  "domain": "{domain}",\n  "domain_description": "{domain_desc}",\n  "variable_mapping": {json.dumps(mapping, indent=2)},\n  "dependent_variable": "{dependent}",\n  "dependent_relationship": "{relationship}"\n}}```\nRespond only with the answer JSON.',
+        },
+    ]
+
+
+def OPENAI_SEMANTICS_GEN_SUMMARY_MESSAGES(dataset):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given the following descriptions of the columns of a dataset, your task is to come up with a natural language overview of the dataset, which should include (1) what the dataset is about, (2) how the data was collected, (3) when the data was collected, and (3) for what purpose the data was collected. Be specific and creative.\n\nExample dataset:\n```json\n{{  \n  "dataset": {{                                                                                                                                                                                       \n    "x6": {{"description": "Ancient artifact significance score", "name": "artifact_significance_score", "is_target": true}},\n    "x1": {{"description": "Distance to ancient city center", "name": "dist_to_ancient_city_ctr"}},\n    "x2": {{"description": "Quantity of discovered relics", "name": "relic_discovery_qty"}},\n    "x3": {{"description": "Years since last archaeological expedition", "name": "years_since_exp"}},\n    "x4": {{"description": "Number of artifacts in excavation site", "name": "artifact_qty"}},\n    "x5": {{"description": "Soil fertility coefficient", "name": "soil_fertility_coef"}},\n    "x7": {{"description": "Distance to ancient burial grounds", "name": "dist_to_burial_grounds"}},\n    "x8": {{"description": "Population estimate of ancient civilization", "name": "ancient_civilization_pop_estimate"}},\n    "x9": {{"description": "Temperature variation in excavation region", "name": "temp_variation"}}\n  }}\n}}```\nExample description:\nThis dataset is about archaeological explorations and findings linked to ancient civilizations. The data was collected in the form of field metrics during various archaeological expeditions during the late mid-20th century. The purpose of the data collection is to evaluate the significance of ancient artifacts discovered during excavations.\n\nHere is a new test dataset.\n{json.dumps(dataset, indent=2)}\nProvide only the description.',
+        },
+    ]
+
+
+def OPENAI_GEN_HYPO_MESSAGES(dataset):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {
+            'role': 'user',
+            'content': f'Given a dataset with its descriptions and the true functional relationship between its variables, your task is to generate 3 levels of hypotheses for the stated relationship in plain English. The three levels are "broad", "medium" and "narrow". Make sure that the hypotheses sound natural. *Only include concepts for variables that are present in the provided functional relationship.* Give your answer as a JSON.\n\nFor example, an example dataset might be the following:\n```json\n{{\n  "domain": "cybersecurity",\n  "summary": "This dataset is about measuring cybersecurity threats in a system. The data was collected by monitoring various cybersecurity metrics in a network environment. The purpose of the data collection is to assess and predict potential cybersecurity risks and vulnerabilities.",\n  "variables": [\n    {{\n      "description": "Level of cybersecurity threat",\n      "name": "cybersecurity_threat",\n      "is_target": true\n    }},\n    {{\n      "description": "Number of failed login attempts",\n      "name": "failed_login_attempts"\n    }},\n    {{\n      "description": "Amount of encrypted data",\n      "name": "encrypted_data"\n    }},\n    {{\n      "description": "Frequency of software updates",\n      "name": "software_updates"\n    }},\n    {{\n      "description": "Number of antivirus software installed",\n      "name": "antivirus_software"\n    }},\n    {{\n      "description": "Quality of firewall protection",\n      "name": "firewall_quality"\n    }}\n  ],\n  "relationship": {{\n    "dependent": "cybersecurity_threat",\n    "relation": "-53.5*encrypted_data**2 - 53.85*failed_login_attempts**2 + 67.75*firewall_quality - 92.16 - 36.68/software_updates**3"\n  }}\n}}```\nGiven this dataset, the following is a valid answer:\n```json\n{{\n  "broad": {{\n    "instruction": "Be vague. Only indicate which concepts might be related but not how they are related",\n    "hypothesis": "Threat to cybersecurity is influenced by several factors including the amount of encrypted data, the number of failed login attempts, the quality of the firewall, as well as how often the software is updated."\n  }},\n  "medium": {{\n    "instruction": "Be slightly more specific. For each factor, indicate carefully whether it positively or negatively affects the relationship, but do not indicate what the exponent is.",\n    "hypothesis": "Cybersecurity threat tends to decrease with the amount of data encryption, the number of failed login attempts, as well as the frequency of software updates to some extent, while improvement in the firewall quality has a positive effect."\n  }},\n  "narrow": {{\n    "instruction": "Be specific. Communicate the concepts, whether there is a positive or negative effect (be careful), and the meaning of the exponent",\n    "hypothesis": "The threat to cybersecurity interacts in a complex manner with various factors. As the amount of encrypted data increases, there is a quadratic decrease in threat. Similarly for the number of failed login attempts, there is a negative quadratic relationship. The quality of the firewall protection on the other hand demonstrates a positive and linear relationship. Finally, the frequency of software updates has an inverse cubic relationship to the threat."\n  }},\n}}\n```\n\nBased on this, provide an answer for the following test dataset:\n```json\n{dataset}```\nRespond only with a JSON.',
+        },
+    ]
+
+
+def create_prompt(usr_msg):
+    return [
+        {
+            'role': 'system',
+            'content': 'You are a helpful assistant who is not talkative. You only respond with the exact answer to a query without additional conversation.',
+        },
+        {'role': 'user', 'content': usr_msg},
+    ]
+
+
+def get_response(client, prompt, max_retry=5, model='gpt-3.5-turbo', verbose=False):
+    n_try = 0
+    while n_try < max_retry:
+        response = client.chat.completions.create(
+            model=model, messages=create_prompt(prompt), **OPENAI_GEN_HYP
+        )
+
+        # COMMENT: changed from
+        # response.choices[0].message.content.strip().strip('```json').strip('```')
+        content = response.choices[0].message.content
+        cleaned_content = content.split('```json')[1].split('```')[0].strip()
+        output = cleaned_content
+        try:
+            response_json = json.loads(output)
+            return response_json
+        except ValueError:
+            if verbose:
+                print(f'Bad JSON output:\n\n{output}')
+            n_try += 1
+            if n_try < max_retry:
+                if verbose:
+                    print('Retrying...')
+            else:
+                if verbose:
+                    print('Retry limit reached')
+    return None
+
+
+def get_code_fix(
+    client, code, error, max_retry=5, model='gpt-3.5-turbo', verbose=False
+):
+    prompt = f"""\
+Given the following code snippet and error message, provide a single-line fix for the error. \
+Note that the code is going to be executed using python `eval`. \
+The code should be executable and should not produce the error message. Be as specific as possible.
+
+Here's the code and the error:
+{{
+    "code": "{code}",
+    "error": "{error}"
+}}
+
+Return only a JSON object with the fixed code in the following format:
+```json
+{{
+    "fixed_code": "..."
+}}"""
+    response = get_response(
+        client, prompt, max_retry=max_retry, model=model, verbose=verbose
+    )
+    return response
+
+
+def get_new_hypothesis(
+    client, target, old, expr, cols, model='gpt-3.5-turbo', verbose=False
+):
+    prompt = f"""\
+Given a target column from a dataset, a pandas expression to derive the column from existing columns, a list of \
+existing columns, and a previously written hypothesis text, carefully check if the hypothesis text is consistent with \
+the pandas expression or not. If it is consistent, simply return the hypothesis as it is. If it is not consistent, \
+provide a new natural language hypothesis that is consistent with the pandas expression using only the provided \
+information. Be specific.
+
+Here's the information:
+```json
+{{
+    "target_column": "{target}",
+    "pandas_expression": "{expr}",
+    "existing_columns": {json.dumps(cols, indent=4)}
+    "old_hypothesis": "{old}",
+}}```
+
+Give your answer as a new JSON with the following format:
+```json
+{{
+    "hypothesis": "..."
+}}"""
+    response = get_response(client, prompt, model=model, verbose=verbose)
+    return response
+
+
+def replace_variable(client, expr, old, new, model='gpt-3.5-turbo', verbose=False):
+    prompt = f"""\
+Given a pandas "expression", replace mentions of the "old" column with its "new" value such that the resultant \
+expression is equivalent to the original expression.
+
+Here's the information:
+```json
+{{
+    "expression": "{expr}",
+    "old": "{old}",
+    "new": "{new}"
+}}```
+
+Give your answer as a new JSON with the following format:
+```json
+{{
+    "new_expression": "..."
+}}"""
+    response = get_response(client, prompt, model=model, verbose=verbose)
+    return response

evaluation/benchmarks/discoverybench/eval_utils/openai_semantic_gen_prompts.py

@@ -0,0 +1,151 @@
+common_hypothesis_features = [
+    '1-2 sentences',
+    'surprising finding',
+    'includes numeric concepts',
+    'includes categorical concepts',
+    'includes binary concepts',
+]
+hypothesis_features = [
+    ['requires within-cluster analysis'],
+    ['requires across-cluster analysis'],
+    ['corresponds to a polynomial relationship of some columns'],
+    ['corresponds to a ratio between some columns'],
+    ['requires temporal analysis'],
+    ['relationship is based on descriptive statistics of some columns'],
+    ['requires concepts based on percentage or percentiles'],
+    ['relationship is only applicable to one cluster in the data and not the others'],
+]
+
+column_features = [
+    [
+        'must have one target column',
+        'must have quantifiable columns',
+        'must have a few categorical columns',
+        'make sure the categorical column values do not contain special characters',
+        'include a few distractor columns',
+    ]
+]
+
+common_pandas_features = [
+    'must be executable using python `eval` to create the target column in variable `df` (pandas dataframe)',
+    "for e.g., df['A']**2 + 3*df['B'] + 9, np.where(df['A'] > 3, 'Yes', 'No'), etc.",
+    'variables in pandas_expression must be from the existing columns listed above',
+    'variables in pandas_expression must NOT contain the target column itself',
+]
+pandas_features = [
+    ['expression is a quadratic polynomial'],
+    ['expression is a cubic polynomial'],
+    ['expression is a ratio of existing columns'],
+    ['expression is derived through logical combination of existing columns'],
+    # workflow
+]
+pandas_features = [common_pandas_features + p for p in pandas_features]
+
+common_derived_features = [
+    '1-2 sentences',
+    'includes numeric concepts',
+    'includes categorical concepts',
+    'includes binary concepts',
+]
+derived_features = [common_derived_features + h for h in hypothesis_features]
+hypothesis_features = [common_hypothesis_features + h for h in hypothesis_features]
+
+PROMPT_HYP = """\
+Given a dataset topic and description, generate an interesting hypothesis based on \
+the provided instructions. Be creative and come up with an unusual finding.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis_features": %s,
+    "hypothesis": "..."
+}```
+
+Give your answer as a new JSON with the following format:
+```json
+{
+    "hypothesis": "..."
+}
+```"""
+
+PROMPT_COL = """\
+Given a dataset topic, its description, and a true hypothesis that can be determined from it, \
+generate a list of valid columns based on the provided instructions.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis": "%s",
+    "column_instructions": %s,
+    "columns": [
+        {
+            "col_name": "...",  # should be an "_"-separated string
+            "description": "...",
+            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
+            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
+            "is_distractor": true/false,  # boolean indicating whether this is a distractor that could cause confusion during data analysis
+            "is_target": true/false  # boolean indicating whether this is the target variable for the hypothesis; at least one column should be the target
+        },
+        ...
+    ],
+    "pandas_instructions": %s,
+    "pandas_equation_for_hypothesis": {
+        "target_col": "...",
+        "target_col_type": "...",
+        "target_col_range": {...},
+        "independent_cols_in_pandas_expression": [],  # list of column names that will be used to derive the target column
+        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
+    }
+}```
+
+Give your answer as a new JSON with the "columns" and "pandas_equation_for_hypothesis" keys filled using the following format:
+```json
+{
+    "columns": [...],
+    "pandas_equation_for_hypothesis": {...}
+}
+```"""
+
+PROMPT_DER = """\
+Given a dataset topic, description, a true hypothesis that can be determined from the data, \
+and a target column from the dataset, generate a hypothesis for the target column using new independent columns not present in the existing columns.
+
+```json
+{
+    "topic": "%s",
+    "description": "%s",
+    "hypothesis": "%s",
+    "existing_columns": %s,
+    "target_column": "%s",
+    "new_to_target_instructions": %s,
+    "new_to_target_hypothesis": "...",  # describe a relationship between new columns that explains the target column
+    "new_columns_for_target": [  # do not repeat any of the existing columns in the dataset
+        {
+            "col_name": "...",  # should be an "_"-separated string
+            "description": "...",
+            "data_type": "...",  # should be executable using python's `eval` function. E.g., str, float, int, bool
+            "data_range": {...},  # should be either {"min": ..., "max": ...} or {"values": [...]}
+        },
+        ...
+    ],
+    "pandas_instructions": %s,
+    "pandas_equation_for_new_to_target_hypothesis": {
+        "target_col": "...",
+        "target_col_type": "...",
+        "target_col_range": {...},
+        "independent_cols_in_pandas_expression": [],  # list of column names from new_columns_for_target that will be used to derive target_col
+        "pandas_expression": "..."  # expression to derive df[target_col] using df[ind_col1], df[ind_col2], etc.
+    }
+}```
+
+Give your answer as a new JSON with the "new_to_target_hypothesis", "new_columns_for_target", and \
+"pandas_equation_for_new_to_target_hypothesis" keys filled using the following format:
+```json
+{
+    "new_to_target_hypothesis": "...",
+    "new_columns_for_target": [...],
+    "pandas_equation_for_new_to_target_hypothesis": {...}
+}
+```"""

evaluation/benchmarks/discoverybench/eval_utils/response_parser.py

@@ -0,0 +1,52 @@
+workflow_summary_markers = [
+    'WORKFLOW SUMMARY',
+    'WORKFLOW_SUMMARY',
+    'WORKFLOW-SUMMARY',
+    'Workflow Summary',
+]
+
+final_answer_markers = [
+    'FINAL ANSWER',
+    'FINAL_ANSWER',
+    'FINAL-ANSWER',
+    'Final Answer',
+    'Scientific Hypothesis',
+    'Hypothesis',
+]
+
+next_agent_markers = [
+    'NEXT AGENT',
+    'NEXT-AGENT',
+    'NEXT_AGENT',
+    'FEEDBACK',
+]
+
+
+def extract_between(content, start_markers, end_markers=None):
+    for marker in start_markers:
+        if marker in content:
+            result = content.split(marker, 1)[1]
+            if end_markers:
+                for end_marker in end_markers:
+                    if end_marker in result:
+                        result = result.split(end_marker, 1)[0]
+            return result
+    return ''
+
+
+def extract_gen_hypo_from_logs(content: str):
+    error = ''
+
+    gen_workflow = extract_between(
+        content, workflow_summary_markers, final_answer_markers
+    )
+
+    if not gen_workflow:
+        error += 'No Workflow Summary found in the line. | '
+
+    gen_hypothesis = extract_between(content, final_answer_markers, next_agent_markers)
+
+    if not gen_hypothesis:
+        error += 'No Final Answer in the line.'
+
+    return gen_hypothesis, gen_workflow, error

evaluation/benchmarks/discoverybench/run_infer.py

@@ -0,0 +1,489 @@
+import asyncio
+import json
+import os
+
+import git
+import pandas as pd
+
+from evaluation.benchmarks.discoverybench.eval_utils.eval_w_subhypo_gen import (
+    run_eval_gold_vs_gen_NL_hypo_workflow,
+)
+from evaluation.benchmarks.discoverybench.eval_utils.response_parser import (
+    extract_gen_hypo_from_logs,
+)
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    codeact_user_response,
+    compatibility_for_eval_history_pairs,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AgentConfig,
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    parse_arguments,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.base import Runtime
+from openhands.utils.async_utils import call_async_from_sync
+
+EVALUATION_LLM = 'gpt-4-1106-preview'
+
+DATA_FILES = {}
+
+LIBRARIES = [
+    'pandas',
+    'numpy',
+    'scipy',
+    'matplotlib',
+    'seaborn',
+    'scikit-learn',
+    'statsmodels',
+]
+
+AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
+    'CodeActAgent': codeact_user_response,
+}
+
+AGENT_CLS_TO_INST_SUFFIX = {
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please finish the interaction using the "finish" tool.\n'
+}
+
+
+def get_config(
+    metadata: EvalMetadata,
+) -> AppConfig:
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='eventstream',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=True,
+            use_host_network=False,
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    agent_config = AgentConfig(
+        function_calling=False,
+        codeact_enable_jupyter=True,
+        codeact_enable_browsing_delegate=True,
+    )
+    config.set_agent_config(agent_config)
+    return config
+
+
+def get_dv_query_for_real(
+    datasets, question, domain_knowledge=None, workflow_tags=None
+):
+    """
+    Prepare a structured query for the agent to execute on the specified datasets.
+
+    This function constructs a query by compiling metadata from the provided datasets, along with any relevant domain knowledge and workflow tags.
+
+    Args:
+        datasets: List of datasets
+        question: Query to be answered
+        domain_knowledge: Domain knowledge if any
+        workflow_tags: Workflow tags if any
+
+    Returns:
+        query_to_dv: Query to be run on the dataset
+        dataset_meta: Metadata of the dataset
+    """
+
+    dataset_meta = ''
+    for dataset_metadata in datasets:
+        dataset_meta += 'Dataset name: ' + dataset_metadata['name']
+        dataset_meta += 'Dataset description: ' + dataset_metadata['description']
+        dataset_meta += '\nBrief description of columns: '
+        for col in dataset_metadata['columns']['raw']:
+            dataset_meta += col['name'] + ': ' + col['description'] + ', '
+
+    query_to_dv = dataset_meta
+
+    query_to_dv += f'\nQuery: {question}'
+
+    if domain_knowledge:
+        query_to_dv += (
+            '\nAdditionally, we provide some hints that might be useful to solve the task. Domain Knowledge: \n'
+            + domain_knowledge
+            + '.\n'
+        )
+
+    if workflow_tags:
+        query_to_dv += 'The meta tags are: ' + workflow_tags + '.\n'
+
+    query_to_dv += (
+        'In the final answer, please write down a scientific hypothesis in '
+        'natural language, derived from the provided dataset, clearly stating the '
+        'context of hypothesis (if any), variables chosen (if any) and '
+        'relationship between those variables (if any) including any statistical significance.'
+        'Also generate a summary of the full workflow starting from data loading that led to the final answer as WORKFLOW SUMMARY:'
+    )
+
+    # Run the NL query through datavoyager
+    return query_to_dv, dataset_meta
+
+
+def initialize_runtime(runtime: Runtime, data_files: list[str]):
+    """
+    Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    action = CmdRunAction(command='mkdir -p /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    action = CmdRunAction(command='cd /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    for file in data_files:
+        runtime.copy_to(
+            file,
+            '/workspace',
+        )
+
+    for lib in LIBRARIES:
+        action = CmdRunAction(command=f'pip install {lib}')
+        logger.info(action, extra={'msg_type': 'ACTION'})
+        obs = runtime.run_action(action)
+        assert obs.exit_code == 0
+
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+
+
+def get_last_agent_finish_action(state: State) -> AgentFinishAction:
+    for event in reversed(state.history):
+        if isinstance(event, AgentFinishAction):
+            return event
+    return None
+
+
+def get_last_message_action(state: State) -> MessageAction:
+    for event in reversed(state.history):
+        if isinstance(event, MessageAction):
+            return event
+    return None
+
+
+def complete_runtime(state: State):
+    last_agent_finish_action = get_last_agent_finish_action(state)
+    last_agent_message_action = get_last_message_action(state)
+
+    if last_agent_finish_action is not None:
+        final_message_1 = last_agent_finish_action.thought
+        gen_hypo_1, gen_workflow_1, error_1 = extract_gen_hypo_from_logs(
+            final_message_1
+        )
+    else:
+        gen_hypo_1, gen_workflow_1, error_1 = '', '', ''
+
+    if last_agent_message_action is not None:
+        final_message_2 = last_agent_message_action.content
+        gen_hypo_2, gen_workflow_2, error_2 = extract_gen_hypo_from_logs(
+            final_message_2
+        )
+    else:
+        gen_hypo_2, gen_workflow_2, error_2 = '', '', ''
+
+    if gen_hypo_1 and gen_hypo_2:
+        test_result = {
+            'gen_hypo': last_agent_finish_action.thought
+            if last_agent_finish_action
+            else last_agent_message_action.content,
+            'gen_workflow': '',
+            'error': '',
+        }
+        return test_result
+
+    test_result = {
+        'gen_hypo': gen_hypo_1 if gen_hypo_1 else gen_hypo_2,
+        'gen_workflow': gen_workflow_1 if gen_workflow_1 else gen_workflow_2,
+        'error': error_1 if error_1 else error_2,
+    }
+
+    return test_result
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+):
+    """
+    Process and evaluate a single instance of the dataset.
+
+    This function executes the OpenHands agent
+    for a specific instance of the dataset. It retrieves
+    the agent's results and evaluates them against the gold
+    hypothesis.
+
+    Args:
+        instance: A single row of the dataset
+        metadata: Metadata for the evaluation
+        reset_logger: Whether to reset the logger
+
+    Returns:
+        output: EvalOutput object
+    """
+
+    config = get_config(metadata)
+
+    # Setup the logger properly, so you can run
+    # multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, instance.instance_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
+
+    problem_statement, dataset_metadata = get_dv_query_for_real(
+        datasets=instance.datasets,
+        question=instance.query,
+        domain_knowledge=instance.domain_knowledge,
+        workflow_tags=instance.workflow_tags,
+    )
+
+    # Prepare instruction
+    instruction = (
+        f'You are a discovery agent who can execute a python code only once to answer a query based on one or more datasets. The datasets will be present in the current directory.\n\n'
+        'Environment has been set up for you to start working. You may assume all necessary tools and datasets are installed.\n\n'
+        '# Problem Statement\n'
+        f'{problem_statement}\n\n'
+    )
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
+        'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
+        'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
+    )
+    # NOTE: You can actually set slightly different instruction for different agents
+    instruction += AGENT_CLS_TO_INST_SUFFIX[metadata.agent_class]
+
+    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    runtime = create_runtime(config)
+    call_async_from_sync(runtime.connect)
+    initialize_runtime(runtime, instance.data_files)
+
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=MessageAction(content=instruction),
+            runtime=runtime,
+            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(
+                metadata.agent_class
+            ),
+        )
+    )
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    metrics = state.metrics.get() if state.metrics else None
+    test_result = complete_runtime(state)
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = compatibility_for_eval_history_pairs(state.history)
+
+    # DiscoveryBench Evaluation
+    eval_rec = run_eval_gold_vs_gen_NL_hypo_workflow(
+        query=instance.query,
+        gold_hypo=instance.gold_hypo,
+        gold_workflow='',
+        gen_hypo=test_result['gen_hypo'],
+        gen_workflow='',
+        dataset_meta=instance.dataset_metadata,
+        llm_used=EVALUATION_LLM,
+        dataset_type='real',
+    )
+
+    test_result['eval_rec'] = eval_rec
+
+    output = EvalOutput(
+        instance_id=str(instance.instance_id),
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result=test_result,
+    )
+
+    return output
+
+
+def update_csv_name(name):
+    name = name.replace('-', '_')
+
+    if 'meta_regression' in name:
+        name = name.replace('meta_regression', 'meta-regression')
+    if 'ML_enabled' in name:
+        name = name.replace('ML_enabled', 'ML-enabled')
+
+    return name
+
+
+def list_csv_files(list_of_datasets):
+    res = []
+    for ele in list_of_datasets:
+        for key, value in ele.items():
+            if key == 'name':
+                csv_file_name = update_csv_name(value)
+                res.append(DATA_FILES[csv_file_name])
+    return res
+
+
+def create_dataset(repo_location: str, split: str = 'test'):
+    """
+    Create a dataset from the discoverybench repository
+    by walking through the repository and extracting metadata
+    from the metadata_{}.json files
+
+    Args:
+        repo_location: Location of the repository
+        split: Split of the dataset to use
+
+    Returns:
+        df: DataFrame containing the dataset instances
+    """
+
+    data_dict = {}
+
+    data_location = os.path.join(repo_location, 'discoverybench', 'real', split)
+    answer_key_location = os.path.join(repo_location, 'eval', 'answer_key_real.csv')
+
+    idx = 0
+
+    for root, dirs, files in os.walk(data_location):
+        for file in files:
+            if file.endswith('.json'):
+                if 'metadata' in file:
+                    metadata = json.load(open(os.path.join(root, file)))
+
+                    dataset = root.split('/')[-1]
+                    metadata_id = file.split('_')[-1].split('.')[0]
+                    domain = metadata.get('domain', '')
+                    domain_knowledge = metadata.get('domain_knowledge', '')
+                    workflow_tags = metadata.get('workflow_tags', '')
+                    datasets = metadata.get('datasets', [])
+                    queries = metadata.get('queries', [])
+                    gold_workflow = metadata.get('workflow')
+
+                    # loop through queries list to get queries
+                    # and each query has qid; add that to dictionary
+                    for query in queries[0]:
+                        qid = query.get('qid', '')
+
+                        data = {
+                            'dataset': dataset,
+                            'metadata_id': metadata_id,
+                            'qid': qid,
+                            'domain': domain,
+                            'domain_knowledge': domain_knowledge,
+                            'workflow_tags': workflow_tags,
+                            'datasets': datasets,
+                            'question_type': query['question_type'],
+                            'query': query['question'],
+                            'gold_workflow': gold_workflow,
+                            'dataset_metadata': metadata,
+                        }
+
+                        data_dict[idx] = data
+                        idx += 1
+
+            if file.endswith('.csv'):
+                DATA_FILES[file] = os.path.join(root, file)
+            if file.endswith('.txt'):
+                DATA_FILES[file] = os.path.join(root, file)
+
+    df = pd.DataFrame.from_dict(data_dict, orient='index')
+
+    df['instance_id'] = df.index
+
+    df['data_files'] = df['datasets'].apply(lambda x: list_csv_files(x))
+
+    answer_key = pd.read_csv(answer_key_location)
+
+    answer_key = answer_key.rename(
+        columns={
+            'metadataid': 'metadata_id',
+            'query_id': 'qid',
+            'gold_hypothesis': 'gold_hypothesis',
+        }
+    )
+
+    df['qid'] = df['qid'].astype(int)
+    df['metadata_id'] = df['metadata_id'].astype(int)
+
+    answer_key['qid'] = answer_key['qid'].astype(int)
+    answer_key['metadata_id'] = answer_key['metadata_id'].astype(int)
+
+    df = pd.merge(df, answer_key, on=['dataset', 'metadata_id', 'qid'], how='left')
+
+    return df
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+
+    # clone git repositor for csv files
+    repo_url = 'https://github.com/allenai/discoverybench.git'
+    repo_location = 'git-discoverybench-allenai'
+
+    try:
+        git.Repo.clone_from(repo_url, repo_location)
+    except git.exc.GitCommandError:
+        print('Repository already exists')
+
+    dataset = create_dataset(repo_location)
+
+    # check if there is any empty csv_file
+    if dataset['data_files'].isnull().any():
+        raise ValueError('Some csv files are missing.')
+
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+
+    metadata = make_metadata(
+        llm_config,
+        'discoverybench-python',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+    )

evaluation/benchmarks/discoverybench/scripts/run_infer.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=$5
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+
+# ################################################################################
+
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default CodeActAgent"
+  AGENT="CodeActAgent"
+fi
+
+get_openhands_version
+
+echo "AGENT: $AGENT"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+COMMAND="poetry run python evaluation/benchmarks/discoverybench/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 10 \
+  --max-chars 10000000 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $OPENHANDS_VERSION"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND

evaluation/benchmarks/gaia/README.md

@@ -4,17 +4,18 @@ This folder contains evaluation harness for evaluating agents on the [GAIA bench
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## Run the evaluation
+
 We are using the GAIA dataset hosted on [Hugging Face](https://huggingface.co/datasets/gaia-benchmark/GAIA).
 Please accept the terms and make sure to have logged in on your computer by `huggingface-cli login` before running the evaluation.
 
-Following is the basic command to start the evaluation. Here we are evaluating on the validation set for the `2023_all` split. You can adjust `./evaluation/gaia/scripts/run_infer.sh` to change the subset you want to evaluate on.
+Following is the basic command to start the evaluation. Here we are evaluating on the validation set for the `2023_all` split. You can adjust `./evaluation/benchmarks/gaia/scripts/run_infer.sh` to change the subset you want to evaluate on.
 
 ```bash
-./evaluation/gaia/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [gaia_subset]
-# e.g., ./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 300
+./evaluation/benchmarks/gaia/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [gaia_subset]
+# e.g., ./evaluation/benchmarks/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 300
 ```
 
 where `model_config` is mandatory, while `git-version`, `agent`, `eval_limit` and `gaia_subset` are optional.
@@ -35,13 +36,14 @@ to `CodeActAgent`.
 For example,
 
 ```bash
-./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 10
+./evaluation/benchmarks/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 10
 ```
 
 ## Get score
 
 Then you can get stats by running the following command:
+
 ```bash
-python ./evaluation/gaia/get_score.py \
+python ./evaluation/benchmarks/gaia/get_score.py \
 --file <path_to/output.json>
 ```

evaluation/benchmarks/gaia/run_infer.py

@@ -7,11 +7,12 @@ import huggingface_hub
 import pandas as pd
 from datasets import load_dataset
 
-from evaluation.gaia.scorer import question_scorer
+from evaluation.benchmarks.gaia.scorer import question_scorer
 from evaluation.utils.shared import (
     EvalMetadata,
     EvalOutput,
     codeact_user_response,
+    compatibility_for_eval_history_pairs,
     make_metadata,
     prepare_dataset,
     reset_logger_for_multiprocessing,
@@ -166,7 +167,7 @@ def process_instance(
 
     model_answer_raw = ''
     # get the last message or thought from the agent
-    for event in state.history.get_events(reverse=True):
+    for event in reversed(state.history):
         if event.source == 'agent':
             if isinstance(event, AgentFinishAction):
                 model_answer_raw = event.thought
@@ -203,7 +204,7 @@ def process_instance(
     # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
     # for compatibility with the existing output format, we can remake the pairs here
     # remove when it becomes unnecessary
-    histories = state.history.compatibility_for_eval_history_pairs()
+    histories = compatibility_for_eval_history_pairs(state.history)
 
     # Save the output
     output = EvalOutput(

evaluation/benchmarks/gaia/scripts/run_infer.sh

@@ -21,28 +21,28 @@ if [ -z "$AGENT" ]; then
   AGENT="CodeActAgent"
 fi
 
-get_agent_version
+get_openhands_version
 
 if [ -z "$LEVELS" ]; then
   LEVELS="2023_level1"
   echo "Levels not specified, use default $LEVELS"
 fi
 
-get_agent_version
+get_openhands_version
 
 echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
+echo "OPENHANDS_VERSION: $OPENHANDS_VERSION"
 echo "MODEL_CONFIG: $MODEL_CONFIG"
 echo "LEVELS: $LEVELS"
 
-COMMAND="poetry run python ./evaluation/gaia/run_infer.py \
+COMMAND="poetry run python ./evaluation/benchmarks/gaia/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
   --max-iterations 30 \
   --level $LEVELS \
   --data-split validation \
   --eval-num-workers $NUM_WORKERS \
-  --eval-note ${AGENT_VERSION}_${LEVELS}"
+  --eval-note ${OPENHANDS_VERSION}_${LEVELS}"
 
 if [ -n "$EVAL_LIMIT" ]; then
   echo "EVAL_LIMIT: $EVAL_LIMIT"

evaluation/benchmarks/gorilla/README.md

@@ -4,14 +4,14 @@ This folder contains evaluation harness we built on top of the original [Gorilla
 
 ## Setup Environment and LLM Configuration
 
-Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
+Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
 
 ## Run Inference on APIBench Instances
 
 Make sure your Docker daemon is running, then run this bash script:
 
 ```bash
-./evaluation/gorilla/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [hubs]
+./evaluation/benchmarks/gorilla/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [hubs]
 ```
 
 where `model_config` is mandatory, while all other arguments are optional.
@@ -35,5 +35,5 @@ Note: in order to use `eval_limit`, you must also set `agent`; in order to use `
 For example,
 
 ```bash
-./evaluation/gorilla/scripts/run_infer.sh llm 0.6.2 CodeActAgent 10 th
+./evaluation/benchmarks/gorilla/scripts/run_infer.sh llm 0.6.2 CodeActAgent 10 th
 ```