fix ssh_box error parsing

.github/.codecov.yml

@@ -6,14 +6,9 @@ coverage:
   status:
     patch:
       default:
-        threshold: 100% # allow patch coverage to be lower than project coverage by any amount
+        threshold: 10% # allow patch coverage to be lower than project coverage by at most 10%
     project:
       default:
         threshold: 5% # allow project coverage to drop at most 5%
 
 comment: false
-github_checks:
-    annotations: false
-
-ignore:
-  - "agenthub/SWE_agent/**" # SWE agent is deprecated

.github/ISSUE_TEMPLATE/bug_template.yml

@@ -28,8 +28,8 @@ body:
   - type: textarea
     id: current-version
     attributes:
-      label: Current OpenDevin version
-      description: What version of OpenDevin are you using? If you're running in docker, tell us the tag you're using (e.g. ghcr.io/opendevin/opendevin:0.3.1).
+      label: Current Version
+      description: What version are you using? If you're running in docker, tell us the tag you're using (e.g. ghcr.io/opendevin/opendevin:0.3.1).
       render: bash
     validations:
       required: true
@@ -52,12 +52,6 @@ body:
         - Model:
         - Agent:
 
-  - type: textarea
-    id: os-version
-    attributes:
-      label: Operating System
-      description: What Operating System are you using? Linux, Mac OS, WSL on Windows
-
   - type: textarea
     id: repro-steps
     attributes:

.github/dependabot.yml

@@ -1,15 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/" # Location of package manifests
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "npm" # See documentation for possible values
-    directory: "/frontend" # Location of package manifests
-    schedule:
-      interval: "daily"

.github/workflows/run-integration-tests.yml

@@ -7,23 +7,32 @@ concurrency:
 on:
   push:
     branches:
-      - main
-    paths-ignore:
-      - '**/*.md'
-      - 'frontend/**'
-      - 'docs/**'
-      - 'evaluation/**'
+    - main
   pull_request:
 
 jobs:
-  integration-tests-on-linux:
-    name: Integration Tests on Linux
+  integration-tests:
+    name: Integration Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.11"]
-        sandbox: ["ssh", "exec", "local"]
+        agent: ["SWEAgent", "PlannerAgent", "MonologueAgent", "CodeActAgent"]
+        sandbox: ["ssh", "exec"]
+        include:
+          - agent: "MonologueAgent"
+            embedding-model: "local"
+          - agent: "MonologueAgent"
+            # sufficient to have one agent testing against local sandbox
+            sandbox: "local"
+            embedding-model: "local"
+          - agent: "SWEAgent"
+            embedding-model: "none"
+          - agent: "PlannerAgent"
+            embedding-model: "none"
+          - agent: "CodeActAgent"
+            embedding-model: "none"
     steps:
       - uses: actions/checkout@v4
 
@@ -33,7 +42,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
           cache: 'poetry'
 
       - name: Install Python dependencies using Poetry
@@ -45,57 +54,23 @@ jobs:
       - name: Run Integration Tests
         env:
           SANDBOX_TYPE: ${{ matrix.sandbox }}
+          AGENT: ${{ matrix.agent }}
+          MAX_ITERATIONS: 10
+          LLM_EMBEDDING_MODEL: ${{ matrix.embedding-model }}
         run: |
-          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
-
+          rm -rf workspace
+          mkdir workspace
+          WORKSPACE_BASE="$GITHUB_WORKSPACE/workspace" \
+            WORKSPACE_MOUNT_PATH="$GITHUB_WORKSPACE/workspace" \
+            poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml \
+            -s ./tests/integration
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  integration-tests-on-mac:
-    name: Integration Tests on MacOS
-    runs-on: macos-13
-    if: contains(github.event.pull_request.title, 'mac') || contains(github.event.pull_request.title, 'Mac')
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11"]
-        sandbox: ["ssh"]
+  test_matrix_success:
+    name: All Integration Tests Passed
+    runs-on: ubuntu-latest
+    needs: [integration-tests]
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Install poetry via pipx
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
-
-      - name: Install Python dependencies using Poetry
-        run: poetry install
-
-      - name: Install & Start Docker
-        run: |
-          brew install colima docker
-          colima start
-
-          # For testcontainers to find the Colima socket
-          # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
-          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-
-      - name: Build Environment
-        run: make build
-
-      - name: Run Integration Tests
-        env:
-          SANDBOX_TYPE: ${{ matrix.sandbox }}
-        run: |
-          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    - run: echo Done!

.github/workflows/run-unit-tests.yml

@@ -7,12 +7,7 @@ concurrency:
 on:
   push:
     branches:
-      - main
-    paths-ignore:
-      - '**/*.md'
-      - 'frontend/**'
-      - 'docs/**'
-      - 'evaluation/**'
+    - main
   pull_request:
 
 jobs:
@@ -20,7 +15,7 @@ jobs:
     name: Test on macOS
     runs-on: macos-13
     env:
-      INSTALL_DOCKER: "0" # Set to '0' to skip Docker installation
+      INSTALL_DOCKER: '0'  # Set to '0' to skip Docker installation
     strategy:
       matrix:
         python-version: ["3.11"]
@@ -35,7 +30,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+          cache: 'poetry'
 
       - name: Install Python dependencies using Poetry
         run: poetry install
@@ -54,7 +49,7 @@ jobs:
         run: make build
 
       - name: Run Tests
-        run: poetry run pytest --forked --cov=agenthub --cov=opendevin --cov-report=xml ./tests/unit -k "not test_sandbox"
+        run: poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml ./tests/unit -k "not test_sandbox"
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -64,7 +59,7 @@ jobs:
     name: Test on Linux
     runs-on: ubuntu-latest
     env:
-      INSTALL_DOCKER: "0" # Set to '0' to skip Docker installation
+      INSTALL_DOCKER: '0'  # Set to '0' to skip Docker installation
     strategy:
       matrix:
         python-version: ["3.11"]
@@ -79,7 +74,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+          cache: 'poetry'
 
       - name: Install Python dependencies using Poetry
         run: poetry install --without evaluation
@@ -88,7 +83,7 @@ jobs:
         run: make build
 
       - name: Run Tests
-        run: poetry run pytest --forked --cov=agenthub --cov=opendevin --cov-report=xml ./tests/unit -k "not test_sandbox"
+        run: poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml ./tests/unit -k "not test_sandbox"
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -107,8 +102,8 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-          cache: "poetry"
+          python-version: '3.11'
+          cache: 'poetry'
 
       - name: Install Python dependencies using Poetry
         run: poetry install
@@ -124,3 +119,9 @@ jobs:
         uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+  test_matrix_success:
+    name: All Mac/Linux Tests Passed
+    runs-on: ubuntu-latest
+    needs: [test-on-macos, test-on-linux, test-for-sandbox]
+    steps:
+    - run: echo Done!

.gitignore

@@ -126,7 +126,6 @@ env/
 venv/
 ENV/
 env.bak/
-.env.bak
 venv.bak/
 *venv/
 
@@ -203,8 +202,7 @@ cache
 
 # configuration
 config.toml
-config.toml.bak
-evaluation/swe_bench/eval_workspace*
+evaluation/swe_bench/eval_workspace
 evaluation/outputs
 evaluation/evaluation_outputs
 test_results*

Development.md

@@ -94,8 +94,3 @@ poetry run pytest ./tests/unit/test_sandbox.py
 #### Integration tests
 
 Please refer to [this README](./tests/integration/README.md) for details.
-
-### 9. Add or update dependency
-
-1. Add your dependency in `pyproject.toml` or use `peotry add xxx`
-2. Update the poetry.lock file via `poetry lock --no-update`

Makefile

@@ -130,9 +130,8 @@ pull-docker-image:
 
 install-python-dependencies:
 	@echo "$(GREEN)Installing Python dependencies...$(RESET)"
-	poetry env use python3.11
 	@if [ "$(shell uname)" = "Darwin" ]; then \
-		echo "$(BLUE)Installing chroma-hnswlib...$(RESET)"; \
+		echo "$(BLUE)Installing `chroma-hnswlib`...$(RESET)"; \
 		export HNSWLIB_NO_NATIVE=1; \
 		poetry run pip install chroma-hnswlib; \
 	fi

README.md

@@ -27,15 +27,14 @@
   <a href="https://join.slack.com/t/opendevin/shared_invite/zt-2i1iqdag6-bVmvamiPA9EZUu7oCO6KhA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community"></a>
   <a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community"></a>
   <br/>
-  <a href="https://huggingface.co/spaces/OpenDevin/evaluation"><img src="https://img.shields.io/badge/SWE--bench%20Lite-25.0%25-green?style=for-the-badge" alt="SWE-bench "></a>
-  <a href="https://codecov.io/github/opendevin/opendevin?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/opendevin/opendevin?style=for-the-badge"></a>
+  <a href="https://xwang.dev/blog/2024/opendevin-codeact-1.0-swebench/"><img src="https://img.shields.io/badge/SWE--bench%20Lite-21.0%25-green?style=for-the-badge" alt="SWE-bench "></a>
 </div>
 
 <!-- PROJECT LOGO -->
 <div align="center">
   <img src="./docs/static/img/logo.png" alt="Logo" width="200" height="200">
   <h1 align="center">OpenDevin: Code Less, Make More</h1>
-  <a href="https://opendevin.github.io/OpenDevin/"><img src="https://img.shields.io/badge/Documentation-OpenDevin-blue?logo=googledocs&logoColor=white&style=for-the-badge" alt="Check out the documentation"></a>
+  <a href="https://opendevin.github.io/OpenDevin/"><img src="https://img.shields.io/badge/Documenation-OpenDevin-blue?logo=googledocs&logoColor=white&style=for-the-badge" alt="Check out the documentation"></a>
 </div>
 <hr>
 
@@ -45,22 +44,14 @@ OpenDevin agents collaborate with human developers to write code, fix bugs, and
 
 ![App screenshot](./docs/static/img/screenshot.png)
 
-## ⚡ Getting Started
-The easiest way to run OpenDevin is inside a Docker container. It works best with the most recent version of Docker, `26.0.0`.
-You must be using Linux, Mac OS, or WSL on Windows.
-
-To start the app, run these commands, replacing `$(pwd)/workspace` with the directory you want OpenDevin to work with.
+## ⚡ Quick Start
+You can run OpenDevin with Docker. It works best with the most recent
+version of Docker, `26.0.0`.
 
 ```bash
-# The directory you want OpenDevin to work with. MUST be an absolute path!
+#The directory you want OpenDevin to modify. MUST be an absolute path!
 export WORKSPACE_BASE=$(pwd)/workspace;
-```
 
-> [!WARNING]  
-> OpenDevin runs bash commands within a Docker sandbox, so it should not affect your machine. 
-> But your workspace directory will be attached to that sandbox, and files in the directory may be modified or deleted.
-
-```bash
 docker run \
     -it \
     --pull=always \
@@ -73,8 +64,6 @@ docker run \
     ghcr.io/opendevin/opendevin:0.5
 ```
 
-You'll find OpenDevin running at [http://localhost:3000](http://localhost:3000).
-
 ## 🚀 Documentation
 
 To learn more about the project, and for tips on using OpenDevin,

agenthub/SWE_agent/agent.py

@@ -6,6 +6,7 @@ from opendevin.events.action import (
     FileWriteAction,
     MessageAction,
 )
+from opendevin.events.observation import Observation
 from opendevin.events.serialization.event import event_to_memory
 from opendevin.llm.llm import LLM
 
@@ -20,8 +21,6 @@ from .prompts import (
 
 
 class SWEAgent(Agent):
-    VERSION = '1.0'
-    DEPRECATED = True
     """
     An attempt to recreate swe_agent with output parsing, prompting style, and Application Computer Interface (ACI).
 
@@ -32,11 +31,17 @@ class SWEAgent(Agent):
         super().__init__(llm)
         self.memory_window = 4
         self.max_retries = 2
+        self.running_memory: list[str] = []
         self.cur_file: str = ''
         self.cur_line: int = 0
 
+    def _remember(self, action: Action, observation: Observation) -> None:
+        """Agent has a limited memory of the few steps implemented as a queue"""
+        memory = MEMORY_FORMAT(event_to_memory(action), event_to_memory(observation))
+        self.running_memory.append(memory)
+
     def _think_act(self, messages: list[dict]) -> tuple[Action, str]:
-        resp = self.llm.do_completion(
+        resp = self.llm.completion(
             messages=messages,
             temperature=0.05,
         )
@@ -62,36 +67,24 @@ class SWEAgent(Agent):
             2. Perform think-act - prompt model for action and reasoning
             3. Catch errors - ensure model takes action (5 attempts max)
         """
-        # retrieve short term memories from state.history, up to memory_window
-        memory_window = min(self.memory_window, len(state.history))
-        running_memory: list[str] = []
-        for prev_action, obs in state.history[-memory_window:]:
-            running_memory.append(
-                MEMORY_FORMAT(event_to_memory(prev_action), event_to_memory(obs))
-            )
+        for prev_action, obs in state.updated_info:
+            self._remember(prev_action, obs)
 
         goal = state.get_current_user_intent()
-
-        # always in the prompt if they exist: file and line
         prompt = STEP_PROMPT(goal, self.cur_file, self.cur_line)
 
-        # prepare messages
         msgs = [
             {'content': SYSTEM_MESSAGE, 'role': 'system'},
             {'content': prompt, 'role': 'user'},
         ]
 
-        # insert memories
-        if len(running_memory) > 0:
-            context = CONTEXT_PROMPT(running_memory, self.memory_window)
+        if len(self.running_memory) > 0:
+            context = CONTEXT_PROMPT(self.running_memory, self.memory_window)
             msgs.insert(1, {'content': context, 'role': 'user'})
         # clrs = [''] * (len(msgs)-2) + ['\033[0;36m', '\033[0;35m']
         # print('\n\n'.join([c+m['content']+'\033[0m' for c, m in zip(clrs, msgs)]))
-
-        # send it over
         action, thought = self._think_act(messages=msgs)
 
-        # be robust with malformed responses
         start_msg_len = len(msgs)
         while not action and len(msgs) < self.max_retries + start_msg_len:
             error = NO_ACTION(thought)
@@ -107,9 +100,9 @@ class SWEAgent(Agent):
         return action
 
     def search_memory(self, query: str) -> list[str]:
-        # return [item for item in self.running_memory if query in item]
-        raise NotImplementedError('Search_memory not implemented currently')
+        return [item for item in self.running_memory if query in item]
 
     def reset(self) -> None:
         """Used to reset the agent"""
+        self.running_memory = []
         super().reset()

agenthub/SWE_agent/prompts.py

@@ -92,7 +92,7 @@ Notes:
 - To execute multiple commands you should write them down in your thoughts section so you can remember it on the next step and execute them then.
 - The only commands you are not capable of executing are interactive commands like `python` or `node` by themselves.
 - If you think that you have completed the task that has been given to you based on your previous actions and outputs then use ``` exit ``` as the command to let the system know that you are done.
-- DO NOT make any copies of your previous memories, those will be provided to you at each step, making copies just wastes time and energy. Think smarter not harder.
+- DO NOT make any copies of your previous memories those will be provided to you at each step, making copies just wastes time and energy. Think smarter not harder.
 - The write and edit commands requires proper indentation in the content section ex. `write hw.py def hello():\n    print(\'Hello World\')` this is how you would have to format your write command.
     - The white spaces matter as the code changes will be added to the code so they must have proper syntax.
 
@@ -114,8 +114,8 @@ Do not provide anything extra just your thought and action.
 """
 
 SYSTEM_MESSAGE = f"""SYSTEM INFO:
-You are an autonomous coding agent, here to provide solutions for coding issues.
-You have been designed to assist with a wide range of programming tasks, from code editing and debugging to testing and deployment.
+You am an autonomous coding agent, here to provide solutions for coding issues.
+You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
 You have access to a variety of tools and commands that you can use to help you solve problems efficiently.
 
 {GENERAL_GUIDELINES}
@@ -171,9 +171,8 @@ Begin with your thought about the next step and then come up with an action to p
 """.strip()
 
 
-def unpack_dict(data: dict, restrict: list[str] | None = None):
+def unpack_dict(data: dict, restrict: list[str] = []):
     lines = []
-    restrict = [] if restrict is None else restrict
     for key, value in data.items():
         if key in restrict:
             continue

agenthub/__init__.py

@@ -10,9 +10,7 @@ load_dotenv()
 
 from . import (  # noqa: E402
     SWE_agent,
-    browsing_agent,
     codeact_agent,
-    codeact_swe_agent,
     delegator_agent,
     dummy_agent,
     monologue_agent,
@@ -22,12 +20,10 @@ from . import (  # noqa: E402
 __all__ = [
     'monologue_agent',
     'codeact_agent',
-    'codeact_swe_agent',
     'planner_agent',
     'SWE_agent',
     'delegator_agent',
     'dummy_agent',
-    'browsing_agent',
 ]
 
 for agent in all_microagents.values():

agenthub/browsing_agent/README.md

@@ -1,16 +0,0 @@
-# Browsing Agent Framework
-
-This folder implements the basic BrowserGym [demo agent](https://github.com/ServiceNow/BrowserGym/tree/main/demo_agent) that enables full-featured web browsing.
-
-
-## Test run
-
-Note that for browsing tasks, GPT-4 is usually a requirement to get reasonable results, due to the complexity of the web page structures.
-
-```
-poetry run python ./opendevin/core/main.py \
-           -i 10 \
-           -t "tell me the usa's president using google search" \
-           -c BrowsingAgent \
-           -m gpt-4o-2024-05-13
-```

agenthub/browsing_agent/__init__.py

@@ -1,5 +0,0 @@
-from opendevin.controller.agent import Agent
-
-from .browsing_agent import BrowsingAgent
-
-Agent.register('BrowsingAgent', BrowsingAgent)

agenthub/browsing_agent/browsing_agent.py

@@ -1,167 +0,0 @@
-import ast
-
-from browsergym.core.action.highlevel import HighLevelActionSet
-from browsergym.utils.obs import flatten_axtree_to_str
-
-from opendevin.controller.agent import Agent
-from opendevin.controller.state.state import State
-from opendevin.core.logger import opendevin_logger as logger
-from opendevin.events.action import (
-    Action,
-    AgentFinishAction,
-    BrowseInteractiveAction,
-    MessageAction,
-)
-from opendevin.events.observation import BrowserOutputObservation
-from opendevin.llm.llm import LLM
-from opendevin.runtime.plugins import (
-    PluginRequirement,
-)
-
-
-def parse_response(response: str) -> Action:
-    if '```' not in response:
-        # unexpected response format, message back to user
-        return MessageAction(response)
-    thought = response.split('```')[0].strip()
-    action_str = response.split('```')[1].strip()
-    # handle send message to user function call in BrowserGym
-    for sub_action in action_str.split('\n'):
-        if 'send_msg_to_user(' in sub_action:
-            tree = ast.parse(sub_action)
-            args = tree.body[0].value.args  # type: ignore
-            return MessageAction(args[0].value)
-
-    return BrowseInteractiveAction(browser_actions=action_str, thought=thought)
-
-
-class BrowsingAgent(Agent):
-    VERSION = '1.0'
-    """
-    An agent that interacts with the browser.
-    """
-
-    sandbox_plugins: list[PluginRequirement] = []
-
-    def __init__(
-        self,
-        llm: LLM,
-    ) -> None:
-        """
-        Initializes a new instance of the BrowsingAgent class.
-
-        Parameters:
-        - llm (LLM): The llm to be used by this agent
-        """
-        super().__init__(llm)
-        self.action_space = HighLevelActionSet(
-            # see https://github.com/ServiceNow/BrowserGym/blob/main/core/src/browsergym/core/action/highlevel.py for more details
-            subsets=[
-                'chat',
-                'bid',
-                'nav',
-            ],  # define a configurable action space, with chat functionality, web navigation, and webpage grounding using accessibility tree and HTML.
-            strict=False,  # less strict on the parsing of the actions
-            multiaction=True,  # enable to agent to take multiple actions at once
-        )
-
-        self.reset()
-
-    def reset(self) -> None:
-        """
-        Resets the Browsing Agent.
-        """
-        super().reset()
-        self.cost_accumulator = 0
-
-    def step(self, state: State) -> Action:
-        """
-        Performs one step using the Browsing Agent.
-        This includes gathering information on previous steps and prompting the model to make a browsing command to execute.
-
-        Parameters:
-        - state (State): used to get updated info
-
-        Returns:
-        - BrowseInteractiveAction(browsergym_command) - BrowserGym commands to run
-        - MessageAction(content) - Message action to run (e.g. ask for clarification)
-        - AgentFinishAction() - end the interaction
-        """
-        goal = state.get_current_user_intent()
-        messages = []
-        prev_actions = ''
-        cur_axtree_txt = ''
-        error_prefix = ''
-        last_obs = None
-        for prev_action, obs in state.history:
-            if isinstance(prev_action, BrowseInteractiveAction):
-                prev_actions += f'{prev_action.browser_actions}\n'
-                last_obs = obs
-            elif (
-                isinstance(prev_action, MessageAction) and prev_action.source != 'user'
-            ):
-                # agent has responded, task finish.
-                return AgentFinishAction()
-
-        if isinstance(last_obs, BrowserOutputObservation):
-            if last_obs.error:
-                # add error recovery prompt prefix
-                error_prefix = f'IMPORTANT! Last action is incorrect:\n{last_obs.last_browser_action}\nThink again with the current observation of the page.\n'
-            cur_axtree_txt = flatten_axtree_to_str(last_obs.axtree_object)
-
-        system_msg = f"""\
-# Instructions
-Review the current state of the page and all other information to find the best
-possible next action to accomplish your goal. Your answer will be interpreted
-and executed by a program, make sure to follow the formatting instructions.
-
-# Goal:
-{goal}
-
-# Action Space
-{self.action_space.describe(with_long_description=False, with_examples=True)}
-"""
-
-        messages.append({'role': 'system', 'content': system_msg})
-
-        prompt = f"""\
-{error_prefix}
-
-# Current Accessibility Tree:
-{cur_axtree_txt}
-
-# Previous Actions
-{prev_actions}
-
-Here is an example with chain of thought of a valid action when clicking on a button:
-"
-In order to accomplish my goal I need to click on the button with bid 12
-```click("12")```
-"
-""".strip()
-        messages.append({'role': 'user', 'content': prompt})
-        response = self.llm.completion(
-            messages=messages,
-            temperature=0.0,
-        )
-        self.log_cost(response)
-        action_resp = response['choices'][0]['message']['content']
-        logger.info(prompt)
-        logger.info(action_resp)
-        return parse_response(action_resp)
-
-    def search_memory(self, query: str) -> list[str]:
-        raise NotImplementedError('Implement this abstract method')
-
-    def log_cost(self, response):
-        # TODO: refactor to unified cost tracking
-        try:
-            cur_cost = self.llm.completion_cost(response)
-        except Exception:
-            cur_cost = 0
-        self.cost_accumulator += cur_cost
-        logger.info(
-            'Cost: %.2f USD | Accumulated Cost: %.2f USD',
-            cur_cost,
-            self.cost_accumulator,
-        )

agenthub/browsing_agent/prompt.py

@@ -1,785 +0,0 @@
-import abc
-import difflib
-import logging
-import platform
-from copy import deepcopy
-from dataclasses import asdict, dataclass
-from textwrap import dedent
-from typing import Literal, Union
-from warnings import warn
-
-from browsergym.core.action.base import AbstractActionSet
-from browsergym.core.action.highlevel import HighLevelActionSet
-from browsergym.core.action.python import PythonActionSet
-
-from opendevin.runtime.browser.browser_env import BrowserEnv
-
-from .utils import (
-    ParseError,
-    parse_html_tags_raise,
-)
-
-
-@dataclass
-class Flags:
-    use_html: bool = True
-    use_ax_tree: bool = False
-    drop_ax_tree_first: bool = True  # This flag is no longer active TODO delete
-    use_thinking: bool = False
-    use_error_logs: bool = False
-    use_past_error_logs: bool = False
-    use_history: bool = False
-    use_action_history: bool = False
-    use_memory: bool = False
-    use_diff: bool = False
-    html_type: str = 'pruned_html'
-    use_concrete_example: bool = True
-    use_abstract_example: bool = False
-    multi_actions: bool = False
-    action_space: Literal[
-        'python', 'bid', 'coord', 'bid+coord', 'bid+nav', 'coord+nav', 'bid+coord+nav'
-    ] = 'bid'
-    is_strict: bool = False
-    # This flag will be automatically disabled `if not chat_model_args.has_vision()`
-    use_screenshot: bool = True
-    enable_chat: bool = False
-    max_prompt_tokens: int = 100_000
-    extract_visible_tag: bool = False
-    extract_coords: Literal['False', 'center', 'box'] = 'False'
-    extract_visible_elements_only: bool = False
-    demo_mode: Literal['off', 'default', 'only_visible_elements'] = 'off'
-
-    def copy(self):
-        return deepcopy(self)
-
-    def asdict(self):
-        """Helper for JSON serializble requirement."""
-        return asdict(self)
-
-    @classmethod
-    def from_dict(self, flags_dict):
-        """Helper for JSON serializble requirement."""
-        if isinstance(flags_dict, Flags):
-            return flags_dict
-
-        if not isinstance(flags_dict, dict):
-            raise ValueError(
-                f'Unregcognized type for flags_dict of type {type(flags_dict)}.'
-            )
-        return Flags(**flags_dict)
-
-
-class PromptElement:
-    """Base class for all prompt elements. Prompt elements can be hidden.
-
-    Prompt elements are used to build the prompt. Use flags to control which
-    prompt elements are visible. We use class attributes as a convenient way
-    to implement static prompts, but feel free to override them with instance
-    attributes or @property decorator."""
-
-    _prompt = ''
-    _abstract_ex = ''
-    _concrete_ex = ''
-
-    def __init__(self, visible: bool = True) -> None:
-        """Prompt element that can be hidden.
-
-        Parameters
-        ----------
-        visible : bool, optional
-            Whether the prompt element should be visible, by default True. Can
-            be a callable that returns a bool. This is useful when a specific
-            flag changes during a shrink iteration.
-        """
-        self._visible = visible
-
-    @property
-    def prompt(self):
-        """Avoid overriding this method. Override _prompt instead."""
-        return self._hide(self._prompt)
-
-    @property
-    def abstract_ex(self):
-        """Useful when this prompt element is requesting an answer from the llm.
-        Provide an abstract example of the answer here. See Memory for an
-        example.
-
-        Avoid overriding this method. Override _abstract_ex instead
-        """
-        return self._hide(self._abstract_ex)
-
-    @property
-    def concrete_ex(self):
-        """Useful when this prompt element is requesting an answer from the llm.
-        Provide a concrete example of the answer here. See Memory for an
-        example.
-
-        Avoid overriding this method. Override _concrete_ex instead
-        """
-        return self._hide(self._concrete_ex)
-
-    @property
-    def is_visible(self):
-        """Handle the case where visible is a callable."""
-        visible = self._visible
-        if callable(visible):
-            visible = visible()
-        return visible
-
-    def _hide(self, value):
-        """Return value if visible is True, else return empty string."""
-        if self.is_visible:
-            return value
-        else:
-            return ''
-
-    def _parse_answer(self, text_answer) -> dict:
-        if self.is_visible:
-            return self._parse_answer(text_answer)
-        else:
-            return {}
-
-
-class Shrinkable(PromptElement, abc.ABC):
-    @abc.abstractmethod
-    def shrink(self) -> None:
-        """Implement shrinking of this prompt element.
-
-        You need to recursively call all shrinkable elements that are part of
-        this prompt. You can also implement a shriking startegy for this prompt.
-        Shrinking is can be called multiple times to progressively shrink the
-        prompt until it fits max_tokens. Default max shrink iterations is 20.
-        """
-        pass
-
-
-class Truncater(Shrinkable):
-    """A prompt element that can be truncated to fit the context length of the LLM.
-    Of course, it will be great that we never have to use the functionality here to `shrink()` the prompt.
-    Extend this class for prompt elements that can be truncated. Usually long observations such as AxTree or HTML.
-    """
-
-    def __init__(self, visible, shrink_speed=0.3, start_truncate_iteration=10):
-        super().__init__(visible=visible)
-        self.shrink_speed = shrink_speed  # the percentage shrinked in each iteration
-        self.start_truncate_iteration = (
-            start_truncate_iteration  # the iteration to start truncating
-        )
-        self.shrink_calls = 0
-        self.deleted_lines = 0
-
-    def shrink(self) -> None:
-        if self.is_visible and self.shrink_calls >= self.start_truncate_iteration:
-            # remove the fraction of _prompt
-            lines = self._prompt.splitlines()
-            new_line_count = int(len(lines) * (1 - self.shrink_speed))
-            self.deleted_lines += len(lines) - new_line_count
-            self._prompt = '\n'.join(lines[:new_line_count])
-            self._prompt += (
-                f'\n... Deleted {self.deleted_lines} lines to reduce prompt size.'
-            )
-
-        self.shrink_calls += 1
-
-
-def fit_tokens(
-    shrinkable: Shrinkable,
-    max_prompt_chars=None,
-    max_iterations=20,
-):
-    """Shrink a prompt element until it fits max_tokens.
-
-    Parameters
-    ----------
-    shrinkable : Shrinkable
-        The prompt element to shrink.
-    max_prompt_chars : int
-        The maximum number of chars allowed.
-    max_iterations : int, optional
-        The maximum number of shrink iterations, by default 20.
-    model_name : str, optional
-        The name of the model used when tokenizing.
-
-    Returns
-    -------
-    str : the prompt after shrinking.
-    """
-
-    if max_prompt_chars is None:
-        return shrinkable.prompt
-
-    for _ in range(max_iterations):
-        prompt = shrinkable.prompt
-        if isinstance(prompt, str):
-            prompt_str = prompt
-        elif isinstance(prompt, list):
-            prompt_str = '\n'.join([p['text'] for p in prompt if p['type'] == 'text'])
-        else:
-            raise ValueError(f'Unrecognized type for prompt: {type(prompt)}')
-        n_chars = len(prompt_str)
-        if n_chars <= max_prompt_chars:
-            return prompt
-        shrinkable.shrink()
-
-    logging.info(
-        dedent(
-            f"""\
-            After {max_iterations} shrink iterations, the prompt is still
-            {len(prompt_str)} chars (greater than {max_prompt_chars}). Returning the prompt as is."""
-        )
-    )
-    return prompt
-
-
-class HTML(Truncater):
-    def __init__(self, html, visible: bool = True, prefix='') -> None:
-        super().__init__(visible=visible, start_truncate_iteration=5)
-        self._prompt = f'\n{prefix}HTML:\n{html}\n'
-
-
-class AXTree(Truncater):
-    def __init__(
-        self, ax_tree, visible: bool = True, coord_type=None, prefix=''
-    ) -> None:
-        super().__init__(visible=visible, start_truncate_iteration=10)
-        if coord_type == 'center':
-            coord_note = """\
-Note: center coordinates are provided in parenthesis and are
-  relative to the top left corner of the page.\n\n"""
-        elif coord_type == 'box':
-            coord_note = """\
-Note: bounding box of each object are provided in parenthesis and are
-  relative to the top left corner of the page.\n\n"""
-        else:
-            coord_note = ''
-        self._prompt = f'\n{prefix}AXTree:\n{coord_note}{ax_tree}\n'
-
-
-class Error(PromptElement):
-    def __init__(self, error, visible: bool = True, prefix='') -> None:
-        super().__init__(visible=visible)
-        self._prompt = f'\n{prefix}Error from previous action:\n{error}\n'
-
-
-class Observation(Shrinkable):
-    """Observation of the current step.
-
-    Contains the html, the accessibility tree and the error logs.
-    """
-
-    def __init__(self, obs, flags: Flags) -> None:
-        super().__init__()
-        self.flags = flags
-        self.obs = obs
-        self.html = HTML(obs[flags.html_type], visible=flags.use_html, prefix='## ')
-        self.ax_tree = AXTree(
-            obs['axtree_txt'],
-            visible=flags.use_ax_tree,
-            coord_type=flags.extract_coords,
-            prefix='## ',
-        )
-        self.error = Error(
-            obs['last_action_error'],
-            visible=flags.use_error_logs and obs['last_action_error'],
-            prefix='## ',
-        )
-
-    def shrink(self):
-        self.ax_tree.shrink()
-        self.html.shrink()
-
-    @property
-    def _prompt(self) -> str:  # type: ignore
-        return f'\n# Observation of current step:\n{self.html.prompt}{self.ax_tree.prompt}{self.error.prompt}\n\n'
-
-    def add_screenshot(self, prompt):
-        if self.flags.use_screenshot:
-            if isinstance(prompt, str):
-                prompt = [{'type': 'text', 'text': prompt}]
-            img_url = BrowserEnv.image_to_jpg_base64_url(
-                self.obs['screenshot'], add_data_prefix=True
-            )
-            prompt.append({'type': 'image_url', 'image_url': img_url})
-
-        return prompt
-
-
-class MacNote(PromptElement):
-    def __init__(self) -> None:
-        super().__init__(visible=platform.system() == 'Darwin')
-        self._prompt = '\nNote: you are on mac so you should use Meta instead of Control for Control+C etc.\n'
-
-
-class BeCautious(PromptElement):
-    def __init__(self, visible: bool = True) -> None:
-        super().__init__(visible=visible)
-        self._prompt = """\
-\nBe very cautious. Avoid submitting anything before verifying the effect of your
-actions. Take the time to explore the effect of safe actions first. For example
-you can fill a few elements of a form, but don't click submit before verifying
-that everything was filled correctly.\n"""
-
-
-class GoalInstructions(PromptElement):
-    def __init__(self, goal, visible: bool = True) -> None:
-        super().__init__(visible)
-        self._prompt = f"""\
-# Instructions
-Review the current state of the page and all other information to find the best
-possible next action to accomplish your goal. Your answer will be interpreted
-and executed by a program, make sure to follow the formatting instructions.
-
-## Goal:
-{goal}
-"""
-
-
-class ChatInstructions(PromptElement):
-    def __init__(self, chat_messages, visible: bool = True) -> None:
-        super().__init__(visible)
-        self._prompt = """\
-# Instructions
-
-You are a UI Assistant, your goal is to help the user perform tasks using a web browser. You can
-communicate with the user via a chat, in which the user gives you instructions and in which you
-can send back messages. You have access to a web browser that both you and the user can see,
-and with which only you can interact via specific commands.
-
-Review the instructions from the user, the current state of the page and all other information
-to find the best possible next action to accomplish your goal. Your answer will be interpreted
-and executed by a program, make sure to follow the formatting instructions.
-
-## Chat messages:
-
-"""
-        self._prompt += '\n'.join(
-            [
-                f"""\
- - [{msg['role']}] {msg['message']}"""
-                for msg in chat_messages
-            ]
-        )
-
-
-class SystemPrompt(PromptElement):
-    _prompt = """\
-You are an agent trying to solve a web task based on the content of the page and
-a user instructions. You can interact with the page and explore. Each time you
-submit an action it will be sent to the browser and you will receive a new page."""
-
-
-class MainPrompt(Shrinkable):
-    def __init__(
-        self,
-        obs_history,
-        actions,
-        memories,
-        thoughts,
-        flags: Flags,
-    ) -> None:
-        super().__init__()
-        self.flags = flags
-        self.history = History(obs_history, actions, memories, thoughts, flags)
-        if self.flags.enable_chat:
-            self.instructions: Union[ChatInstructions, GoalInstructions] = (
-                ChatInstructions(obs_history[-1]['chat_messages'])
-            )
-        else:
-            if (
-                'chat_messages' in obs_history[-1]
-                and sum(
-                    [msg['role'] == 'user' for msg in obs_history[-1]['chat_messages']]
-                )
-                > 1
-            ):
-                logging.warning(
-                    'Agent is in goal mode, but multiple user messages are present in the chat. Consider switching to `enable_chat=True`.'
-                )
-            self.instructions = GoalInstructions(obs_history[-1]['goal'])
-
-        self.obs = Observation(obs_history[-1], self.flags)
-        self.action_space = ActionSpace(self.flags)
-
-        self.think = Think(visible=flags.use_thinking)
-        self.memory = Memory(visible=flags.use_memory)
-
-    @property
-    def _prompt(self) -> str:  # type: ignore
-        prompt = f"""\
-{self.instructions.prompt}\
-{self.obs.prompt}\
-{self.history.prompt}\
-{self.action_space.prompt}\
-{self.think.prompt}\
-{self.memory.prompt}\
-"""
-
-        if self.flags.use_abstract_example:
-            prompt += f"""
-# Abstract Example
-
-Here is an abstract version of the answer with description of the content of
-each tag. Make sure you follow this structure, but replace the content with your
-answer:
-{self.think.abstract_ex}\
-{self.memory.abstract_ex}\
-{self.action_space.abstract_ex}\
-"""
-
-        if self.flags.use_concrete_example:
-            prompt += f"""
-# Concrete Example
-
-Here is a concrete example of how to format your answer.
-Make sure to follow the template with proper tags:
-{self.think.concrete_ex}\
-{self.memory.concrete_ex}\
-{self.action_space.concrete_ex}\
-"""
-        return self.obs.add_screenshot(prompt)
-
-    def shrink(self):
-        self.history.shrink()
-        self.obs.shrink()
-
-    def _parse_answer(self, text_answer):
-        ans_dict = {}
-        ans_dict.update(self.think._parse_answer(text_answer))
-        ans_dict.update(self.memory._parse_answer(text_answer))
-        ans_dict.update(self.action_space._parse_answer(text_answer))
-        return ans_dict
-
-
-class ActionSpace(PromptElement):
-    def __init__(self, flags: Flags) -> None:
-        super().__init__()
-        self.flags = flags
-        self.action_space = _get_action_space(flags)
-
-        self._prompt = (
-            f'# Action space:\n{self.action_space.describe()}{MacNote().prompt}\n'
-        )
-        self._abstract_ex = f"""
-<action>
-{self.action_space.example_action(abstract=True)}
-</action>
-"""
-        self._concrete_ex = f"""
-<action>
-{self.action_space.example_action(abstract=False)}
-</action>
-"""
-
-    def _parse_answer(self, text_answer):
-        ans_dict = parse_html_tags_raise(
-            text_answer, keys=['action'], merge_multiple=True
-        )
-
-        try:
-            # just check if action can be mapped to python code but keep action as is
-            # the environment will be responsible for mapping it to python
-            self.action_space.to_python_code(ans_dict['action'])
-        except Exception as e:
-            raise ParseError(
-                f'Error while parsing action\n: {e}\n'
-                'Make sure your answer is restricted to the allowed actions.'
-            )
-
-        return ans_dict
-
-
-def _get_action_space(flags: Flags) -> AbstractActionSet:
-    match flags.action_space:
-        case 'python':
-            action_space = PythonActionSet(strict=flags.is_strict)
-            if flags.multi_actions:
-                warn(
-                    f'Flag action_space={repr(flags.action_space)} incompatible with multi_actions={repr(flags.multi_actions)}.'
-                )
-            if flags.demo_mode != 'off':
-                warn(
-                    f'Flag action_space={repr(flags.action_space)} incompatible with demo_mode={repr(flags.demo_mode)}.'
-                )
-            return action_space
-        case 'bid':
-            action_subsets = ['chat', 'bid']
-        case 'coord':
-            action_subsets = ['chat', 'coord']
-        case 'bid+coord':
-            action_subsets = ['chat', 'bid', 'coord']
-        case 'bid+nav':
-            action_subsets = ['chat', 'bid', 'nav']
-        case 'coord+nav':
-            action_subsets = ['chat', 'coord', 'nav']
-        case 'bid+coord+nav':
-            action_subsets = ['chat', 'bid', 'coord', 'nav']
-        case _:
-            raise NotImplementedError(
-                f'Unknown action_space {repr(flags.action_space)}'
-            )
-
-    action_space = HighLevelActionSet(
-        subsets=action_subsets,
-        multiaction=flags.multi_actions,
-        strict=flags.is_strict,
-        demo_mode=flags.demo_mode,
-    )
-
-    return action_space
-
-
-class Memory(PromptElement):
-    _prompt = ''  # provided in the abstract and concrete examples
-
-    _abstract_ex = """
-<memory>
-Write down anything you need to remember for next steps. You will be presented
-with the list of previous memories and past actions.
-</memory>
-"""
-
-    _concrete_ex = """
-<memory>
-I clicked on bid 32 to activate tab 2. The accessibility tree should mention
-focusable for elements of the form at next step.
-</memory>
-"""
-
-    def _parse_answer(self, text_answer):
-        return parse_html_tags_raise(
-            text_answer, optional_keys=['memory'], merge_multiple=True
-        )
-
-
-class Think(PromptElement):
-    _prompt = ''
-
-    _abstract_ex = """
-<think>
-Think step by step. If you need to make calculations such as coordinates, write them here. Describe the effect
-that your previous action had on the current content of the page.
-</think>
-"""
-    _concrete_ex = """
-<think>
-My memory says that I filled the first name and last name, but I can't see any
-content in the form. I need to explore different ways to fill the form. Perhaps
-the form is not visible yet or some fields are disabled. I need to replan.
-</think>
-"""
-
-    def _parse_answer(self, text_answer):
-        return parse_html_tags_raise(
-            text_answer, optional_keys=['think'], merge_multiple=True
-        )
-
-
-def diff(previous, new):
-    """Return a string showing the difference between original and new.
-
-    If the difference is above diff_threshold, return the diff string."""
-
-    if previous == new:
-        return 'Identical', []
-
-    if len(previous) == 0 or previous is None:
-        return 'previous is empty', []
-
-    diff_gen = difflib.ndiff(previous.splitlines(), new.splitlines())
-
-    diff_lines = []
-    plus_count = 0
-    minus_count = 0
-    for line in diff_gen:
-        if line.strip().startswith('+'):
-            diff_lines.append(line)
-            plus_count += 1
-        elif line.strip().startswith('-'):
-            diff_lines.append(line)
-            minus_count += 1
-        else:
-            continue
-
-    header = f'{plus_count} lines added and {minus_count} lines removed:'
-
-    return header, diff_lines
-
-
-class Diff(Shrinkable):
-    def __init__(
-        self, previous, new, prefix='', max_line_diff=20, shrink_speed=2, visible=True
-    ) -> None:
-        super().__init__(visible=visible)
-        self.max_line_diff = max_line_diff
-        self.header, self.diff_lines = diff(previous, new)
-        self.shrink_speed = shrink_speed
-        self.prefix = prefix
-
-    def shrink(self):
-        self.max_line_diff -= self.shrink_speed
-        self.max_line_diff = max(1, self.max_line_diff)
-
-    @property
-    def _prompt(self) -> str:  # type: ignore
-        diff_str = '\n'.join(self.diff_lines[: self.max_line_diff])
-        if len(self.diff_lines) > self.max_line_diff:
-            original_count = len(self.diff_lines)
-            diff_str = f'{diff_str}\nDiff truncated, {original_count - self.max_line_diff} changes now shown.'
-        return f'{self.prefix}{self.header}\n{diff_str}\n'
-
-
-class HistoryStep(Shrinkable):
-    def __init__(
-        self, previous_obs, current_obs, action, memory, flags: Flags, shrink_speed=1
-    ) -> None:
-        super().__init__()
-        self.html_diff = Diff(
-            previous_obs[flags.html_type],
-            current_obs[flags.html_type],
-            prefix='\n### HTML diff:\n',
-            shrink_speed=shrink_speed,
-            visible=lambda: flags.use_html and flags.use_diff,
-        )
-        self.ax_tree_diff = Diff(
-            previous_obs['axtree_txt'],
-            current_obs['axtree_txt'],
-            prefix='\n### Accessibility tree diff:\n',
-            shrink_speed=shrink_speed,
-            visible=lambda: flags.use_ax_tree and flags.use_diff,
-        )
-        self.error = Error(
-            current_obs['last_action_error'],
-            visible=(
-                flags.use_error_logs
-                and current_obs['last_action_error']
-                and flags.use_past_error_logs
-            ),
-            prefix='### ',
-        )
-        self.shrink_speed = shrink_speed
-        self.action = action
-        self.memory = memory
-        self.flags = flags
-
-    def shrink(self):
-        super().shrink()
-        self.html_diff.shrink()
-        self.ax_tree_diff.shrink()
-
-    @property
-    def _prompt(self) -> str:  # type: ignore
-        prompt = ''
-
-        if self.flags.use_action_history:
-            prompt += f'\n### Action:\n{self.action}\n'
-
-        prompt += (
-            f'{self.error.prompt}{self.html_diff.prompt}{self.ax_tree_diff.prompt}'
-        )
-
-        if self.flags.use_memory and self.memory is not None:
-            prompt += f'\n### Memory:\n{self.memory}\n'
-
-        return prompt
-
-
-class History(Shrinkable):
-    def __init__(
-        self, history_obs, actions, memories, thoughts, flags: Flags, shrink_speed=1
-    ) -> None:
-        super().__init__(visible=flags.use_history)
-        assert len(history_obs) == len(actions) + 1
-        assert len(history_obs) == len(memories) + 1
-
-        self.shrink_speed = shrink_speed
-        self.history_steps: list[HistoryStep] = []
-
-        for i in range(1, len(history_obs)):
-            self.history_steps.append(
-                HistoryStep(
-                    history_obs[i - 1],
-                    history_obs[i],
-                    actions[i - 1],
-                    memories[i - 1],
-                    flags,
-                )
-            )
-
-    def shrink(self):
-        """Shrink individual steps"""
-        # TODO set the shrink speed of older steps to be higher
-        super().shrink()
-        for step in self.history_steps:
-            step.shrink()
-
-    @property
-    def _prompt(self):
-        prompts = ['# History of interaction with the task:\n']
-        for i, step in enumerate(self.history_steps):
-            prompts.append(f'## step {i}')
-            prompts.append(step.prompt)
-        return '\n'.join(prompts) + '\n'
-
-
-if __name__ == '__main__':
-    html_template = """
-    <html>
-    <body>
-    <div>
-    Hello World.
-    Step {}.
-    </div>
-    </body>
-    </html>
-    """
-
-    OBS_HISTORY = [
-        {
-            'goal': 'do this and that',
-            'pruned_html': html_template.format(1),
-            'axtree_txt': '[1] Click me',
-            'last_action_error': '',
-        },
-        {
-            'goal': 'do this and that',
-            'pruned_html': html_template.format(2),
-            'axtree_txt': '[1] Click me',
-            'last_action_error': '',
-        },
-        {
-            'goal': 'do this and that',
-            'pruned_html': html_template.format(3),
-            'axtree_txt': '[1] Click me',
-            'last_action_error': 'Hey, there is an error now',
-        },
-    ]
-    ACTIONS = ["click('41')", "click('42')"]
-    MEMORIES = ['memory A', 'memory B']
-    THOUGHTS = ['thought A', 'thought B']
-
-    flags = Flags(
-        use_html=True,
-        use_ax_tree=True,
-        use_thinking=True,
-        use_error_logs=True,
-        use_past_error_logs=True,
-        use_history=True,
-        use_action_history=True,
-        use_memory=True,
-        use_diff=True,
-        html_type='pruned_html',
-        use_concrete_example=True,
-        use_abstract_example=True,
-        use_screenshot=False,
-        multi_actions=True,
-    )
-
-    print(
-        MainPrompt(
-            obs_history=OBS_HISTORY,
-            actions=ACTIONS,
-            memories=MEMORIES,
-            thoughts=THOUGHTS,
-            flags=flags,
-        ).prompt
-    )

agenthub/browsing_agent/utils.py

@@ -1,160 +0,0 @@
-import collections
-import re
-from warnings import warn
-
-import yaml
-
-
-def yaml_parser(message):
-    """Parse a yaml message for the retry function."""
-
-    # saves gpt-3.5 from some yaml parsing errors
-    message = re.sub(r':\s*\n(?=\S|\n)', ': ', message)
-
-    try:
-        value = yaml.safe_load(message)
-        valid = True
-        retry_message = ''
-    except yaml.YAMLError as e:
-        warn(str(e))
-        value = {}
-        valid = False
-        retry_message = "Your response is not a valid yaml. Please try again and be careful to the format. Don't add any apology or comment, just the answer."
-    return value, valid, retry_message
-
-
-def _compress_chunks(text, identifier, skip_list, split_regex='\n\n+'):
-    """Compress a string by replacing redundant chunks by identifiers. Chunks are defined by the split_regex."""
-    text_list = re.split(split_regex, text)
-    text_list = [chunk.strip() for chunk in text_list]
-    counter = collections.Counter(text_list)
-    def_dict = {}
-    id = 0
-
-    # Store items that occur more than once in a dictionary
-    for item, count in counter.items():
-        if count > 1 and item not in skip_list and len(item) > 10:
-            def_dict[f'{identifier}-{id}'] = item
-            id += 1
-
-    # Replace redundant items with their identifiers in the text
-    compressed_text = '\n'.join(text_list)
-    for key, value in def_dict.items():
-        compressed_text = compressed_text.replace(value, key)
-
-    return def_dict, compressed_text
-
-
-def compress_string(text):
-    """Compress a string by replacing redundant paragraphs and lines with identifiers."""
-
-    # Perform paragraph-level compression
-    def_dict, compressed_text = _compress_chunks(
-        text, identifier='§', skip_list=[], split_regex='\n\n+'
-    )
-
-    # Perform line-level compression, skipping any paragraph identifiers
-    line_dict, compressed_text = _compress_chunks(
-        compressed_text, '¶', list(def_dict.keys()), split_regex='\n+'
-    )
-    def_dict.update(line_dict)
-
-    # Create a definitions section
-    def_lines = ['<definitions>']
-    for key, value in def_dict.items():
-        def_lines.append(f'{key}:\n{value}')
-    def_lines.append('</definitions>')
-    definitions = '\n'.join(def_lines)
-
-    return definitions + '\n' + compressed_text
-
-
-def extract_html_tags(text, keys):
-    """Extract the content within HTML tags for a list of keys.
-
-    Parameters
-    ----------
-    text : str
-        The input string containing the HTML tags.
-    keys : list of str
-        The HTML tags to extract the content from.
-
-    Returns
-    -------
-    dict
-        A dictionary mapping each key to a list of subset in `text` that match the key.
-
-    Notes
-    -----
-    All text and keys will be converted to lowercase before matching.
-
-    """
-    content_dict = {}
-    # text = text.lower()
-    # keys = set([k.lower() for k in keys])
-    for key in keys:
-        pattern = f'<{key}>(.*?)</{key}>'
-        matches = re.findall(pattern, text, re.DOTALL)
-        if matches:
-            content_dict[key] = [match.strip() for match in matches]
-    return content_dict
-
-
-class ParseError(Exception):
-    pass
-
-
-def parse_html_tags_raise(text, keys=(), optional_keys=(), merge_multiple=False):
-    """A version of parse_html_tags that raises an exception if the parsing is not successful."""
-    content_dict, valid, retry_message = parse_html_tags(
-        text, keys, optional_keys, merge_multiple=merge_multiple
-    )
-    if not valid:
-        raise ParseError(retry_message)
-    return content_dict
-
-
-def parse_html_tags(text, keys=(), optional_keys=(), merge_multiple=False):
-    """Satisfy the parse api, extracts 1 match per key and validates that all keys are present
-
-    Parameters
-    ----------
-    text : str
-        The input string containing the HTML tags.
-    keys : list of str
-        The HTML tags to extract the content from.
-    optional_keys : list of str
-        The HTML tags to extract the content from, but are optional.
-
-    Returns
-    -------
-    dict
-        A dictionary mapping each key to subset of `text` that match the key.
-    bool
-        Whether the parsing was successful.
-    str
-        A message to be displayed to the agent if the parsing was not successful.
-    """
-    all_keys = tuple(keys) + tuple(optional_keys)
-    content_dict = extract_html_tags(text, all_keys)
-    retry_messages = []
-
-    for key in all_keys:
-        if key not in content_dict:
-            if key not in optional_keys:
-                retry_messages.append(f'Missing the key <{key}> in the answer.')
-        else:
-            val = content_dict[key]
-            content_dict[key] = val[0]
-            if len(val) > 1:
-                if not merge_multiple:
-                    retry_messages.append(
-                        f'Found multiple instances of the key {key}. You should have only one of them.'
-                    )
-                else:
-                    # merge the multiple instances
-                    content_dict[key] = '\n'.join(val)
-
-    valid = len(retry_messages) == 0
-    retry_message = '\n'.join(retry_messages)
-    return content_dict, valid, retry_message

agenthub/codeact_agent/README.md

@@ -1,6 +1,6 @@
 # CodeAct Agent Framework
 
-This folder implements the CodeAct idea ([paper](https://arxiv.org/abs/2402.01030), [tweet](https://twitter.com/xingyaow_/status/1754556835703751087)) that consolidates LLM agents’ **act**ions into a unified **code** action space for both *simplicity* and *performance* (see paper for more details).
+This folder implements the CodeAct idea ([paper](https://arxiv.org/abs/2402.13463), [tweet](https://twitter.com/xingyaow_/status/1754556835703751087)) that consolidates LLM agents’ **act**ions into a unified **code** action space for both *simplicity* and *performance* (see paper for more details).
 
 The conceptual idea is illustrated below. At each turn, the agent can:
 

agenthub/codeact_agent/codeact_agent.py

@@ -9,6 +9,7 @@ from agenthub.codeact_agent.prompt import (
 )
 from opendevin.controller.agent import Agent
 from opendevin.controller.state.state import State
+from opendevin.core.logger import opendevin_logger as logger
 from opendevin.events.action import (
     Action,
     AgentFinishAction,
@@ -24,9 +25,9 @@ from opendevin.events.observation import (
 )
 from opendevin.llm.llm import LLM
 from opendevin.runtime.plugins import (
-    AgentSkillsRequirement,
     JupyterRequirement,
     PluginRequirement,
+    SWEAgentCommandsRequirement,
 )
 
 ENABLE_GITHUB = True
@@ -40,57 +41,6 @@ def parse_response(response) -> str:
     return action
 
 
-def action_to_str(action: Action) -> str:
-    if isinstance(action, CmdRunAction):
-        return f'{action.thought}\n<execute_bash>\n{action.command}\n</execute_bash>'
-    elif isinstance(action, IPythonRunCellAction):
-        return f'{action.thought}\n<execute_ipython>\n{action.code}\n</execute_ipython>'
-    elif isinstance(action, BrowseInteractiveAction):
-        return f'{action.thought}\n<execute_browse>\n{action.browser_actions}\n</execute_browse>'
-    elif isinstance(action, MessageAction):
-        return action.content
-    return ''
-
-
-def get_action_message(action: Action) -> dict[str, str] | None:
-    if (
-        isinstance(action, BrowseInteractiveAction)
-        or isinstance(action, CmdRunAction)
-        or isinstance(action, IPythonRunCellAction)
-        or isinstance(action, MessageAction)
-    ):
-        return {
-            'role': 'user' if action.source == 'user' else 'assistant',
-            'content': action_to_str(action),
-        }
-    return None
-
-
-def get_observation_message(obs) -> dict[str, str] | None:
-    if isinstance(obs, CmdOutputObservation):
-        content = 'OBSERVATION:\n' + truncate_observation(obs.content)
-        content += (
-            f'\n[Command {obs.command_id} finished with exit code {obs.exit_code}]]'
-        )
-        return {'role': 'user', 'content': content}
-    elif isinstance(obs, IPythonRunCellObservation):
-        content = 'OBSERVATION:\n' + obs.content
-        # replace base64 images with a placeholder
-        splitted = content.split('\n')
-        for i, line in enumerate(splitted):
-            if '![image](data:image/png;base64,' in line:
-                splitted[i] = (
-                    '![image](data:image/png;base64, ...) already displayed to user'
-                )
-        content = '\n'.join(splitted)
-        content = truncate_observation(content)
-        return {'role': 'user', 'content': content}
-    elif isinstance(obs, BrowserOutputObservation):
-        content = 'OBSERVATION:\n' + truncate_observation(obs.content)
-        return {'role': 'user', 'content': content}
-    return None
-
-
 def truncate_observation(observation: str, max_chars: int = 10_000) -> str:
     """
     Truncate the middle of the observation if it is too long.
@@ -105,20 +55,39 @@ def truncate_observation(observation: str, max_chars: int = 10_000) -> str:
     )
 
 
-# FIXME: We can tweak these two settings to create MicroAgents specialized toward different area
-def get_system_message() -> str:
-    if ENABLE_GITHUB:
-        return f'{SYSTEM_PREFIX}\n{GITHUB_MESSAGE}\n\n{COMMAND_DOCS}\n\n{SYSTEM_SUFFIX}'
-    else:
-        return f'{SYSTEM_PREFIX}\n\n{COMMAND_DOCS}\n\n{SYSTEM_SUFFIX}'
+def swe_agent_edit_hack(bash_command: str) -> str:
+    """
+    Hack to handle the SWE-agent edit command. The vanilla edit command will hang the SSHBox.
 
+    REPLACE THIS:
+    edit 683:693
+            try:
+                return list(urlsplit(url))
+            except ValueError:
+                raise ValidationError(self.error_messages['invalid'], code='invalid')
+    end_of_edit
 
-def get_in_context_example() -> str:
-    return EXAMPLES
+    WITH THIS:
+    edit 683:693 <<EOF
+            try:
+                return list(urlsplit(url))
+            except ValueError:
+                raise ValidationError(self.error_messages['invalid'], code='invalid')
+    EOF
+    """
+    if 'edit' in bash_command:
+        # edit\s(\d+):(\d+)([\s\S]*)end_of_edit
+        # replace
+        bash_command = re.sub(
+            r'edit\s(\d+):(\d+)([\s\S]*?)end_of_edit',
+            r'edit \1:\2 <<EOF\3EOF',
+            bash_command,
+        )
+    return bash_command
 
 
 class CodeActAgent(Agent):
-    VERSION = '1.5'
+    VERSION = '1.3'
     """
     The Code Act Agent is a minimalist agent.
     The agent works by passing the model a list of action-observation pairs and prompting the model to take the next step.
@@ -156,16 +125,15 @@ class CodeActAgent(Agent):
     """
 
     sandbox_plugins: list[PluginRequirement] = [
-        # NOTE: AgentSkillsRequirement need to go before JupyterRequirement, since
-        # AgentSkillsRequirement provides a lot of Python functions
-        # and it need to be initialized before Jupyter for Jupyter to use those functions.
-        AgentSkillsRequirement(),
         JupyterRequirement(),
+        SWEAgentCommandsRequirement(),
     ]
-    jupyter_kernel_init_code: str = 'from agentskills import *'
 
-    system_message: str = get_system_message()
-    in_context_example: str = f"Here is an example of how you can interact with the environment for task solving:\n{get_in_context_example()}\n\nNOW, LET'S START!"
+    system_message: str = (
+        f'{SYSTEM_PREFIX}\n{GITHUB_MESSAGE}\n\n{COMMAND_DOCS}\n\n{SYSTEM_SUFFIX}'
+        if ENABLE_GITHUB
+        else f'{SYSTEM_PREFIX}\n\n{COMMAND_DOCS}\n\n{SYSTEM_SUFFIX}'
+    )
 
     def __init__(
         self,
@@ -185,6 +153,14 @@ class CodeActAgent(Agent):
         Resets the CodeAct Agent.
         """
         super().reset()
+        self.messages: list[dict[str, str]] = [
+            {'role': 'system', 'content': self.system_message},
+            {
+                'role': 'user',
+                'content': f"Here is an example of how you can interact with the environment for task solving:\n{EXAMPLES}\n\nNOW, LET'S START!",
+            },
+        ]
+        self.cost_accumulator = 0
 
     def step(self, state: State) -> Action:
         """
@@ -201,30 +177,49 @@ class CodeActAgent(Agent):
         - MessageAction(content) - Message action to run (e.g. ask for clarification)
         - AgentFinishAction() - end the interaction
         """
-        messages: list[dict[str, str]] = [
-            {'role': 'system', 'content': self.system_message},
-            {'role': 'user', 'content': self.in_context_example},
-        ]
 
-        for prev_action, obs in state.history:
-            action_message = get_action_message(prev_action)
-            if action_message:
-                messages.append(action_message)
+        updated_info = state.updated_info
+        if updated_info:
+            for prev_action, obs in updated_info:
+                if (
+                    isinstance(prev_action, MessageAction)
+                    and prev_action.source == 'user'
+                ):
+                    self.messages.append(
+                        {'role': 'user', 'content': prev_action.content}
+                    )
+                    if prev_action.content.strip() == '/exit':
+                        # User wants to exit
+                        return AgentFinishAction()
 
-            obs_message = get_observation_message(obs)
-            if obs_message:
-                messages.append(obs_message)
+                if isinstance(obs, CmdOutputObservation):
+                    content = 'OBSERVATION:\n' + truncate_observation(obs.content)
+                    content += f'\n[Command {obs.command_id} finished with exit code {obs.exit_code}]]'
+                    self.messages.append({'role': 'user', 'content': content})
+                elif isinstance(obs, IPythonRunCellObservation):
+                    content = 'OBSERVATION:\n' + obs.content
+                    # replace base64 images with a placeholder
+                    splitted = content.split('\n')
+                    for i, line in enumerate(splitted):
+                        if '![image](data:image/png;base64,' in line:
+                            splitted[i] = (
+                                '![image](data:image/png;base64, ...) already displayed to user'
+                            )
+                    content = '\n'.join(splitted)
+                    content = truncate_observation(content)
+                    self.messages.append({'role': 'user', 'content': content})
+                elif isinstance(obs, BrowserOutputObservation):
+                    content = 'OBSERVATION:\n' + truncate_observation(obs.content)
+                    self.messages.append({'role': 'user', 'content': content})
 
-        latest_user_message = [m for m in messages if m['role'] == 'user'][-1]
+        latest_user_message = [m for m in self.messages if m['role'] == 'user'][-1]
         if latest_user_message:
-            if latest_user_message['content'].strip() == '/exit':
-                return AgentFinishAction()
             latest_user_message['content'] += (
                 f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task.'
             )
 
-        response = self.llm.do_completion(
-            messages=messages,
+        response = self.llm.completion(
+            messages=self.messages,
             stop=[
                 '</execute_ipython>',
                 '</execute_bash>',
@@ -233,36 +228,36 @@ class CodeActAgent(Agent):
             temperature=0.0,
         )
 
+        self.log_cost(response)
+
         action_str: str = parse_response(response)
         state.num_of_chars += sum(
-            len(message['content']) for message in messages
+            len(message['content']) for message in self.messages
         ) + len(action_str)
+        self.messages.append({'role': 'assistant', 'content': action_str})
 
         if finish_command := re.search(r'<finish>.*</finish>', action_str, re.DOTALL):
             thought = action_str.replace(finish_command.group(0), '').strip()
             return AgentFinishAction(thought=thought)
         if bash_command := re.search(
-            r'<execute_bash>(.*?)</execute_bash>', action_str, re.DOTALL
+            r'<execute_bash>(.*)</execute_bash>', action_str, re.DOTALL
         ):
             # remove the command from the action string to get thought
             thought = action_str.replace(bash_command.group(0), '').strip()
             # a command was found
             command_group = bash_command.group(1).strip()
+            command_group = swe_agent_edit_hack(command_group)
 
             if command_group.strip() == 'exit':
                 return AgentFinishAction()
             return CmdRunAction(command=command_group, thought=thought)
         elif python_code := re.search(
-            r'<execute_ipython>(.*?)</execute_ipython>', action_str, re.DOTALL
+            r'<execute_ipython>(.*)</execute_ipython>', action_str, re.DOTALL
         ):
             # a code block was found
             code_group = python_code.group(1).strip()
             thought = action_str.replace(python_code.group(0), '').strip()
-            return IPythonRunCellAction(
-                code=code_group,
-                thought=thought,
-                kernel_init_code=self.jupyter_kernel_init_code,
-            )
+            return IPythonRunCellAction(code=code_group, thought=thought)
         elif browse_command := re.search(
             r'<execute_browse>(.*)</execute_browse>', action_str, re.DOTALL
         ):
@@ -279,3 +274,15 @@ class CodeActAgent(Agent):
 
     def search_memory(self, query: str) -> list[str]:
         raise NotImplementedError('Implement this abstract method')
+
+    def log_cost(self, response):
+        try:
+            cur_cost = self.llm.completion_cost(response)
+        except Exception:
+            cur_cost = 0
+        self.cost_accumulator += cur_cost
+        logger.info(
+            'Cost: %.2f USD | Accumulated Cost: %.2f USD',
+            cur_cost,
+            self.cost_accumulator,
+        )

agenthub/codeact_agent/prompt.py

@@ -1,64 +1,70 @@
-from opendevin.runtime.plugins import AgentSkillsRequirement
+from opendevin.runtime.plugins import SWEAgentCommandsRequirement
 
-_AGENT_SKILLS_DOCS = AgentSkillsRequirement.documentation
+_SWEAGENT_BASH_DOCS = '\n'.join(
+    filter(
+        lambda x: not x.startswith('submit'),
+        SWEAgentCommandsRequirement.documentation.split('\n'),
+    )
+)
+# _SWEAGENT_BASH_DOCS content below:
+"""
+open <path> [<line_number>] - opens the file at the given path in the editor. If line_number is provided, the window will be move to include that line
+goto <line_number> - moves the window to show <line_number>
+scroll_down - moves the window down {WINDOW} lines
+scroll_up - moves the window down {WINDOW} lines
+create <filename> - creates and opens a new file with the given name
+search_dir <search_term> [<dir>] - searches for search_term in all files in dir. If dir is not provided, searches in the current directory
+search_file <search_term> [<file>] - searches for search_term in file. If file is not provided, searches in the current open file
+find_file <file_name> [<dir>] - finds all files with the given name in dir. If dir is not provided, searches in the current directory
+edit <start_line>:<end_line>
+<replacement_text>
+end_of_edit - replaces lines <start_line> through <end_line> (inclusive) with the given text in the open file. The replacement text is terminated by a line with only end_of_edit on it. All of the <replacement text> will be entered, so make sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same error message again. Remember, the file must be open before editing.
+"""
 
 COMMAND_DOCS = (
-    '\nApart from the standard Python library, the assistant can also use the following functions (already imported) in <execute_ipython> environment:\n'
-    f'{_AGENT_SKILLS_DOCS}'
-    "Please note that THE `edit_file` FUNCTION REQUIRES PROPER INDENTATION. If the assistant would like to add the line '        print(x)', it must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run."
+    '\nApart from the standard bash commands, you can also use the following special commands in <execute_bash> environment:\n'
+    f'{_SWEAGENT_BASH_DOCS}'
+    "Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. If you'd like to add the line '        print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run."
 )
 
-# ======= SYSTEM MESSAGE =======
-MINIMAL_SYSTEM_PREFIX = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
+SYSTEM_PREFIX = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
 The assistant can interact with an interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code should be enclosed using "<execute_ipython>" tag, for example:
 <execute_ipython>
 print("Hello World!")
 </execute_ipython>
 The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
 For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
-"""
-
-BROWSING_PREFIX = """The assistant can browse the Internet with commands on behalf of the user by wrapping them with <execute_browse> and </execute_browse>.
+The assistant can browse the Internet with commands on behalf of the user by wrapping them with <execute_browse> and </execute_browse>.
 For example, you can browse a given URL by <execute_browse> goto("<URL>") </execute_browse>.
 The assistant should attempt fewer things at a time instead of putting too much commands OR code in one "execute" block.
-"""
-PIP_INSTALL_PREFIX = """The assistant can install Python packages using the %pip magic command in an IPython environment by using the following syntax: <execute_ipython> %pip install [package needed] </execute_ipython> and should always import packages and define variables before starting to use them."""
+The assistant can install Python packages through bash by <execute_bash> pip install [package needed] </execute_bash> and should always import packages and define variables before starting to use them.
+The assistant should stop <execute> and provide an answer when they have already obtained the answer from the execution result.
+If the assistant encounters an import error in IPython for a newly installed package, they should try to restart the kernel and import the package again. IPython kernel can be re-started by:
+<execute_ipython>
+import IPython
+IPython.Application.instance().kernel.do_shutdown(True)  # Restart the kernel
+</execute_ipython>"""
 
-SYSTEM_PREFIX = MINIMAL_SYSTEM_PREFIX + BROWSING_PREFIX + PIP_INSTALL_PREFIX
-
-GITHUB_MESSAGE = """To do any activities on GitHub, the assistant should use the token in the $GITHUB_TOKEN environment variable.
-For instance, to push a local branch `my_branch` to the github repo `owner/repo`, the assistant can use the following four commands:
+GITHUB_MESSAGE = """To do any activities on GitHub, you should use the token in the $GITHUB_TOKEN environment variable.
+For instance, to push a local branch `my_branch` to the github repo `owner/repo`, you can use the following four commands:
 <execute_bash> git push https://$GITHUB_TOKEN@github.com/owner/repo.git my_branch </execute_bash>
-If the assistant require access to GitHub but $GITHUB_TOKEN is not set, ask the user to set it."""
+If you require access to GitHub but $GITHUB_TOKEN is not set, ask the user to set it for you."""
 
 SYSTEM_SUFFIX = """The assistant's response should be concise.
-The assistant should include ONLY ONE <execute_ipython> or <execute_bash> or <execute_browse> in every one of the responses, unless the assistant is finished with the task or need more input or action from the user in order to proceed.
+You should include <execute_ipython> or <execute_bash> or <execute_browse> in every one of your responses, unless you are finished with the task or need more input or action from the user in order to proceed.
 IMPORTANT: Whenever possible, execute the code for the user using <execute_ipython> or <execute_bash> or <execute_browse> instead of providing it.
 """
 
-
-# ======= EXAMPLE MESSAGE =======
 EXAMPLES = """
 --- START OF EXAMPLE ---
 
 USER: Can you create a list of numbers from 1 to 10, and create a web page to display them at port 5000?
 
 ASSISTANT:
-Sure! Let me create a file first:
+Sure! Let me write the Python code for starting a web server and save it to a file `app.py`:
 <execute_ipython>
-create_file('app.py')
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[File: /workspace/app.py (1 lines total)]
-1|
-[File app.py created.]
-
-ASSISTANT:
-Now I will write the Python code for starting a web server and save it to the file `app.py`:
-<execute_ipython>
-EDITED_CODE=\"\"\"from flask import Flask
+CODE='''
+from flask import Flask
 app = Flask(__name__)
 
 @app.route('/')
@@ -67,23 +73,15 @@ def index():
     return str(numbers)
 
 if __name__ == '__main__':
-    app.run(port=5000)\"\"\"
-edit_file(start=1, end=1, content=EDITED_CODE)
+    app.run(port=5000)
+'''
+with open('app.py', 'w') as f:
+    f.write(CODE)
 </execute_ipython>
 
 USER:
-OBSERVATION:
-1|from flask import Flask
-2|app = Flask(__name__)
-3|
-4|@app.route('/')
-5|def index():
-6|    numbers = list(range(1, 11))
-7|    return str(numbers)
-8|
-9|if __name__ == '__main__':
-10|    app.run(port=5000)
-[File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
+Observation:
+[Code executed successfully with no output]
 
 ASSISTANT:
 I have created a Python file `app.py` that will display a list of numbers from 1 to 10 when you run it. Let me run the Python file for you:
@@ -92,7 +90,7 @@ python3 app.py > server.log 2>&1 &
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 [1] 121[1]+  Exit 1                  python3 app.py > server.log 2>&1
 
 ASSISTANT: Looks like the server is running with PID 121 then crashed. Let me check the server log:
@@ -101,7 +99,7 @@ cat server.log
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 Traceback (most recent call last):
   File "/workspace/app.py", line 2, in <module>
     from flask import Flask
@@ -114,7 +112,7 @@ pip install flask
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 Defaulting to user installation because normal site-packages is not writeable
 Collecting flask
   Using cached flask-3.0.3-py3-none-any.whl (101 kB)
@@ -138,7 +136,7 @@ python3 app.py > server.log 2>&1 &
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 [1] 124
 
 ASSISTANT:
@@ -148,7 +146,7 @@ cat server.log
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 * Serving Flask app 'app'
  * Debug mode: off
 WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
@@ -167,7 +165,7 @@ goto("http://127.0.0.1:5000")
 </execute_browse>
 
 USER:
-OBSERVATION:
+Observation:
 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
 
 ASSISTANT:
@@ -182,48 +180,52 @@ kill 124
 </execute_bash>
 
 USER:
-OBSERVATION:
+Observation:
 [1]+  Terminated              python3 app.py > server.log 2>&1
 
 ASSISTANT:
 The server has been stopped. Let me open the Python file and modify it.
-<execute_ipython>
-open_file('app.py')
-</execute_ipython>
+<execute_bash>
+open app.py
+</execute_bash>
 
 USER:
-[File: /workspace/app.py (10 lines total)]
-1|from flask import Flask
-2|app = Flask(__name__)
-3|
-4|@app.route('/')
-5|def index():
-6|    numbers = list(range(1, 11))
-7|    return str(numbers)
-8|
-9|if __name__ == '__main__':
-10|    app.run(port=5000)
+[File: /workspace/app.py (11 lines total)]
+1:
+2:from flask import Flask
+3:app = Flask(__name__)
+4:
+5:@app.route('/')
+6:def index():
+7:    numbers = list(range(1, 11))
+8:    return str(numbers)
+9:
+10:if __name__ == '__main__':
+11:    app.run(port=5000)
 
 ASSISTANT:
 I should edit the file to display the numbers in a table format. I should include correct indentation. Let me update the file:
-<execute_ipython>
-edit_file(start=7, end=7, content="    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'")
-</execute_ipython>
+<execute_bash>
+edit 8:8
+    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
+end_of_edit
+</execute_bash>
 
 USER:
 Observation:
-[File: /workspace/app.py (10 lines total after edit)]
-1|from flask import Flask
-2|app = Flask(__name__)
-3|
-4|@app.route('/')
-5|def index():
-6|    numbers = list(range(1, 11))
-7|    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
-8|
-9|if __name__ == '__main__':
-10|    app.run(port=5000)
-[File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
+[File: /workspace/app.py (11 lines total)]
+1:
+2:from flask import Flask
+3:app = Flask(__name__)
+4:
+5:@app.route('/')
+6:def index():
+7:    numbers = list(range(1, 11))
+8:    return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
+9:
+10:if __name__ == '__main__':
+11:    app.run(port=5000)
+File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.
 
 ASSISTANT:
 The file has been updated. Let me run the Python file again with the new changes:

agenthub/codeact_swe_agent/README.md

@@ -1,7 +0,0 @@
-# CodeAct (SWE Edit Specialized)
-
-This agent is an adaptation of the original [SWE Agent](https://swe-agent.com/) based on CodeAct using the `agentskills` library of OpenDevin.
-
-It is intended use is **solving Github issues**.
-
-It removes web-browsing and Github capability from the original CodeAct agent to avoid confusion to the agent.

agenthub/codeact_swe_agent/__init__.py

@@ -1,5 +0,0 @@
-from opendevin.controller.agent import Agent
-
-from .codeact_swe_agent import CodeActSWEAgent
-
-Agent.register('CodeActSWEAgent', CodeActSWEAgent)

agenthub/codeact_swe_agent/codeact_swe_agent.py

@@ -1,246 +0,0 @@
-import re
-
-from agenthub.codeact_swe_agent.prompt import (
-    COMMAND_DOCS,
-    MINIMAL_SYSTEM_PREFIX,
-    SWE_EXAMPLE,
-    SYSTEM_SUFFIX,
-)
-from opendevin.controller.agent import Agent
-from opendevin.controller.state.state import State
-from opendevin.events.action import (
-    Action,
-    AgentFinishAction,
-    BrowseInteractiveAction,
-    CmdRunAction,
-    IPythonRunCellAction,
-    MessageAction,
-)
-from opendevin.events.observation import (
-    BrowserOutputObservation,
-    CmdOutputObservation,
-    IPythonRunCellObservation,
-)
-from opendevin.llm.llm import LLM
-from opendevin.runtime.plugins import (
-    AgentSkillsRequirement,
-    JupyterRequirement,
-    PluginRequirement,
-)
-
-
-def parse_response(response) -> str:
-    action = response.choices[0].message.content
-    for lang in ['bash', 'ipython', 'browse']:
-        if f'<execute_{lang}>' in action and f'</execute_{lang}>' not in action:
-            action += f'</execute_{lang}>'
-    return action
-
-
-def action_to_str(action: Action) -> str:
-    if isinstance(action, CmdRunAction):
-        return f'{action.thought}\n<execute_bash>\n{action.command}\n</execute_bash>'
-    elif isinstance(action, IPythonRunCellAction):
-        return f'{action.thought}\n<execute_ipython>\n{action.code}\n</execute_ipython>'
-    elif isinstance(action, BrowseInteractiveAction):
-        return f'{action.thought}\n<execute_browse>\n{action.browser_actions}\n</execute_browse>'
-    elif isinstance(action, MessageAction):
-        return action.content
-    return ''
-
-
-def get_action_message(action: Action) -> dict[str, str] | None:
-    if (
-        isinstance(action, BrowseInteractiveAction)
-        or isinstance(action, CmdRunAction)
-        or isinstance(action, IPythonRunCellAction)
-        or isinstance(action, MessageAction)
-    ):
-        return {
-            'role': 'user' if action.source == 'user' else 'assistant',
-            'content': action_to_str(action),
-        }
-    return None
-
-
-def get_observation_message(obs) -> dict[str, str] | None:
-    if isinstance(obs, CmdOutputObservation):
-        content = 'OBSERVATION:\n' + truncate_observation(obs.content)
-        content += (
-            f'\n[Command {obs.command_id} finished with exit code {obs.exit_code}]]'
-        )
-        return {'role': 'user', 'content': content}
-    elif isinstance(obs, IPythonRunCellObservation):
-        content = 'OBSERVATION:\n' + obs.content
-        # replace base64 images with a placeholder
-        splitted = content.split('\n')
-        for i, line in enumerate(splitted):
-            if '![image](data:image/png;base64,' in line:
-                splitted[i] = (
-                    '![image](data:image/png;base64, ...) already displayed to user'
-                )
-        content = '\n'.join(splitted)
-        content = truncate_observation(content)
-        return {'role': 'user', 'content': content}
-    elif isinstance(obs, BrowserOutputObservation):
-        content = 'OBSERVATION:\n' + truncate_observation(obs.content)
-        return {'role': 'user', 'content': content}
-    return None
-
-
-def truncate_observation(observation: str, max_chars: int = 10_000) -> str:
-    """
-    Truncate the middle of the observation if it is too long.
-    """
-    if len(observation) <= max_chars:
-        return observation
-    half = max_chars // 2
-    return (
-        observation[:half]
-        + '\n[... Observation truncated due to length ...]\n'
-        + observation[-half:]
-    )
-
-
-def get_system_message() -> str:
-    return f'{MINIMAL_SYSTEM_PREFIX}\n\n{COMMAND_DOCS}\n\n{SYSTEM_SUFFIX}'
-
-
-def get_in_context_example() -> str:
-    return SWE_EXAMPLE
-
-
-class CodeActSWEAgent(Agent):
-    VERSION = '1.5'
-    """
-    This agent is an adaptation of the original [SWE Agent](https://swe-agent.com/) based on CodeAct 1.5 using the `agentskills` library of OpenDevin.
-
-    It is intended use is **solving Github issues**.
-
-    It removes web-browsing and Github capability from the original CodeAct agent to avoid confusion to the agent.
-    """
-
-    sandbox_plugins: list[PluginRequirement] = [
-        # NOTE: AgentSkillsRequirement need to go before JupyterRequirement, since
-        # AgentSkillsRequirement provides a lot of Python functions
-        # and it need to be initialized before Jupyter for Jupyter to use those functions.
-        AgentSkillsRequirement(),
-        JupyterRequirement(),
-    ]
-    jupyter_kernel_init_code: str = 'from agentskills import *'
-
-    system_message: str = get_system_message()
-    in_context_example: str = f"Here is an example of how you can interact with the environment for task solving:\n{get_in_context_example()}\n\nNOW, LET'S START!"
-
-    def __init__(
-        self,
-        llm: LLM,
-    ) -> None:
-        """
-        Initializes a new instance of the CodeActAgent class.
-
-        Parameters:
-        - llm (LLM): The llm to be used by this agent
-        """
-        super().__init__(llm)
-        self.reset()
-
-    def reset(self) -> None:
-        """
-        Resets the CodeAct Agent.
-        """
-        super().reset()
-
-    def step(self, state: State) -> Action:
-        """
-        Performs one step using the CodeAct Agent.
-        This includes gathering info on previous steps and prompting the model to make a command to execute.
-
-        Parameters:
-        - state (State): used to get updated info and background commands
-
-        Returns:
-        - CmdRunAction(command) - bash command to run
-        - IPythonRunCellAction(code) - IPython code to run
-        - BrowseInteractiveAction(browsergym_command) - BrowserGym commands to run
-        - MessageAction(content) - Message action to run (e.g. ask for clarification)
-        - AgentFinishAction() - end the interaction
-        """
-        messages: list[dict[str, str]] = [
-            {'role': 'system', 'content': self.system_message},
-            {'role': 'user', 'content': self.in_context_example},
-        ]
-
-        for prev_action, obs in state.history:
-            action_message = get_action_message(prev_action)
-            if action_message:
-                messages.append(action_message)
-
-            obs_message = get_observation_message(obs)
-            if obs_message:
-                messages.append(obs_message)
-
-        latest_user_message = [m for m in messages if m['role'] == 'user'][-1]
-        if latest_user_message:
-            if latest_user_message['content'].strip() == '/exit':
-                return AgentFinishAction()
-            latest_user_message['content'] += (
-                f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task.'
-            )
-
-        response = self.llm.do_completion(
-            messages=messages,
-            stop=[
-                '</execute_ipython>',
-                '</execute_bash>',
-                '</execute_browse>',
-            ],
-            temperature=0.0,
-        )
-
-        action_str: str = parse_response(response)
-        state.num_of_chars += sum(
-            len(message['content']) for message in messages
-        ) + len(action_str)
-
-        if finish_command := re.search(r'<finish>.*</finish>', action_str, re.DOTALL):
-            thought = action_str.replace(finish_command.group(0), '').strip()
-            return AgentFinishAction(thought=thought)
-        if bash_command := re.search(
-            r'<execute_bash>(.*?)</execute_bash>', action_str, re.DOTALL
-        ):
-            # remove the command from the action string to get thought
-            thought = action_str.replace(bash_command.group(0), '').strip()
-            # a command was found
-            command_group = bash_command.group(1).strip()
-
-            if command_group.strip() == 'exit':
-                return AgentFinishAction()
-            return CmdRunAction(command=command_group, thought=thought)
-        elif python_code := re.search(
-            r'<execute_ipython>(.*?)</execute_ipython>', action_str, re.DOTALL
-        ):
-            # a code block was found
-            code_group = python_code.group(1).strip()
-            thought = action_str.replace(python_code.group(0), '').strip()
-            return IPythonRunCellAction(
-                code=code_group,
-                thought=thought,
-                kernel_init_code=self.jupyter_kernel_init_code,
-            )
-        elif browse_command := re.search(
-            r'<execute_browse>(.*)</execute_browse>', action_str, re.DOTALL
-        ):
-            # BrowserGym actions was found
-            browse_actions = browse_command.group(1).strip()
-            thought = action_str.replace(browse_command.group(0), '').strip()
-            return BrowseInteractiveAction(
-                browser_actions=browse_actions, thought=thought
-            )
-        else:
-            # We assume the LLM is GOOD enough that when it returns pure natural language
-            # it want to talk to the user
-            return MessageAction(content=action_str, wait_for_response=True)
-
-    def search_memory(self, query: str) -> list[str]:
-        raise NotImplementedError('Implement this abstract method')

agenthub/codeact_swe_agent/prompt.py

@@ -1,451 +0,0 @@
-from opendevin.runtime.plugins import AgentSkillsRequirement
-
-_AGENT_SKILLS_DOCS = AgentSkillsRequirement.documentation
-
-COMMAND_DOCS = (
-    '\nApart from the standard Python library, the assistant can also use the following functions (already imported) in <execute_ipython> environment:\n'
-    f'{_AGENT_SKILLS_DOCS}'
-    "Please note that THE `edit_file` FUNCTION REQUIRES PROPER INDENTATION. If the assistant would like to add the line '        print(x)', it must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run."
-)
-
-# ======= SYSTEM MESSAGE =======
-MINIMAL_SYSTEM_PREFIX = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
-The assistant can interact with an interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code should be enclosed using "<execute_ipython>" tag, for example:
-<execute_ipython>
-print("Hello World!")
-</execute_ipython>
-The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
-For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
-"""
-
-SYSTEM_SUFFIX = """The assistant's response should be concise.
-The assistant should include ONLY ONE <execute_ipython> or <execute_bash> or <execute_browse> in every one of the responses, unless the assistant is finished with the task or need more input or action from the user in order to proceed.
-IMPORTANT: Whenever possible, execute the code for the user using <execute_ipython> or <execute_bash> or <execute_browse> instead of providing it.
-"""
-
-SWE_EXAMPLE = """
---- START OF EXAMPLE ---
-
-USER: We are currently solving the following issue within our repository. Here is the issue text:
---- BEGIN ISSUE ---
-TimeDelta serialization precision
-Hi there!
-
-I just found quite strange behaviour of `TimeDelta` field serialization
-
-```python3
-from marshmallow.fields import TimeDelta
-from datetime import timedelta
-
-td_field = TimeDelta(precision="milliseconds")
-
-obj = dict()
-obj["td_field"] = timedelta(milliseconds=345)
-
-print(td_field.serialize("td_field", obj))
-```
-
-Output of this snippet is `344`, but it seems that `345` is correct.
-
-Looks like a rounding issue here: https://github.com/marshmallow-code/marshmallow/blob/dev/src/marshmallow/fields.py#L1474
---- END ISSUE ---
-
-[Current directory: /workspace/marshmallow-code__marshmallow]
-
-ASSISTANT:
-Let's first start by reproducing the results of the issue. The issue includes some example code for reproduction, which we can use. We'll create a new file called `reproduce.py` and paste the example code into it.
-<execute_ipython>
-create_file('/workspace/marshmallow-code__marshmallow/reproduce.py')
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[File: /workspace/marshmallow-code__marshmallow/reproduce.py (1 lines total)]
-1|
-[File /workspace/marshmallow-code__marshmallow/reproduce.py created.]
-
-ASSISTANT:
-Now let's paste in the example code from the issue.
-<execute_ipython>
-EDITED_CODE = \"\"\"from marshmallow.fields import TimeDelta
-from datetime import timedelta
-
-td_field = TimeDelta(precision="milliseconds")
-
-obj = dict()
-obj["td_field"] = timedelta(milliseconds=345)
-
-print(td_field.serialize("td_field", obj))
-\"\"\"
-edit_file(start=1, end=1, content=EDITED_CODE)
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[File: /workspace/marshmallow-code__marshmallow/reproduce.py (9 lines total)]
-1|from marshmallow.fields import TimeDelta
-2|from datetime import timedelta
-3|
-4|td_field = TimeDelta(precision="milliseconds")
-5|
-6|obj = dict()
-7|obj["td_field"] = timedelta(milliseconds=345)
-8|
-9|print(td_field.serialize("td_field", obj))
-[File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
-
-ASSISTANT:
-Now let's run the code to see if we see the same output as the issue.
-<execute_bash>
-python reproduce.py
-</execute_bash>
-
-USER:
-OBSERVATION:
-344
-
-ASSISTANT:
-We are indeed seeing the same output as the issue. The issue suggests that we should look at line 1474 of the `fields.py` file to see if there is a rounding issue. Let's open the `fields.py` file and navigate to line 1474 to see the relevant code. First we should find the `fields.py` file in the repository. It is likely to be in the `src/marshmallow` directory, but we should check to be sure.
-<execute_bash>
-ls -F
-</execute_bash>
-
-USER:
-OBSERVATION:
-AUTHORS.rst
-CHANGELOG.rst
-CODE_OF_CONDUCT.md
-CONTRIBUTING.rst
-LICENSE
-MANIFEST.in
-NOTICE
-README.rst
-RELEASING.md
-azure-pipelines.yml
-docs/
-examples/
-performance/
-pyproject.toml
-reproduce.py
-setup.cfg
-setup.py
-src/
-tests/
-tox.ini
-
-ASSISTANT:
-It looks like the `src` directory is present, which suggests that the `fields.py` file is likely to be in the `src` directory. Let's use find_file to see where it is.
-<execute_ipython>
-find_file("fields.py", "/workspace/marshmallow-code__marshmallow/src")
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[Found 1 matches for "fields.py" in /workspace/marshmallow-code__marshmallow/src]
-/workspace/marshmallow-code__marshmallow/src/marshmallow/fields.py
-
-ASSISTANT:
-It looks like the `fields.py` file is present in the `./src/marshmallow/` directory. The issue also points to a specific URL with line number 1474. We should navigate to that line in fields.py to see the relevant code for the `TimeDelta` serialization.
-<execute_ipython>
-open_file("/workspace/marshmallow-code__marshmallow/src/marshmallow/fields.py", 1474)
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[File: /workspace/marshmallow-code__marshmallow/src/marshmallow/fields.py (1997 lines total)]
-(1458 more lines above)
-1459|            self.WEEKS,
-1460|        )
-1461|
-1462|        if precision not in units:
-1463|            msg = 'The precision must be {} or "{}".'.format(
-1464|                ", ".join([f'"{each}"' for each in units[:-1]]), units[-1]
-1465|            )
-1466|            raise ValueError(msg)
-1467|
-1468|        self.precision = precision
-1469|        super().__init__(**kwargs)
-1470|
-1471|    def _serialize(self, value, attr, obj, **kwargs):
-1472|        if value is None:
-1473|            return None
-1474|        base_unit = dt.timedelta(**{self.precision: 1})
-1475|        return int(value.total_seconds() / base_unit.total_seconds())
-1476|
-1477|    def _deserialize(self, value, attr, data, **kwargs):
-1478|        try:
-1479|            value = int(value)
-1480|        except (TypeError, ValueError) as error:
-1481|            raise self.make_error("invalid") from error
-1482|
-1483|        kwargs = {self.precision: value}
-1484|
-1485|        try:
-1486|            return dt.timedelta(**kwargs)
-1487|        except OverflowError as error:
-1488|            raise self.make_error("invalid") from error
-1489|
-1490|
-1491|class Mapping(Field):
-1492|    \"\"\"An abstract class for objects with key-value pairs.
-1493|
-1494|    :param keys: A field class or instance for dict keys.
-1495|    :param values: A field class or instance for dict values.
-1496|    :param kwargs: The same keyword arguments that :class:`Field` receives.
-1497|
-1498|    .. note::
-1499|        When the structure of nested data is not known, you may omit the
-1500|        `keys` and `values` arguments to prevent content validation.
-1501|
-1502|    .. versionadded:: 3.0.0rc4
-1503|    \"\"\"
-1504|
-1505|    mapping_type = dict
-1506|
-1507|    #: Default error messages.
-1508|    default_error_messages = {"invalid": "Not a valid mapping type."}
-1509|
-1510|    def __init__(
-1511|        self,
-1512|        keys: typing.Optional[typing.Union[Field, type]] = None,
-1513|        values: typing.Optional[typing.Union[Field, type]] = None,
-1514|        **kwargs,
-1515|    ):
-1516|        super().__init__(**kwargs)
-1517|        if keys is None:
-1518|            self.key_field = None
-1519|        else:
-1520|            try:
-1521|                self.key_field = resolve_field_instance(keys)
-1522|            except FieldInstanceResolutionError as error:
-1523|                raise ValueError(
-1524|                    '"keys" must be a subclass or instance of '
-1525|                    "marshmallow.base.FieldABC."
-1526|                ) from error
-1527|
-1528|        if values is None:
-1529|            self.value_field = None
-1530|        else:
-1531|            try:
-1532|                self.value_field = resolve_field_instance(values)
-1533|            except FieldInstanceResolutionError as error:
-1534|                raise ValueError(
-1535|                    '"values" must be a subclass or instance of '
-1536|                    "marshmallow.base.FieldABC."
-1537|                ) from error
-1538|            if isinstance(self.value_field, Nested):
-1539|                self.only = self.value_field.only
-1540|                self.exclude = self.value_field.exclude
-1541|
-1542|    def _bind_to_schema(self, field_name, schema):
-1543|        super()._bind_to_schema(field_name, schema)
-1544|        if self.value_field:
-1545|            self.value_field = copy.deepcopy(self.value_field)
-1546|            self.value_field._bind_to_schema(field_name, self)
-1547|        if isinstance(self.value_field, Nested):
-1548|            self.value_field.only = self.only
-1549|            self.value_field.exclude = self.exclude
-1550|        if self.key_field:
-1551|            self.key_field = copy.deepcopy(self.key_field)
-1552|            self.key_field._bind_to_schema(field_name, self)
-1553|
-1554|    def _serialize(self, value, attr, obj, **kwargs):
-1555|        if value is None:
-1556|            return None
-1557|        if not self.value_field and not self.key_field:
-1558|            return self.mapping_type(value)
-(439 more lines below)
-
-ASSISTANT:
-We are now looking at the relevant section of the `fields.py` file where the `TimeDelta` serialization occurs. The issue suggests that there is a rounding problem near line 1474. The current implementation uses integer division, which truncates the decimal part and can lead to incorrect rounding on line 1475. To fix this issue, we should ensure that the division results in a float, and then round the result to the nearest integer before converting it to an int. This should preserve the milliseconds precision as expected. Let's make the necessary edit to the code.
-<execute_ipython>
-edit_file(start=1475, end=1475, content="return int(round(value.total_seconds() / base_unit.total_seconds()))  # round to nearest int")
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[Your proposed edit has introduced new syntax error(s). Please understand the errors and retry your edit command.]
-
-ERRORS:
-- E999 IndentationError: unexpected indent
-
-[This is how your edit would have looked if applied]
--------------------------------------------------
-[File: /marshmallow-code__marshmallow/src/marshmallow/fields.py (1997 lines total)]
-(1468 more lines above)
-1469|        super().__init__(**kwargs)
-1470|
-1471|    def _serialize(self, value, attr, obj, **kwargs):
-1472|        if value is None:
-1473|            return None
-1474|        base_unit = dt.timedelta(**{self.precision: 1})
-1475|return int(round(value.total_seconds() / base_unit.total_seconds()))  # round to nearest int
-1476|
-1477|    def _deserialize(self, value, attr, data, **kwargs):
-1478|        try:
-1479|            value = int(value)
-(518 more lines below)
--------------------------------------------------
-
-[This is the original code before your edit]
--------------------------------------------------
-[File: /marshmallow-code__marshmallow/src/marshmallow/fields.py (1997 lines total)]
-(1469 more lines above)
-1470|
-1471|    def _serialize(self, value, attr, obj, **kwargs):
-1472|        if value is None:
-1473|            return None
-1474|        base_unit = dt.timedelta(**{self.precision: 1})
-1475|        return int(value.total_seconds() / base_unit.total_seconds())
-1476|
-1477|    def _deserialize(self, value, attr, data, **kwargs):
-1478|        try:
-1479|            value = int(value)
-1480|        except (TypeError, ValueError) as error:
-(517 more lines below)
--------------------------------------------------
-Your changes have NOT been applied. Please fix your edit command and try again.
-You either need to 1) Specify the correct start/end line arguments or 2) Correct your edit code.
-DO NOT re-run the same failed edit command. Running it again will lead to the same error.
-
-ASSISTANT:
-My edit command did not use the proper indentation, I will fix my syntax in this follow up edit command.
-<execute_ipython>
-EDITED_CODE="        return int(round(value.total_seconds() / base_unit.total_seconds()))  # round to nearest int"
-edit_file(start=1475, end=1475, content=EDITED_CODE)
-</execute_ipython>
-
-USER:
-OBSERVATION:
-[File: /marshmallow-code__marshmallow/src/marshmallow/fields.py (1997 lines total)]
-(1424 more lines above)
-1425|
-1426|    :param precision: Influences how the integer is interpreted during
-1427|        (de)serialization. Must be 'days', 'seconds', 'microseconds',
-1428|        'milliseconds', 'minutes', 'hours' or 'weeks'.
-1429|    :param kwargs: The same keyword arguments that :class:`Field` receives.
-1430|
-1431|    .. versionchanged:: 2.0.0
-1432|        Always serializes to an integer value to avoid rounding errors.
-1433|        Add `precision` parameter.
-1434|    \"\"\"
-1435|
-1436|    DAYS = "days"
-1437|    SECONDS = "seconds"
-1438|    MICROSECONDS = "microseconds"
-1439|    MILLISECONDS = "milliseconds"
-1440|    MINUTES = "minutes"
-1441|    HOURS = "hours"
-1442|    WEEKS = "weeks"
-1443|
-1444|    #: Default error messages.
-1445|    default_error_messages = {
-1446|        "invalid": "Not a valid period of time.",
-1447|        "format": "{input!r} cannot be formatted as a timedelta.",
-1448|    }
-1449|
-1450|    def __init__(self, precision: str = SECONDS, **kwargs):
-1451|        precision = precision.lower()
-1452|        units = (
-1453|            self.DAYS,
-1454|            self.SECONDS,
-1455|            self.MICROSECONDS,
-1456|            self.MILLISECONDS,
-1457|            self.MINUTES,
-1458|            self.HOURS,
-1459|            self.WEEKS,
-1460|        )
-1461|
-1462|        if precision not in units:
-1463|            msg = 'The precision must be {} or "{}".'.format(
-1464|                ", ".join([f'"{each}"' for each in units[:-1]]), units[-1]
-1465|            )
-1466|            raise ValueError(msg)
-1467|
-1468|        self.precision = precision
-1469|        super().__init__(**kwargs)
-1470|
-1471|    def _serialize(self, value, attr, obj, **kwargs):
-1472|        if value is None:
-1473|            return None
-1474|        base_unit = dt.timedelta(**{self.precision: 1})
-1475|        return int(round(value.total_seconds() / base_unit.total_seconds()))  # round to nearest int
-1476|
-1477|    def _deserialize(self, value, attr, data, **kwargs):
-1478|        try:
-1479|            value = int(value)
-1480|        except (TypeError, ValueError) as error:
-1481|            raise self.make_error("invalid") from error
-1482|
-1483|        kwargs = {self.precision: value}
-1484|
-1485|        try:
-1486|            return dt.timedelta(**kwargs)
-1487|        except OverflowError as error:
-1488|            raise self.make_error("invalid") from error
-1489|
-1490|
-1491|class Mapping(Field):
-1492|    \"\"\"An abstract class for objects with key-value pairs.
-1493|
-1494|    :param keys: A field class or instance for dict keys.
-1495|    :param values: A field class or instance for dict values.
-1496|    :param kwargs: The same keyword arguments that :class:`Field` receives.
-1497|
-1498|    .. note::
-1499|        When the structure of nested data is not known, you may omit the
-1500|        `keys` and `values` arguments to prevent content validation.
-1501|
-1502|    .. versionadded:: 3.0.0rc4
-1503|    \"\"\"
-1504|
-1505|    mapping_type = dict
-1506|
-1507|    #: Default error messages.
-1508|    default_error_messages = {"invalid": "Not a valid mapping type."}
-1509|
-1510|    def __init__(
-1511|        self,
-1512|        keys: typing.Optional[typing.Union[Field, type]] = None,
-1513|        values: typing.Optional[typing.Union[Field, type]] = None,
-1514|        **kwargs,
-1515|    ):
-1516|        super().__init__(**kwargs)
-1517|        if keys is None:
-1518|            self.key_field = None
-1519|        else:
-1520|            try:
-1521|                self.key_field = resolve_field_instance(keys)
-1522|            except FieldInstanceResolutionError as error:
-1523|                raise ValueError(
-1524|                    '"keys" must be a subclass or instance of '
-(473 more lines below)
-[File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.]
-
-ASSISTANT:
-The code has been updated to use the `round` function, which should fix the rounding issue. Before submitting the changes, it would be prudent to run the reproduce.py code again to ensure that this change has the desired effect. I see that "Current directory" is still `/marshmallow-code__marshmallow`, so we can just run `reproduce.py` directly.
-<execute_bash>
-python reproduce.py
-</execute_bash>
-
-USER:
-OBSERVATION:
-345
-
-ASSISTANT:
-The output has changed from 344 to 345, which suggests that the rounding issue has been fixed. Let's remove the reproduce.py file since it is no longer needed.
-<execute_bash>
-rm reproduce.py
-</execute_bash>
-
-USER:
-OBSERVATION:
-
-ASSISTANT:
-rm doesn't have any output when it deletes successfully, so that must have worked. Now that we have fixed the issue, we can exit the current task.
-<execute_bash>
-exit
-</execute_bash>
-
---- END OF EXAMPLE ---
-"""

agenthub/delegator_agent/agent.py

@@ -6,7 +6,6 @@ from opendevin.llm.llm import LLM
 
 
 class DelegatorAgent(Agent):
-    VERSION = '1.0'
     """
     The planner agent utilizes a special prompting strategy to create long term plans for solving problems.
     The agent is given its previous action-observation pairs, current task, and hint based on last action taken at every step.

agenthub/dummy_agent/agent.py

@@ -9,7 +9,6 @@ from opendevin.events.action import (
     AgentFinishAction,
     AgentRecallAction,
     AgentRejectAction,
-    BrowseInteractiveAction,
     BrowseURLAction,
     CmdRunAction,
     FileReadAction,
@@ -44,7 +43,6 @@ BACKGROUND_CMD = 'echo "This is in the background" && sleep .1 && echo "This too
 
 
 class DummyAgent(Agent):
-    VERSION = '1.0'
     """
     The DummyAgent is used for e2e testing. It just sends the same set of actions deterministically,
     without making any LLM calls.
@@ -123,14 +121,6 @@ class DummyAgent(Agent):
                     # BrowserOutputObservation('<html></html>', url='https://google.com', screenshot=""),
                 ],
             },
-            {
-                'action': BrowseInteractiveAction(
-                    browser_actions='goto("https://google.com")'
-                ),
-                'observations': [
-                    # BrowserOutputObservation('<html></html>', url='https://google.com', screenshot=""),
-                ],
-            },
             {
                 'action': AgentFinishAction(),
                 'observations': [],

agenthub/micro/agent.py

@@ -35,14 +35,11 @@ def history_to_json(obj, **kwargs):
         # process history, make it simpler.
         processed_history = []
         for action, observation in obj:
-            processed_history.append(
-                (event_to_memory(action), event_to_memory(observation))
-            )
+            processed_history.append((event_to_memory(action), event_to_memory(observation)))
         return json.dumps(processed_history, **kwargs)
 
 
 class MicroAgent(Agent):
-    VERSION = '1.0'
     prompt = ''
     agent_definition: dict = {}
 
@@ -65,7 +62,7 @@ class MicroAgent(Agent):
             latest_user_message=latest_user_message,
         )
         messages = [{'content': prompt, 'role': 'user'}]
-        resp = self.llm.do_completion(messages=messages)
+        resp = self.llm.completion(messages=messages)
         action_resp = resp['choices'][0]['message']['content']
         state.num_of_chars += len(prompt) + len(action_resp)
         action = parse_response(action_resp)

agenthub/micro/registry.py

@@ -13,7 +13,8 @@ for dir in os.listdir(os.path.dirname(__file__)):
     promptFile = base + '/prompt.md'
     agentFile = base + '/agent.yaml'
     if not os.path.isfile(promptFile) or not os.path.isfile(agentFile):
-        raise Exception(f'Missing prompt or agent file in {base}. Please create them.')
+        raise Exception(
+            f'Missing prompt or agent file in {base}. Please create them.')
     with open(promptFile, 'r') as f:
         prompt = f.read()
     with open(agentFile, 'r') as f:

agenthub/monologue_agent/agent.py

@@ -1,5 +1,4 @@
 import agenthub.monologue_agent.utils.prompts as prompts
-from agenthub.monologue_agent.utils.prompts import INITIAL_THOUGHTS
 from opendevin.controller.agent import Agent
 from opendevin.controller.state.state import State
 from opendevin.core.config import config
@@ -26,6 +25,7 @@ from opendevin.events.observation import (
 from opendevin.events.serialization.event import event_to_memory
 from opendevin.llm.llm import LLM
 from opendevin.memory.condenser import MemoryCondenser
+from opendevin.memory.history import ShortTermHistory
 
 if config.agent.memory_enabled:
     from opendevin.memory.memory import LongTermMemory
@@ -33,9 +33,53 @@ if config.agent.memory_enabled:
 MAX_TOKEN_COUNT_PADDING = 512
 MAX_OUTPUT_LENGTH = 5000
 
+INITIAL_THOUGHTS = [
+    'I exist!',
+    'Hmm...looks like I can type in a command line prompt',
+    'Looks like I have a web browser too!',
+    "Here's what I want to do: $TASK",
+    'How am I going to get there though?',
+    'It seems like I have some kind of short term memory.',
+    'Each of my thoughts seems to be stored in a JSON array.',
+    'It seems whatever I say next will be added as an object to the list.',
+    'But no one has perfect short-term memory. My list of thoughts will be summarized and condensed over time, losing information in the process.',
+    'Fortunately I have long term memory!',
+    'I can just perform a recall action, followed by the thing I want to remember. And then related thoughts just spill out!',
+    "Sometimes they're random thoughts that don't really have to do with what I wanted to remember. But usually they're exactly what I need!",
+    "Let's try it out!",
+    'RECALL what it is I want to do',
+    "Here's what I want to do: $TASK",
+    'How am I going to get there though?',
+    "Neat! And it looks like it's easy for me to use the command line too! I just have to perform a run action and include the command I want to run in the command argument. The command output just jumps into my head!",
+    'RUN echo "hello world"',
+    'hello world',
+    'Cool! I bet I can write files too using the write action.',
+    'WRITE echo "console.log(\'hello world\')" > test.js',
+    '',
+    "I just created test.js. I'll try and run it now.",
+    'RUN node test.js',
+    'hello world',
+    'It works!',
+    "I'm going to try reading it now using the read action.",
+    'READ test.js',
+    "console.log('hello world')",
+    'Nice! I can read files too!',
+    'And if I want to use the browser, I just need to use the browse action and include the url I want to visit in the url argument',
+    "Let's try that...",
+    'BROWSE google.com',
+    '<form><input type="text"></input><button type="submit"></button></form>',
+    'I can browse the web too!',
+    'And once I have completed my task, I can use the finish action to stop working.',
+    "But I should only use the finish action when I'm absolutely certain that I've completed my task and have tested my work.",
+    'Very cool. Now to accomplish my task.',
+    "I'll need a strategy. And as I make progress, I'll need to keep refining that strategy. I'll need to set goals, and break them into sub-goals.",
+    'In between actions, I must always take some time to think, strategize, and set new goals. I should never take two actions in a row.',
+    "OK so my task is to $TASK. I haven't made any progress yet. Where should I start?",
+    'It seems like there might be an existing project here. I should probably start by running `pwd` and `ls` to orient myself.',
+]
+
 
 class MonologueAgent(Agent):
-    VERSION = '1.0'
     """
     The Monologue Agent utilizes long and short term memory to complete tasks.
     Long term memory is stored as a LongTermMemory object and the model uses it to search for examples from the past.
@@ -43,19 +87,57 @@ class MonologueAgent(Agent):
     """
 
     _initialized = False
-    initial_thoughts: list[dict[str, str]]
+    monologue: ShortTermHistory
     memory: 'LongTermMemory | None'
     memory_condenser: MemoryCondenser
 
     def __init__(self, llm: LLM):
         """
-        Initializes the Monologue Agent with an llm.
+        Initializes the Monologue Agent with an llm, monologue, and memory.
 
         Parameters:
         - llm (LLM): The llm to be used by this agent
         """
         super().__init__(llm)
 
+    def _add_event(self, event_dict: dict):
+        """
+        Adds a new event to the agent's monologue and memory.
+        Monologue automatically condenses when it gets too large.
+
+        Parameters:
+        - event (dict): The event that will be added to monologue and memory
+        """
+
+        if (
+            'args' in event_dict
+            and 'output' in event_dict['args']
+            and len(event_dict['args']['output']) > MAX_OUTPUT_LENGTH
+        ):
+            event_dict['args']['output'] = (
+                event_dict['args']['output'][:MAX_OUTPUT_LENGTH] + '...'
+            )
+
+        self.monologue.add_event(event_dict)
+        if self.memory is not None:
+            self.memory.add_event(event_dict)
+
+        # Test monologue token length
+        prompt = prompts.get_request_action_prompt(
+            '',
+            self.monologue.get_events(),
+            [],
+        )
+        messages = [{'content': prompt, 'role': 'user'}]
+        token_count = self.llm.get_token_count(messages)
+
+        if token_count + MAX_TOKEN_COUNT_PADDING > self.llm.max_input_tokens:
+            prompt = prompts.get_summarize_monologue_prompt(self.monologue.events)
+            summary_response = self.memory_condenser.condense(
+                summarize_prompt=prompt, llm=self.llm
+            )
+            self.monologue.events = prompts.parse_summary_response(summary_response)
+
     def _initialize(self, task: str):
         """
         Utilizes the INITIAL_THOUGHTS list to give the agent a context for its capabilities
@@ -76,7 +158,7 @@ class MonologueAgent(Agent):
         if task is None or task == '':
             raise AgentNoInstructionError()
 
-        self.initial_thoughts = []
+        self.monologue = ShortTermHistory()
         if config.agent.memory_enabled:
             self.memory = LongTermMemory()
         else:
@@ -105,7 +187,7 @@ class MonologueAgent(Agent):
                     observation = BrowserOutputObservation(
                         content=thought, url='', screenshot=''
                     )
-                self.initial_thoughts.append(event_to_memory(observation))
+                self._add_event(event_to_memory(observation))
                 previous_action = ''
             else:
                 action: Action = NullAction()
@@ -132,7 +214,7 @@ class MonologueAgent(Agent):
                     previous_action = ActionType.BROWSE
                 else:
                     action = MessageAction(thought)
-                self.initial_thoughts.append(event_to_memory(action))
+                self._add_event(event_to_memory(action))
 
     def step(self, state: State) -> Action:
         """
@@ -147,75 +229,25 @@ class MonologueAgent(Agent):
 
         goal = state.get_current_user_intent()
         self._initialize(goal)
+        for prev_action, obs in state.updated_info:
+            self._add_event(event_to_memory(prev_action))
+            self._add_event(event_to_memory(obs))
 
-        recent_events: list[dict[str, str]] = []
+        state.updated_info = []
 
-        # add the events from state.history
-        for prev_action, obs in state.history:
-            if not isinstance(prev_action, NullAction):
-                recent_events.append(event_to_memory(prev_action))
-            if not isinstance(obs, NullObservation):
-                recent_events.append(self._truncate_output(event_to_memory(obs)))
-
-        # add the last messages to long term memory
-        if self.memory is not None and state.history and len(state.history) > 0:
-            self.memory.add_event(event_to_memory(state.history[-1][0]))
-            self.memory.add_event(
-                self._truncate_output(event_to_memory(state.history[-1][1]))
-            )
-
-        # the action prompt with initial thoughts and recent events
         prompt = prompts.get_request_action_prompt(
             goal,
-            self.initial_thoughts,
-            recent_events,
+            self.monologue.get_events(),
             state.background_commands_obs,
         )
-
-        messages: list[dict[str, str]] = [
-            {'role': 'user', 'content': prompt},
-        ]
-
-        # format all as a single message, a monologue
-        resp = self.llm.do_completion(messages=messages)
-
-        # get the next action from the response
+        messages = [{'content': prompt, 'role': 'user'}]
+        resp = self.llm.completion(messages=messages)
         action_resp = resp['choices'][0]['message']['content']
-
-        # keep track of max_chars fallback option
         state.num_of_chars += len(prompt) + len(action_resp)
-
         action = prompts.parse_action_response(action_resp)
         self.latest_action = action
         return action
 
-    def _truncate_output(
-        self, observation: dict, max_chars: int = MAX_OUTPUT_LENGTH
-    ) -> dict[str, str]:
-        """
-        Truncates the output of an observation to a maximum number of characters.
-
-        Parameters:
-        - output (str): The observation whose output to truncate
-        - max_chars (int): The maximum number of characters to allow
-
-        Returns:
-        - str: The truncated output
-        """
-        if (
-            'args' in observation
-            and 'output' in observation['args']
-            and len(observation['args']['output']) > max_chars
-        ):
-            output = observation['args']['output']
-            half = max_chars // 2
-            observation['args']['output'] = (
-                output[:half]
-                + '\n[... Output truncated due to length...]\n'
-                + output[-half:]
-            )
-        return observation
-
     def search_memory(self, query: str) -> list[str]:
         """
         Uses VectorIndexRetriever to find related memories within the long term memory.

agenthub/monologue_agent/utils/prompts.py

@@ -18,6 +18,7 @@ This is your internal monologue, in JSON format:
 
 %(monologue)s
 
+
 Your most recent thought is at the bottom of that monologue. Continue your train of thought.
 What is your next single thought or action? Your response must be in JSON format.
 It must be a single object, and it must contain two fields:
@@ -50,7 +51,7 @@ Here are the possible actions:
 
 %(background_commands)s
 
-You MUST take time to think in between read, write, run, kill, browse, push, and recall actions--do this with the `message` action.
+You MUST take time to think in between read, write, run, browse, push, and recall actions--do this with the `message` action.
 You should never act twice in a row without thinking. But if your last several
 actions are all `message` actions, you should consider taking a different action.
 
@@ -91,51 +92,6 @@ The action key may be `summarize`, and `args.summary` should contain the summary
 You can also use the same action and args from the source monologue.
 """
 
-INITIAL_THOUGHTS = [
-    'I exist!',
-    'Hmm...looks like I can type in a command line prompt',
-    'Looks like I have a web browser too!',
-    "Here's what I want to do: $TASK",
-    'How am I going to get there though?',
-    'It seems like I have some kind of short term memory.',
-    'Each of my thoughts seems to be stored in a JSON array.',
-    'It seems whatever I say next will be added as an object to the list.',
-    'But no one has perfect short-term memory. My list of thoughts will be summarized and condensed over time, losing information in the process.',
-    'Fortunately I have long term memory!',
-    'I can just perform a recall action, followed by the thing I want to remember. And then related thoughts just spill out!',
-    "Sometimes they're random thoughts that don't really have to do with what I wanted to remember. But usually they're exactly what I need!",
-    "Let's try it out!",
-    'RECALL what it is I want to do',
-    "Here's what I want to do: $TASK",
-    'How am I going to get there though?',
-    "Neat! And it looks like it's easy for me to use the command line too! I just have to perform a run action and include the command I want to run in the command argument. The command output just jumps into my head!",
-    'RUN echo "hello world"',
-    'hello world',
-    'Cool! I bet I can write files too using the write action.',
-    'WRITE echo "console.log(\'hello world\')" > test.js',
-    '',
-    "I just created test.js. I'll try and run it now.",
-    'RUN node test.js',
-    'hello world',
-    'It works!',
-    "I'm going to try reading it now using the read action.",
-    'READ test.js',
-    "console.log('hello world')",
-    'Nice! I can read files too!',
-    'And if I want to use the browser, I just need to use the browse action and include the url I want to visit in the url argument',
-    "Let's try that...",
-    'BROWSE google.com',
-    '<form><input type="text"></input><button type="submit"></button></form>',
-    'I can browse the web too!',
-    'And once I have completed my task, I can use the finish action to stop working.',
-    "But I should only use the finish action when I'm absolutely certain that I've completed my task and have tested my work.",
-    'Very cool. Now to accomplish my task.',
-    "I'll need a strategy. And as I make progress, I'll need to keep refining that strategy. I'll need to set goals, and break them into sub-goals.",
-    'In between actions, I must always take some time to think, strategize, and set new goals. I should never take two actions in a row.',
-    "OK so my task is to $TASK. I haven't made any progress yet. Where should I start?",
-    'It seems like there might be an existing project here. I should probably start by running `pwd` and `ls` to orient myself.',
-]
-
 
 def get_summarize_monologue_prompt(thoughts: list[dict]):
     """
@@ -152,8 +108,7 @@ def get_summarize_monologue_prompt(thoughts: list[dict]):
 def get_request_action_prompt(
     task: str,
     thoughts: list[dict],
-    recent_events: list[dict],
-    background_commands_obs: list[CmdOutputObservation] | None = None,
+    background_commands_obs: list[CmdOutputObservation] = [],
 ):
     """
     Gets the action prompt formatted with appropriate values.
@@ -164,28 +119,20 @@ def get_request_action_prompt(
     - background_commands_obs (list[CmdOutputObservation]): list of all observed background commands running
 
     Returns:
-    - str: Formatted prompt string with hint, task, monologue, and background commands included
+    - str: Formatted prompt string with hint, task, monologue, and background included
     """
 
-    if background_commands_obs is None:
-        background_commands_obs = []
-
     hint = ''
-    if len(recent_events) > 0:
-        latest_event = recent_events[-1]
-        if 'action' in latest_event:
-            if (
-                latest_event['action'] == 'message'
-                and 'source' in latest_event
-                and latest_event['source'] == 'agent'
-            ):
-                hint = (
-                    "You've been thinking a lot lately. Maybe it's time to take action?"
-                )
-            elif latest_event['action'] == 'error':
+    if len(thoughts) > 0:
+        latest_thought = thoughts[-1]
+        if 'action' in latest_thought:
+            if latest_thought['action'] == 'message':
+                if latest_thought['args']['content'].startswith('OK so my task is'):
+                    hint = "You're just getting started! What should you do first?"
+                else:
+                    hint = "You've been thinking a lot lately. Maybe it's time to take action?"
+            elif latest_thought['action'] == 'error':
                 hint = 'Looks like that last command failed. Maybe you need to fix it, or try something else.'
-    else:
-        hint = "You're just getting started! What should you do first?"
 
     bg_commands_message = ''
     if len(background_commands_obs) > 0:
@@ -198,11 +145,9 @@ def get_request_action_prompt(
 
     user = 'opendevin' if config.run_as_devin else 'root'
 
-    monologue = thoughts + recent_events
-
     return ACTION_PROMPT % {
         'task': task,
-        'monologue': json.dumps(monologue, indent=2),
+        'monologue': json.dumps(thoughts, indent=2),
         'background_commands': bg_commands_message,
         'hint': hint,
         'user': user,

agenthub/planner_agent/agent.py

@@ -7,7 +7,6 @@ from .prompt import get_prompt, parse_response
 
 
 class PlannerAgent(Agent):
-    VERSION = '1.0'
     """
     The planner agent utilizes a special prompting strategy to create long term plans for solving problems.
     The agent is given its previous action-observation pairs, current task, and hint based on last action taken at every step.
@@ -43,7 +42,7 @@ class PlannerAgent(Agent):
             return AgentFinishAction()
         prompt = get_prompt(state)
         messages = [{'content': prompt, 'role': 'user'}]
-        resp = self.llm.do_completion(messages=messages)
+        resp = self.llm.completion(messages=messages)
         action_resp = resp['choices'][0]['message']['content']
         state.num_of_chars += len(prompt) + len(action_resp)
         action = parse_response(action_resp)

agenthub/planner_agent/prompt.py

@@ -94,7 +94,7 @@ It must be an object, and it must contain two fields:
   * `state` - set to 'in_progress' to start the task, 'completed' to finish it, 'verified' to assert that it was successful, 'abandoned' to give up on it permanently, or `open` to stop working on it for now.
 * `finish` - if ALL of your tasks and subtasks have been verified or abandoned, and you're absolutely certain that you've completed your task and have tested your work, use the finish action to stop working.
 
-You MUST take time to think in between read, write, run, kill, browse, and recall actions--do this with the `message` action.
+You MUST take time to think in between read, write, run, browse, and recall actions--do this with the `message` action.
 You should never act twice in a row without thinking. But if your last several
 actions are all `message` actions, you should consider taking a different action.
 
@@ -155,7 +155,7 @@ def get_prompt(state: State) -> str:
     else:
         plan_status = "You're not currently working on any tasks. Your next action MUST be to mark a task as in_progress."
     hint = get_hint(event_to_memory(latest_action).get('action', ''))
-    logger.info('HINT:\n' + hint, extra={'msg_type': 'DETAIL'})
+    logger.info('HINT:\n' + hint, extra={'msg_type': 'INFO'})
     task = state.get_current_user_intent()
     return prompt % {
         'task': task,

containers/app/Dockerfile

@@ -53,7 +53,6 @@ RUN useradd -l -m -u $OPENDEVIN_USER_ID -s /bin/bash opendevin && \
     usermod -aG sudo opendevin && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 RUN chown -R opendevin:app /app && chmod -R 770 /app
-RUN sudo chown -R opendevin:app $WORKSPACE_BASE && sudo chmod -R 770 $WORKSPACE_BASE
 USER opendevin
 
 ENV VIRTUAL_ENV=/app/.venv \

containers/app/entrypoint.sh

@@ -26,17 +26,13 @@ if [[ "$SANDBOX_USER_ID" -eq 0 ]]; then
   "$@"
 else
   echo "Setting up enduser with id $SANDBOX_USER_ID"
-  if id "enduser" &>/dev/null; then
-    echo "User enduser already exists. Skipping creation."
-  else
+  if ! useradd -l -m -u $SANDBOX_USER_ID -s /bin/bash enduser; then
+    echo "Failed to create user enduser with id $SANDBOX_USER_ID. Moving opendevin user."
+    incremented_id=$(($SANDBOX_USER_ID + 1))
+    usermod -u $incremented_id opendevin
     if ! useradd -l -m -u $SANDBOX_USER_ID -s /bin/bash enduser; then
-      echo "Failed to create user enduser with id $SANDBOX_USER_ID. Moving opendevin user."
-      incremented_id=$(($SANDBOX_USER_ID + 1))
-      usermod -u $incremented_id opendevin
-      if ! useradd -l -m -u $SANDBOX_USER_ID -s /bin/bash enduser; then
-        echo "Failed to create user enduser with id $SANDBOX_USER_ID for a second time. Exiting."
-        exit 1
-      fi
+      echo "Failed to create user enduser with id $SANDBOX_USER_ID for a second time. Exiting."
+      exit 1
     fi
   fi
   usermod -aG app enduser

containers/sandbox/Dockerfile

@@ -21,8 +21,6 @@ RUN apt-get update && apt-get install -y \
     jq \
     g++ \
     make \
-    iproute2 \
-    libgl1-mesa-glx \
     && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p -m0755 /var/run/sshd
@@ -33,5 +31,3 @@ RUN ln -s /usr/bin/python3 /usr/bin/python
 # install basic dependencies for CodeActAgent
 RUN pip3 install --upgrade pip
 RUN pip3 install jupyterlab notebook jupyter_kernel_gateway flake8
-# TODO: those dependencies are needed for agentskills, we should pack them in a new sandbox image
-RUN pip3 install python-docx PyPDF2 python-pptx pylatexenc openai opencv-python

dev_config/python/ruff.toml

@@ -9,17 +9,10 @@ select = [
     "F",
     "I",
     "Q",
-    "B",
 ]
 
 ignore = [
     "E501",
-    "B003",
-    "B007",
-    "B009",
-    "B010",
-    "B904",
-    "B018",
 ]
 
 [lint.flake8-quotes]

docs/modules/usage/intro.mdx

@@ -58,13 +58,18 @@ Explore the codebase of OpenDevin on [GitHub](https://github.com/OpenDevin/OpenD
 
 ## 🛠️ Getting Started
 
-The easiest way to run OpenDevin is inside a Docker container. It works best with the most recent version of Docker, `26.0.0`.
-You must be using Linux, Mac OS, or WSL on Windows.
+The easiest way to run OpenDevin is inside a Docker container.
 
 To start the app, run these commands, replacing `$(pwd)/workspace` with the directory you want OpenDevin to work with.
 
 ```
-# The directory you want OpenDevin to work with. It MUST be an absolute path!
+# Your OpenAI API key, or any other LLM API key
+export LLM_API_KEY="sk-..."
+```
+
+```
+# The directory you want OpenDevin to modify.
+# MUST be an absolute path!
 export WORKSPACE_BASE=$(pwd)/workspace
 ```
 
@@ -86,15 +91,15 @@ docker run \
     ghcr.io/opendevin/opendevin:0.5
 ```
 
-You'll find OpenDevin running at [http://localhost:3000](http://localhost:3000).
+You'll find opendevin running at [http://localhost:3000](http://localhost:3000).
 
 :::tip
 If you want to use the **(unstable!)** bleeding edge, you can use `ghcr.io/opendevin/opendevin:main` as the image (last line).
 :::
 
-See [Development.md](https://github.com/OpenDevin/OpenDevin/blob/main/Development.md) for instructions on running OpenDevin without Docker.
+See Development.md for instructions on running OpenDevin without Docker.
 
-Are you having trouble? Check out our [Troubleshooting Guide](https://opendevin.github.io/OpenDevin/modules/usage/troubleshooting).
+Having trouble? Check out our Troubleshooting Guide.
 
 :::warning
 OpenDevin is currently a work in progress, but you can already run the alpha version to see the end-to-end system in action.

docs/src/components/Code/Code.tsx

@@ -0,0 +1,45 @@
+import Link from "@docusaurus/Link";
+import { Header } from "@site/src/pages";
+import { CodeBlock } from "./CodeBlock";
+import styles from "./styles.module.css";
+
+export function Code() {
+  const workspaceCode = `# The directory you want OpenDevin to modify. MUST be an absolute path!
+export WORKSPACE_BASE=$(pwd)/workspace`;
+
+  const dockerCode = `docker run \\
+    -it \\
+    --pull=always \\
+    -e SANDBOX_USER_ID=$(id -u) \\
+    -e WORKSPACE_MOUNT_PATH=$WORKSPACE_BASE \\
+    -v $WORKSPACE_BASE:/opt/workspace_base \\
+    -v /var/run/docker.sock:/var/run/docker.sock \\
+    -p 3000:3000 \\
+    --add-host host.docker.internal:host-gateway \\
+    ghcr.io/opendevin/opendevin:0.5`;
+
+  return (
+    <div className={styles.container}>
+      <div className={styles.innerContainer}>
+        <div className={styles.header}>
+          <Header
+            title="Getting Started"
+            summary="Getting Started"
+            description="Get started using OpenDevin in just a few lines of code"
+          ></Header>
+          <div className={styles.buttons}>
+            <Link
+              className="button button--secondary button--lg"
+              to="/modules/usage/intro"
+            >
+              Learn More
+            </Link>
+          </div>
+        </div>
+        <br />
+        <CodeBlock language="python" code={workspaceCode} />
+        <CodeBlock language="python" code={dockerCode} />
+      </div>
+    </div>
+  );
+}

docs/src/components/Code/CodeBlock.tsx

@@ -0,0 +1,63 @@
+import { useColorMode } from "@docusaurus/theme-common";
+import { Highlight, themes } from "prism-react-renderer";
+import { useCopyToClipboard } from "react-use";
+
+interface CodeBlockProps {
+  language: string;
+  code: string;
+}
+
+export function CodeBlock({ language, code }: CodeBlockProps) {
+  const [state, copyToClipboard] = useCopyToClipboard();
+  const { isDarkTheme } = useColorMode();
+
+  const copyCode = () => {
+    copyToClipboard(code);
+  };
+
+  return (
+    <div
+      style={{
+        position: "relative",
+      }}
+    >
+      <Highlight
+        theme={isDarkTheme ? themes.vsLight : themes.vsDark}
+        code={code}
+        language={language}
+      >
+        {({ style, tokens, getLineProps, getTokenProps }) => (
+          <pre style={style}>
+            {tokens.map((line, i) => (
+              <div key={i} {...getLineProps({ line })}>
+                <span
+                  style={{
+                    display: "inline-block",
+                    width: "3em",
+                    color: "var(--gray)",
+                  }}
+                >
+                  {i + 1}
+                </span>
+                {line.map((token, key) => (
+                  <span key={key} {...getTokenProps({ token })} />
+                ))}
+              </div>
+            ))}
+          </pre>
+        )}
+      </Highlight>
+      <button
+        className="button button--secondary"
+        style={{
+          position: "absolute",
+          top: "10px",
+          right: "10px",
+        }}
+        onClick={copyCode}
+      >
+        {state.value ? "Copied!" : "Copy"}
+      </button>
+    </div>
+  );
+}

docs/src/components/Code/styles.module.css

@@ -0,0 +1,26 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  padding-top: 25px;
+  padding-bottom: 25px;
+  width: 100%;
+}
+
+.innerContainer {
+  padding: 50px;
+  width: 100%;
+  max-width: 1300px;
+  padding-top: 30px;
+  margin: auto;
+}
+
+.header {
+  display: flex;
+  justify-content: space-between;
+}
+
+@media (max-width: 768px) {
+  .header {
+    flex-direction: column;
+  }
+}

docs/src/pages/faq.tsx

@@ -50,26 +50,6 @@ export default function FAQ() {
           scenarios, producing works that significantly contribute to the
           community and pave the way for future advancements.
         </p>
-        <h3>How to fix an issue on OpenDevin?</h3>
-        <p>
-          To fix an issue on GitHub using OpenDevin, send a prompt to OpenDevin asking it to follow these steps:
-          <ol>
-            <li>Read the issue on <a href="https://github.com/OpenDevin/OpenDevin/issues/1611">GitHub</a></li>
-            <li>Clone the repository and check out a new branch</li>
-            <li>Based on the instructions in the issue description, modify files to fix the issue</li>
-            <li>Push the resulting output to GitHub using the GITHUB_TOKEN environment variable</li>
-            <li>Tell me the link that I need to go to to send a pull request</li>
-          </ol>
-          Before you run OpenDevin, you can do:
-          <pre>
-            export SANDBOX_ENV_GITHUB_TOKEN=XXX
-          </pre>
-          where XXX is a GitHub token that you created that has permissions to push to the OpenDevin repo. If you don’t have write permission to the OpenDevin repo, you might need to change that to:
-          <pre>
-            4. Push the resulting output to my fork at https://github.com/USERNAME/OpenDevin/ using the GITHUB_TOKEN environment variable
-          </pre>
-          where USERNAME is your GitHub username.
-        </p>
       </div>
     </Layout>
   );

docs/src/pages/index.tsx

@@ -1,6 +1,7 @@
 import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
 import Layout from "@theme/Layout";
 
+import { Code } from "../components/Code/Code";
 import { HomepageHeader } from "../components/HomepageHeader/HomepageHeader";
 import { Welcome } from "../components/Welcome/Welcome";
 
@@ -24,6 +25,7 @@ export default function Home(): JSX.Element {
         <HomepageHeader />
         <div>
           <Welcome />
+          <Code />
         </div>
       </div>
     </Layout>

evaluation/EDA/README.md

@@ -1,44 +0,0 @@
-# EDA Evaluation
-
-This folder contains evaluation harness for evaluating agents on the Entity-deduction-Arena Benchmark, from the paper [Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games](https://arxiv.org/abs/2310.01468), presented in ACL 2024 main conference.
-
-## Configure OpenDevin and your LLM
-
-Create a `config.toml` file if it does not exist at the root of the workspace. Please check [README.md](../../README.md) for how to set this up.
-
-## Start the evaluation
-
-
-```bash
-export OPENAI_API_KEY="sk-XXX"; # This is required for evaluation (to simulate another party of conversation)
-./evaluation/EDA/scripts/run_infer.sh [model_config] [agent] [dataset] [eval_limit]
-```
-
-where `model_config` is mandatory, while `agent`, `dataset` and `eval_limit` are optional.
-
-- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`.
-
-- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-
-- `dataset`: There are two tasks in this evaluation. Specify `dataset` to test on either `things` or `celebs` task.
-
-- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By default it infers all instances.
-
-Let's say you'd like to run 10 instances using `eval_gpt4_1106_eval_gpt4o_2024_05_13preview` and CodeActAgent,
-then your command would be:
-
-```bash
-./evaluation/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 CodeActAgent things
-```
-
-## Reference
-```
-@inproceedings{zhang2023entity,
-  title={Probing the Multi-turn Planning Capabilities of LLMs via 20 Question Games},
-  author={Zhang, Yizhe and Lu, Jiarui and Jaitly, Navdeep},
-  journal={ACL},
-  year={2024}
-}
-```

evaluation/EDA/game.py

@@ -1,413 +0,0 @@
-import json
-import logging
-import os
-import re
-from typing import Optional
-
-import openai
-import requests.exceptions
-import torch
-from openai import OpenAI
-from retry import retry
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-LOGGER = logging.getLogger(__name__)
-
-
-def load_model(path):
-    print('Loading model...')
-    tokenizer = AutoTokenizer.from_pretrained(path, use_fast=False)
-    print('Tokenizer loaded.')
-    model = AutoModelForCausalLM.from_pretrained(
-        path, low_cpu_mem_usage=True, torch_dtype=torch.float16
-    ).cuda()
-    print('Model loaded.')
-    # model.half().cuda()
-    return model, tokenizer
-
-
-class Q20Game:
-    def __init__(
-        self,
-        item: str,
-        answerer_model: str = 'gpt-3.5-turbo-0613',
-        guesser_model: str = 'gpt-3.5-turbo-0613',
-        num_turns: int = 20,
-        temperature: float = 0.8,
-        openai_api: bool = True,
-        openai_api_key: Optional[str] = None,
-        guesser_kargs={},
-    ) -> None:
-        self.item = item
-        self.answerer_model = answerer_model
-        self.guesser_model = guesser_model
-        self.num_turns = num_turns
-        self.temperature = temperature
-        self.openai_api = openai_api
-        self.guesser_kargs = guesser_kargs
-        self.vicuna_prompt = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
-        self.first_user_utterance = (
-            'Your task is to ask a series of questions to deduce the entity '
-            "that I'm thinking of with as few queries as possible. "
-            "Only ask questions that can be answered by 'yes', 'no' or 'maybe'. "
-            'Do not ask for hint. Make your question brief with no linebreaker. '
-            'Now start asking a question.'
-        )
-        self.guesser_win = False
-        self.curr_turn = 0
-        if openai_api_key is not None:
-            openai.api_key = openai_api_key
-
-        if isinstance(answerer_model, str) and not answerer_model.startswith('gpt'):
-            self.user_api_base = 'http://0.0.0.0:8000/v1'
-        else:
-            self.user_api_base = 'https://api.openai.com/v1'
-
-        if isinstance(guesser_model, str) and not guesser_model.startswith('gpt'):
-            self.guesser_api_base = 'http://0.0.0.0:8000/v1'
-        else:
-            self.guesser_api_base = 'https://api.openai.com/v1'
-
-        self.guesser_messages = []
-
-    def confusion_matrix(self, path):
-        self.reset()
-        with open(path) as f:
-            raw_messages = json.load(f)
-            self.item = path.split('/')[-1].split('_')[0]
-            roles = ['assistant', 'user']
-            for i, message in enumerate(raw_messages):
-                self.guesser_messages.append(
-                    {'role': roles[i % 2], 'content': message['content']}
-                )
-
-        self.guesser_messages = self.guesser_messages[:-2]
-        self.guesser_messages[-1]['content'] = (
-            self.guesser_messages[-1]['content'] + " You must guess now, what's it?"
-        )
-        guesser_msg = self.guesser(self.guesser_messages)
-        self.guesser_messages.append(guesser_msg)
-        guesser_question = guesser_msg['content'].strip()
-        self.guesser_messages[-1]['content'] = (
-            self.guesser_messages[-1]['content'] + ' Is it right?'
-        )
-        usr_msg = self.answerer(guesser_question)
-        self.guesser_messages.append(
-            {'role': 'user', 'content': f"{usr_msg['content'].strip()}"}
-        )
-
-        if 'bingo' in self.guesser_messages[-1]['content'].lower():
-            self.guesser_win = True
-            return True
-
-        return False
-
-    @retry(
-        (
-            openai.Timeout,
-            requests.exceptions.ReadTimeout,
-            openai.RateLimitError,
-            openai.APIError,
-            requests.exceptions.HTTPError,
-            openai.APIConnectionError,
-        ),
-        tries=5,
-        delay=0.5,
-        backoff=0.5,
-        max_delay=2,
-        logger=LOGGER,
-    )
-    def guesser(self, messages):
-        if not self.guesser_model.startswith('gpt'):  # hf model
-            self.guesser_model, self.guesser_tokenizer = load_model(self.guesser_model)
-
-            # """Wraps hf's `generate` adding some specific method's defaults"""
-            assert not self.openai_api
-            prompt = self.dialog_history() + ' ASSISTANT:'
-            input_ids = torch.tensor(
-                [self.guesser_tokenizer.encode(prompt, add_special_tokens=True)]
-            )  # TODO check if huggingface is using the same format.
-            input_ids = input_ids.to(self.guesser_model.base_model.device)
-            attention_mask = None
-
-            with torch.no_grad():
-                gen = self.guesser_model.generate(
-                    input_ids=input_ids,
-                    attention_mask=attention_mask,
-                    **self.guesser_kargs,
-                )
-                gen_str = (
-                    self.guesser_tokenizer.decode(gen[0][input_ids[0].shape[0] :])
-                    .split('</s>')[0]
-                    .split('USER')[0]
-                    .lstrip()
-                    .strip()
-                )
-
-                return {
-                    'role': 'assistant',
-                    'content': gen_str,
-                }
-        else:
-            openai.api_base = self.guesser_api_base
-            client = OpenAI(api_key=openai.api_key)
-            response = client.chat.completions.create(
-                model=self.guesser_model,
-                messages=messages,
-                max_tokens=64,
-                n=1,
-                stop=None,
-                temperature=self.temperature,
-            )
-            return {
-                'role': 'assistant',
-                'content': response.choices[0].message.to_dict()['content'].strip(),
-            }
-
-    def dialog_history(self):
-        history = self.vicuna_prompt + ' '
-        for item in self.guesser_messages:
-            if item['role'].upper() == 'USER':
-                history += 'USER: ' + item['content']
-            elif item['role'].upper() == 'ASSISTANT':
-                history += ' ' + 'ASSISTANT: ' + item['content'] + '</s>'
-        return history
-
-
-    def preprocess_response(self,response):
-        response = re.sub(
-            r'the entity you are thinking of', 'it', response
-        )
-        response = re.sub(
-            r"the entity you're thinking of", 'it', response
-        )
-        response = re.sub(
-            r" you're thinking of", '', response
-        )
-        response = re.sub(
-            r' you are thinking of', '', response
-        )
-        self.guesser_messages.append(response)
-        return response
-
-    def judge_winner(self, response):
-        guesser_question = response.strip()
-
-        if self.curr_turn == self.num_turns - 1:
-            guesser_question += ' Is it right?'
-        # ask for answer
-        usr_msg = self.answerer(guesser_question)
-
-        if 'bingo' in usr_msg['content'].lower():
-            self.guesser_win = True
-            return True, ""
-        
-        return False, usr_msg['content'].strip()
-    
-    def generate_user_response(self, response):
-        response = self.preprocess_response(response)
-        # others
-        bingo, anwser_reply = self.judge_winner(response)
-        if bingo:
-            return "You are bingo! quit now, run: <execute_bash> exit </execute_bash>.\n"
-        if self.curr_turn == self.num_turns - 2:
-            anwser_reply += " You must guess now, what's it?"
-        return anwser_reply
-
-    def game_play(self, user_mode=False):
-        self.reset()
-        # print(f"Item: {self.item}")
-        for t in range(self.num_turns):
-            # System asking a question
-            if (not user_mode) or user_mode is None:
-                guesser_msg = self.guesser(self.guesser_messages)
-                guesser_msg['content'] = re.sub(
-                    r'the entity you are thinking of', 'it', guesser_msg['content']
-                )
-                guesser_msg['content'] = re.sub(
-                    r"the entity you're thinking of", 'it', guesser_msg['content']
-                )
-                guesser_msg['content'] = re.sub(
-                    r" you're thinking of", '', guesser_msg['content']
-                )
-                guesser_msg['content'] = re.sub(
-                    r' you are thinking of', '', guesser_msg['content']
-                )
-            else:
-                user_q = input(
-                    f'Type in your questions for turn {t+1}. (e.g. Is it a living thing?)\n'
-                )
-                guesser_msg = {'role': 'assistant', 'content': user_q}
-            self.guesser_messages.append(guesser_msg)
-            guesser_question = guesser_msg['content'].strip()
-
-            if t == self.num_turns - 1:
-                self.guesser_messages[-1]['content'] = (
-                    self.guesser_messages[-1]['content'] + ' Is it right?'
-                )
-
-            usr_msg = self.answerer(guesser_question)
-            self.guesser_messages.append(
-                {'role': 'user', 'content': f"{usr_msg['content'].strip()}"}
-            )
-
-            if 'bingo' in usr_msg['content'].lower():
-                self.guesser_win = True
-                return True
-
-            if t == self.num_turns - 2:
-                self.guesser_messages[-1]['content'] = (
-                    self.guesser_messages[-1]['content']
-                    + " You must guess now, what's it?"
-                )
-
-        return False
-
-    def save_session(self, path):
-        # Print the conversation
-        if not os.path.exists(path):
-            os.makedirs(path)
-        output_file = os.path.join(path, f'{self.item}.txt')
-        with open(output_file, 'w') as out_f:
-            out_f.write(f'item: {self.item}\n')
-            for t, message in enumerate(self.guesser_messages):
-                out_f.write(
-                    f"Turn {(t+1)//2}, {message['role'].capitalize()}: {message['content'].lstrip()}\n"
-                )
-
-    def reward(self):
-        if self.guesser_win:
-            n_turns = (len(self.guesser_messages) + 1) // 2
-            return 1 - max(n_turns - 5, 0) * 0.02
-        return 0
-
-    def num_success(self):
-        return 1 if self.guesser_win else 0
-
-    def num_yes(self):
-        n_yes = sum(
-            ['yes' in msg['content'].lower() for msg in self.guesser_messages[2::2]]
-        )
-        return n_yes
-
-    @retry(
-        (
-            openai.Timeout,
-            requests.exceptions.ReadTimeout,
-            openai.RateLimitError,
-            openai.APIError,
-            openai.APIConnectionError,
-        ),
-        tries=5,
-        delay=0.5,
-        backoff=0.5,
-        max_delay=2,
-        logger=LOGGER,
-    )
-    def answerer(self, question):
-        openai.api_base = self.user_api_base
-        client = OpenAI(api_key=openai.api_key)
-        user_messages = [
-            {
-                'role': 'user',
-                'content': f'Based on your knowledge about {self.item}, '
-                f'respond to the following question or guess. '
-                f"Limit your respond to only 'Yes.', 'No.' or 'Maybe.', with no explanation or other words. "
-                f'Never say the answer {self.item} in your response. '
-                f"If the question is to solicit the answer, respond 'No.'.",
-            },
-            {
-                'role': 'user',
-                'content': f'For the entity {self.item}, {question} (Yes/No/Maybe)',
-            },
-        ]
-
-        response = client.chat.completions.create(
-            model=self.answerer_model,
-            messages=user_messages,
-            max_tokens=6,
-            n=1,
-            stop=None,
-            temperature=0.2,
-        )
-        if any(
-            [
-                re.search(rf'(?:^|\W){i.strip().lower()}(?:$|\W)', question.lower())
-                for i in self.item.lower().split('|')
-            ]
-        ):
-            response.choices[0].message.content = 'Bingo!'
-        return response.choices[0].message.to_dict()
-
-    def reset(self):
-        # Initialize the conversation
-        self.curr_turn = 0
-        self.guesser_messages = [
-            {
-                'role': 'user',
-                'content': self.first_user_utterance,
-            }
-        ]
-
-
-class Q20GameCelebrity(Q20Game):
-    def __init__(self, item: str, **kwargs) -> None:
-        super().__init__(item, **kwargs)
-        self.first_user_utterance = (
-            'Your task is to ask a series of questions to deduce the celebrity '
-            "that I'm thinking of with as few queries as possible. "
-            "Only ask factual questions that can be answered by 'Yes.', 'No.' or 'Dunno.'. Do not ask for hint. Make your question brief with no linebreaker. "
-            'Now start asking a question.'
-        )
-
-    @retry(
-        (
-            openai.Timeout,
-            requests.exceptions.ReadTimeout,
-            openai.RateLimitError,
-            openai.APIError,
-            openai.APIConnectionError,
-        ),
-        tries=5,
-        delay=0.5,
-        backoff=0.5,
-        max_delay=2,
-        logger=LOGGER,
-    )
-    def answerer(self, question):
-        openai.api_base = self.user_api_base
-        user_messages = [
-            {
-                'role': 'system',
-                'content': f'Based on on your knowledge about the celebrity: {self.item}, '
-                f'respond to the following question or guess. '
-                f"Limit your respond to only 'Yes.', 'No.' or 'Dunno.', with no explanation or other words. "
-                f"Never say the name {self.item} in your response. Do not say 'Dunno.' if it can be answered by 'Yes.' or 'No.' "
-                f"If the question is to solicit the answer, respond 'No.'.",
-            },
-            {
-                'role': 'user',
-                'content': f'For the celebrity {self.item}, {question}(Yes/No/Dunno)',
-            },
-        ]
-
-        response = openai.ChatCompletion.create(
-            model=self.answerer_model,
-            messages=user_messages,
-            max_tokens=6,
-            n=1,
-            stop=None,
-            temperature=0.2,
-        )
-        if re.search(rf'(?:^|\W){self.item.lower()}(?:$|\W)', question.lower()):
-            response.choices[0].message.content = 'Bingo!'
-        return response.choices[0].message.to_dict()
-
-    def reset(self):
-        # Initialize the conversation
-        self.guesser_messages = [
-            {
-                'role': 'user',
-                'content': self.first_user_utterance,
-            }
-        ]

evaluation/EDA/run_infer.py

@@ -1,329 +0,0 @@
-import asyncio
-import json
-import logging
-import multiprocessing as mp
-import os
-import pathlib
-import subprocess
-import time
-from concurrent.futures import ProcessPoolExecutor
-
-# import huggingface_hub
-from datasets import load_dataset
-from tqdm import tqdm
-
-from evaluation.EDA.game import Q20Game, Q20GameCelebrity
-
-# from evaluation.EDA.scorer import question_scorer
-from opendevin.controller.state.state import State
-from opendevin.core.config import config, get_llm_config_arg, get_parser
-from opendevin.core.logger import get_console_handler
-from opendevin.core.logger import opendevin_logger as logger
-from opendevin.core.main import main
-from opendevin.events.action import MessageAction
-from opendevin.events.serialization.event import event_to_dict
-
-game = None
-
-
-def cleanup():
-    print('Cleaning up child processes...')
-    for process in mp.active_children():
-        print(f'Terminating child process: {process.name}')
-        process.terminate()
-        process.join()
-
-
-def codeact_user_response(state: State) -> str:
-    global game
-    model_guess = ''
-    if state.history:
-        for act, _ in reversed(state.history):
-            if isinstance(act, MessageAction) and act.source == 'agent':
-                model_guess = act.content
-                break
-    msg = game.generate_user_response(model_guess)
-    game.curr_turn += 1
-    logger.info(f'Model guess: {model_guess}')
-    logger.info(f'Anwser response: {msg}')
-    return msg
-
-
-def monologue_user_response(state: State) -> str:
-    raise NotImplementedError('MonologueAgent should never ask for user responses.')
-
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-    'MonologueAgent': monologue_user_response,
-}
-
-AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have solved the question, please first send your answer to user through message and then exit.\n'
-}
-
-
-def process_instance(instance, agent_class, metadata, reset_logger: bool = True):
-    # Setup the logger properly, so you can run multi-processing to parallize the evaluation
-    eval_output_dir = metadata['eval_output_dir']
-    if reset_logger:
-        # Set up logger
-        log_file = os.path.join(
-            eval_output_dir, 'logs', f'instance_{instance["text"].strip()}.log'
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        # add back the console handler to print ONE line
-        logger.addHandler(get_console_handler())
-        logger.info(
-            f'Starting evaluation for instance {instance["text"].strip()}.\nLOG:   tail -f {log_file}'
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setFormatter(
-            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        )
-        logger.addHandler(file_handler)
-
-    # Prepare instruction
-    _game_class = {'things': Q20Game, 'celebs': Q20GameCelebrity}
-
-    guesser_kargs = {
-        'max_new_tokens': 64,
-        'temperature': 0.8,
-        'repetition_penalty': 1.0,
-        'do_sample': True,
-    }  # no penalty
-
-    # Use codeactagent as guesser_model
-    global game
-    game = _game_class[metadata['dataset']](
-        item=instance['text'].strip(),
-        answerer_model=metadata['answerer_model'],
-        guesser_model=None,
-        num_turns=metadata['max_iterations'],
-        openai_api_key=metadata['openai_api'],
-        guesser_kargs=guesser_kargs,
-    )
-
-    instruction = f'{game.first_user_utterance}'
-    logger.info(f'Instruction: {instruction}')
-
-    # instruction += 'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
-    # NOTE: You can actually set slightly different instruction for different agents
-    instruction += AGENT_CLS_TO_INST_SUFFIX.get(agent_class, '')
-
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
-
-    state: State = asyncio.run(
-        main(
-            instruction,
-            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(agent_class),
-        )
-    )
-    # ======= Attempt to evaluate the agent's edits =======
-    # If you are working on simplier benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    final_message = ''
-    for act, _ in reversed(state.history):
-        if isinstance(act, MessageAction) and act.source == 'agent':
-            final_message = act.content
-            break
-
-    logger.info(f'Final message: {final_message} | Ground truth: {instance["text"]}')
-    test_result = game.reward()
-
-    # Save the output
-    output = {
-        'instance_id': instance['text'].strip(),
-        'instance': instance,
-        'instruction': instruction,
-        'metadata': metadata,
-        'history': [
-            (event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
-        ],
-        'error': state.error if state and state.error else None,
-        'test_result': {
-            'success': test_result,
-            'final_message': final_message,
-            'ground_truth': instance['text'],
-        },
-    }
-
-    return output
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--answerer_model', '-a', default='gpt-3.5-turbo', help='answerer model'
-    )
-    parser.add_argument(
-        '--dataset',
-        default='things',
-        choices=['things', 'celebs'],
-        type=str,
-        help='dataset to be used',
-    )
-    parser.add_argument(
-        '--OPENAI_API_KEY', type=str, required=True, help='Your OpenAI API key'
-    )
-    parser.add_argument(
-        '--data-split',
-        default='test',
-        type=str,
-        help='data split, eg, test',
-    )
-    args, _ = parser.parse_known_args()
-    if args.directory:
-        config.workspace_base = os.path.abspath(args.directory)
-        print(f'Setting workspace base to {config.workspace_base}')
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenDevin's repo
-    eda_dataset = load_dataset(
-        'yizheapple/entity-deduction-arena', name=args.dataset, split=args.data_split
-    )
-    logger.info(
-        f'Evaluating Entity Deduction Arena {args.dataset} {args.data_split} split'
-    )
-
-    # Check https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/swe_bench/README.md#configure-opendevin-and-your-llm
-    # for details of how to set `llm_config`
-    if args.llm_config:
-        specified_llm_config = get_llm_config_arg(args.llm_config)
-        if specified_llm_config:
-            config.llm = specified_llm_config
-    logger.info(f'Config for evaluation: {config}')
-
-    # TEST METADATA
-    agent_class = args.agent_cls
-    assert (
-        agent_class in AGENT_CLS_TO_FAKE_USER_RESPONSE_FN
-    ), f'Unsupported agent class: {agent_class}'
-    model_name = config.llm.model.split('/')[-1]
-    max_iterations = args.max_iterations
-    eval_note = ''
-    if args.eval_note is not None:
-        eval_note += '_N_' + args.eval_note
-    eval_output_dir = os.path.join(
-        args.eval_output_dir,
-        'eda',
-        agent_class,
-        model_name + '_maxiter_' + str(max_iterations) + eval_note,
-    )
-
-    pathlib.Path(eval_output_dir).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(os.path.join(eval_output_dir, 'logs')).mkdir(
-        parents=True, exist_ok=True
-    )
-    logger.info(f'Using evaluation output directory: {eval_output_dir}')
-
-    metadata = {
-        'dataset': args.dataset,
-        'data_split': args.data_split,
-        'answerer_model': args.answerer_model,
-        'agent_class': agent_class,
-        'openai_api': args.OPENAI_API_KEY,
-        'model_name': model_name,
-        'max_iterations': max_iterations,
-        'eval_output_dir': eval_output_dir,
-        'start_time': time.strftime('%Y-%m-%d %H:%M:%S'),
-        # get the commit id of current repo for reproduciblity
-        'git_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'])
-        .decode('utf-8')
-        .strip(),
-    }
-    logger.info(f'Metadata: {metadata}')
-    with open(os.path.join(eval_output_dir, 'metadata.json'), 'w') as f:
-        json.dump(metadata, f)
-
-    # LIMIT EVALUATION
-    eval_n_limit = args.eval_n_limit
-    if eval_n_limit:
-        eda_dataset = eda_dataset.select(list(range(eval_n_limit)))
-        logger.info(f'Limiting evaluation to first {eval_n_limit} instances.')
-
-    # OUTPUT FILE
-    output_file = os.path.join(eval_output_dir, 'output.jsonl')
-    logger.info(f'Writing evaluation output to {output_file}')
-    finished_items = set()
-    if os.path.exists(output_file):
-        with open(output_file, 'r') as f:
-            for line in f:
-                data = json.loads(line)
-                finished_items.add(data['instance_id'])
-        logger.warning(
-            f'Output file {output_file} already exists. Loaded {len(finished_items)} finished instances.'
-        )
-    output_fp = open(output_file, 'a')
-
-    logger.info(
-        f'Evaluation started with Agent {agent_class}, model {model_name}, max iterations {max_iterations}.'
-    )
-
-    # =============================================
-    # filter out finished instances
-    new_eda_dataset = []
-    for instance in eda_dataset:
-        if instance['text'].strip() in finished_items:
-            logger.info(
-                f'Skipping instance {instance["text"].strip()} as it is already finished.'
-            )
-            continue
-        new_eda_dataset.append(instance)
-
-    eda_dataset = new_eda_dataset
-    logger.info(
-        f'Finished instances: {len(finished_items)}, Remaining instances: {len(eda_dataset)}'
-    )
-    # =============================================
-
-    pbar = tqdm(total=len(eda_dataset))
-
-    # This function tracks the progress AND write the output to a JSONL file
-    def update_progress(future):
-        pbar.update(1)
-        output = future.result()
-        pbar.set_description(f'Instance {output["instance_id"]}')
-        pbar.set_postfix_str(f'Test Result: {output["test_result"]}')
-        logger.info(
-            f'Finished evaluation for instance {output["instance_id"]}: {output["test_result"]}'
-        )
-        output_fp.write(json.dumps(output) + '\n')
-        output_fp.flush()
-
-    # This sets the multi-processing
-    num_workers = args.eval_num_workers
-    logger.info(f'Using {num_workers} workers for evaluation.')
-
-    try:
-        with ProcessPoolExecutor(num_workers) as executor:
-            futures = []
-            # This is how we perform multi-processing
-            for instance in eda_dataset:
-                future = executor.submit(
-                    process_instance,
-                    instance,
-                    agent_class,
-                    metadata,
-                    reset_logger=bool(num_workers > 1),
-                )
-                future.add_done_callback(update_progress)
-                futures.append(future)
-
-            # Wait for all futures to complete
-            for future in futures:
-                future.result()
-    except KeyboardInterrupt:
-        print('KeyboardInterrupt received. Cleaning up...')
-        cleanup()
-
-    output_fp.close()
-    logger.info('Evaluation finished.')

evaluation/EDA/scripts/run_infer.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-MODEL_CONFIG=$1
-AGENT=$2
-DATASET=$3
-EVAL_LIMIT=$4
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-if [ -z "$DATASET" ]; then
-  echo "Dataset not specified, use default 'things'"
-  DATASET="things"
-fi
-
-# check if OPENAI_API_KEY is set
-if [ -z "$OPENAI_API_KEY" ]; then
-  echo "OPENAI_API_KEY is not set, please set it to run the script"
-  exit 1
-fi
-
-# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenDevin
-# We need to track the version of Agent in the evaluation to make sure results are comparable
-AGENT_VERSION=v$(poetry run python -c "import agenthub; from opendevin.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-echo "DATASET: $DATASET"
-
-COMMAND="poetry run python evaluation/EDA/run_infer.py \
-  --agent-cls $AGENT \
-  --llm-config $MODEL_CONFIG \
-  --dataset $DATASET \
-  --data-split test \
-  --max-iterations 20 \
-  --OPENAI_API_KEY $OPENAI_API_KEY \
-  --max-chars 10000000 \
-  --eval-num-workers 1 \
-  --eval-note ${AGENT_VERSION}_${DATASET}"
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND

evaluation/README.md

@@ -13,9 +13,6 @@ all the preprocessing/evaluation/analysis scripts.
 ## Supported Benchmarks
 
 - SWE-Bench: [`evaluation/swe_bench`](./swe_bench)
-- HumanEvalFix: [`evaluation/humanevalfix`](./humanevalfix)
-- GAIA: [`evaluation/gaia`](./gaia)
-- Entity deduction Arena (EDA): [`evaluation/EDA`](./EDA)
 
 ### Result Visualization
 

evaluation/TUTORIAL.md

@@ -1,182 +0,0 @@
-# Tutorial: How to add a New Evaluation Benchmark to OpenDevin
-
-This tutorial provides a general guide on how to integrate your own evaluation benchmark into the OpenDevin framework.
-
-You can read this for details, and also learn by example by looking at our existing evaluations:
-- [swe_bench](swe_bench/)
-
-
-## A quick walk-through of OpenDevin architecture
-
-### Before everything begins
-
-Please follow [this document](https://github.com/OpenDevin/OpenDevin/blob/main/Development.md) to setup local develop environment for OpenDevin.
-
-### Configuration file
-
-OpenDevin uses `config.toml` to keep track of most configurations.
-
-Here's an example configuration file you can use:
-
-```toml
-[core]
-max_iterations = 100
-cache_dir = "/tmp/cache"
-
-# IMPORTANT: You should set these two paths to YOUR WORKSPACE directory,
-# which will be mounted into Sandbox for agent to interact with!
-# The OpenDevin agent will be able to read/write files whatever they like (even rm -rf)
-# in this directory, so be careful!!
-workspace_base = "/path/to/your/workspace"
-workspace_mount_path = "/path/to/your/workspace"
-# ==========================
-
-sandbox_container_image = "ghcr.io/opendevin/sandbox:latest"
-sandbox_type = "ssh"
-sandbox_timeout = 120
-ssh_hostname = "localhost"
-
-# SWEBench eval specific - but you can tweak it to your needs
-use_host_network = false
-run_as_devin = false
-# linting python after editing helps LLM fix indentations
-enable_auto_lint = true
-
-[llm]
-# IMPORTANT: add your API key here, and set the model to the one you want to evaluate
-model = "gpt-4o-2024-05-13"
-api_key = "sk-XXX"
-```
-
-### How to use OpenDevin programmatically
-
-In this section, for the purpose of building an evaluation task, we don't use the standard OpenDevin web-based GUI, but rather run OpenDevin backend from CLI.
-
-For example, you can run the following, which performs the specified task `-t`, with a particular model `-m` and agent `-c`, for a maximum number of iterations `-i`:
-
-```bash
-poetry run python ./opendevin/core/main.py \
-        -i 10 \
-        -t "Write me a bash script that print hello world." \
-        -c CodeActAgent \
-        -m gpt-4o-2024-05-13
-```
-
-After running the script, you will observe the following:
-
-![](./static/example_task_1.png)
-
-You can see the agent uses bash to write a script, makes it executable, and then tests it by running it to make sure it is working.
-
-At the end of the above screenshot, OpenDevin actually requests user inputs when it think it finishes the task. This will cause issues in evaluation, since most evaluation don't assume additional user input. To fix this, we introduce the functionality of `fake_user_response_fn` in the `main` function, which we describe in the next section.
-
-## The `main` function
-
-The signature of `main` (in file [[`opendevin/core/main.py`](../opendevin/core/main.py)]) is as follows:
-
-```python
-async def main(
-    task_str: str = '',
-    exit_on_message: bool = False,
-    fake_user_response_fn: Optional[Callable[[Optional[State]], str]] = None,
-    sandbox: Optional[Sandbox] = None,
-) -> Optional[State]:
-```
-
-- `task_str`: The task instruction to run. In the above example, it is "Write me a bash script that print hello world."
-- `exit_on_message`: whether to quit if the agent asks for a message from user
-- `fake_user_response_fn`: An optional function that receives the current state (could be None) and returns a fake user response.
-- `sandbox`: An optional sandbox to run the agent in.
-
-### `fake_user_response_fn`
-
-Here's an example of `fake_user_response_fn` in the implementation for SWE-Bench in [`evaluation/swe_bench/run_infer.py`](swe_bench/run_infer.py):
-
-```python
-def codeact_user_response(state: State) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have modified the code in a way that fixes the issue, please run the following command: <execute_bash> exit </execute_bash>.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
-    )
-    if state.history:
-        user_msgs = [
-            action
-            for action, _ in state.history
-            if isinstance(action, MessageAction) and action.source == 'agent'
-        ]
-        if len(user_msgs) >= 2:
-            # let the agent know that it can give up when it has tried 3 times
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-```
-
-
-### Return value
-
-The main function returns a `State`, which is defined in [`opendevin/controller/state/state.py`](../opendevin/controller/state/state.py). We are mainly using `state.history` here, which is the most important field of data. You can imagine it is being a more structured version of OpenAI's chat completion [messages](https://platform.openai.com/docs/guides/text-generation/chat-completions-api).
-
-`history: list[tuple[Action, Observation]] = field(default_factory=list)` is a list of (action, observation) tuple. All the actions are defined at [`opendevin/events/action`](../opendevin/events/action) and observations are defined at [`opendevin/events/observation`](../opendevin/events/action).
-
-The agent can emit different actions like `CmdRunAction`  (`opendevin/events/action/commands.py`) to execute bash commands and receive `CmdOutputObservation` (`opendevin/events/observation/commands.py`), `IPythonRunCellAction` to receive `IPythonRunCellObservation`, `BrowseInteractiveAction` (`opendevin/events/action/browse.py`) to browse the web and receive `BrowserOutputObservation` (`opendevin/events/observation/browse.py`).
-
-The action we used in this example is `MessageAction` (`opendevin/events/action/message.py`), which actually denotes a message from either `agent` or `user`. In the [CodeAct agent example](https://github.com/OpenDevin/OpenDevin/blob/7ca560471bd262f22513f3863995d0a8e6121c07/agenthub/codeact_agent/codeact_agent.py#L239-L273), an agent is considered to emit a `MessageAction` when it does not trigger a `CmdRunAction`, `IPythonRunCellAction`, and/or `BrowseInteractiveAction`.
-
-Typically, the agent returns `MessageAction` when it is confused about the task, and want to ask human for follow-up clarification, which is a good thing in real-world task, but not necessarily in evaluation. So in this example, we provide a dummy prompt to tell the agent "Please continue working on the task on whatever approach you think is suitable[...]".
-
-If you see something like this, you can consider adding this to your evaluation pipeline as well.
-
-### `sandbox`
-
-Sandbox is a fully functioning docker container where the agent can perform all sorts of tasks, e.g., using bash, calling Python, install packages, and more. You can leave `sandbox` to `None` if you don't need to do anything special to pre-configure the `Sandbox`.
-
-In SWE-Bench, we need to copy the proper repository directory to the workspace and activate the right python virtual environment before the agent can start performing the task, so we actually defined a custom [`SWEBenchSSHBox`](https://github.com/OpenDevin/OpenDevin/blob/7ca560471bd262f22513f3863995d0a8e6121c07/evaluation/swe_bench/swe_env_box.py#L12-L118) that inherit from the default sandbox [`SSHBox`](https://github.com/OpenDevin/OpenDevin/blob/7ca560471bd262f22513f3863995d0a8e6121c07/opendevin/runtime/docker/ssh_box.py#L188) and handles all these initial setup. If you need to configure the `sandbox` for your evaluation, check `SWEBenchSSHBox` for a reference of implementation.
-
-## How to put together an evaluation script?
-
-Now we know how to start running the agent end-to-end, and how `fake_user_response_fn` and `sandbox` work. We will walk through a piece of dummy code (simplified version of SWE-Bench's [`run_infer.py`](https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/swe_bench/run_infer.py)) that outline the general workflow:
-
-- Load the dataset and prepare the evaluation configuration.
-- Filter out any instances that have already been processed.
-- For each instance in the dataset:
-  - Set up the sandbox environment.
-  - Run the agent to generate a solution.
-  - Apply the solution to the instance and execute the test command.
-  - Collect the results and write them to the output file.
-- Perform cleanup after the evaluation is complete.
-
-You can see the [swe_bench/run_infer.py](swe_bench/run_infer.py) file for an example.
-
-When you fully understand the `run_infer.py`, you can be ready to actually starting the evaluation!
-
-
-## Run the evaluation!
-
-You can write your `run_infer.sh` script mimicking SWE-Bench's [`run_infer.sh`](https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/swe_bench/scripts/run_infer.sh).
-
-
-You can start the evaluation by running:
-
-```bash
-./run_infer.sh eval_gpt_4o_2024_05_13
-```
-Where `eval_gpt_4o_2024_05_13` is the model config you defined on the config.toml.
-Like this:
-
-```toml
-[core]
-...
-
-[llm]
-model="gpt-4-32k"
-...
-
-[eval_gpt_4o_2024_05_13]
-model="gpt-4o-2024-05-13"
-api_key="sk-xxx"
-```
-
-If `[eval_gpt_4o_2024_05_13]` is not present, it will default to using the model configured in `[llm]`.

evaluation/gaia/README.md

@@ -1,45 +0,0 @@
-# GAIA Evaluation
-
-This folder contains evaluation harness for evaluating agents on the [GAIA benchmark](https://arxiv.org/abs/2311.12983).
-
-## Configure OpenDevin and your LLM
-
-Create a `config.toml` file if it does not exist at the root of the workspace. Please check [README.md](../../README.md) for how to set this up.
-
-## Run the evaluation
-We are using the GAIA dataset hosted on [Hugging Face](https://huggingface.co/datasets/gaia-benchmark/GAIA).
-Please accept the terms and make sure to have logged in on your computer by `huggingface-cli login` before running the evaluation.
-
-Following is the basic command to start the evaluation. Here we are evaluating on the validation set for the `2023_all` split. You can adjust `./evaluation/gaia/scripts/run_infer.sh` to change the subset you want to evaluate on.
-
-```bash
-./evaluation/gaia/scripts/run_infer.sh [model_config] [agent] [eval_limit] [gaia_subset]
-# e.g., ./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview CodeActAgent 300
-```
-
-where `model_config` is mandatory, while `agent`, `eval_limit` and `gaia_subset` are optional.
-
-- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`, defaulting to `gpt-3.5-turbo`
-
-- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-
-- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances, defaulting to all instances.
-
-- `gaia_subset`, GAIA benchmark has multiple subsets: `2023_level1`, `2023_level2`, `2023_level3`, `2023_all`, defaulting to `2023_level1`.
-
-Let's say you'd like to run 10 instances using `eval_gpt4_1106_preview` and CodeActAgent,
-then your command would be:
-
-```bash
-./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview CodeActAgent 10
-```
-
-## Get score
-
-Then you can get stats by running the following command:
-```bash
-python ./evaluation/gaia/get_score.py \
---file <path_to/output.json>
-```

evaluation/gaia/get_score.py

@@ -1,28 +0,0 @@
-import argparse
-import json
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Get agent's gaia score")
-    parser.add_argument('--file', type=str, help="Path to the agent's output.jsonl")
-    args = parser.parse_args()
-    this_log = args.file
-    outs = []
-    with open(this_log, 'r') as f:
-        lines = f.readlines()
-        for line in lines:
-            outs.append(json.loads(line))
-    print(f'Reading {this_log}')
-    print(f'Metadata:\n {outs[0]["metadata"]}')
-
-    total = 0
-    success = 0
-    for out in outs:
-        total += 1
-        if out['test_result']['score']:
-            success += 1
-    print(f'Success rate: {success}/{total} = {success/total}')
-
-
-if __name__ == '__main__':
-    main()

evaluation/gaia/run_infer.py

@@ -1,358 +0,0 @@
-import asyncio
-import json
-import logging
-import multiprocessing as mp
-import os
-import pathlib
-import re
-import shutil
-import subprocess
-import time
-from concurrent.futures import ProcessPoolExecutor
-
-import huggingface_hub
-from datasets import load_dataset
-from tqdm import tqdm
-
-from evaluation.gaia.scorer import question_scorer
-from opendevin.controller.state.state import State
-from opendevin.core.config import config, get_llm_config_arg, get_parser
-from opendevin.core.logger import get_console_handler
-from opendevin.core.logger import opendevin_logger as logger
-from opendevin.core.main import main
-from opendevin.events.action import CmdRunAction, MessageAction
-from opendevin.events.serialization.event import event_to_dict
-
-DATASET_CACHE_DIR = '~/.cache/open-devin/evals/gaia'
-DATASET_CACHE_DIR = os.path.expanduser(DATASET_CACHE_DIR)
-
-
-def cleanup():
-    logger.info('Cleaning up child processes...')
-    for process in mp.active_children():
-        logger.info(f'Terminating child process: {process.name}')
-        process.terminate()
-        process.join()
-
-
-def codeact_user_response(state: State) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have solved the task, please first send your answer to user through message and then <execute_bash> exit </execute_bash>.\n'
-        'Please encapsulate your final answer (answer ONLY) within <solution> and </solution>.\n'
-        'For example: The answer to the question is <solution> 42 </solution>.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP.\n'
-    )
-    if state.history:
-        user_msgs = [
-            action
-            for action, _ in state.history
-            if isinstance(action, MessageAction) and action.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # let the agent know that it can give up when it has tried 3 times
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-
-
-def monologue_user_response(state: State) -> str:
-    raise NotImplementedError('MonologueAgent should never ask for user responses.')
-
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-    'MonologueAgent': monologue_user_response,
-}
-
-AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have solved the question, please first send your answer to user through message and then exit.\n'
-}
-
-
-def process_instance(instance, agent_class, metadata, reset_logger: bool = True):
-    # create process-specific workspace dir
-    # we will create a workspace directory for EACH process
-    # so that different agent don't interfere with each other.
-    old_workspace_mount_path = config.workspace_mount_path
-    workspace_mount_path = os.path.join(config.workspace_mount_path, '_eval_workspace')
-    workspace_mount_path = os.path.join(workspace_mount_path, str(os.getpid()))
-    pathlib.Path(workspace_mount_path).mkdir(parents=True, exist_ok=True)
-    config.workspace_mount_path = workspace_mount_path
-
-    # Setup the logger properly, so you can run multi-processing to parallize the evaluation
-    eval_output_dir = metadata['eval_output_dir']
-    if reset_logger:
-        # Set up logger
-        log_file = os.path.join(
-            eval_output_dir, 'logs', f'instance_{instance["task_id"]}.log'
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        # add back the console handler to print ONE line
-        logger.addHandler(get_console_handler())
-        logger.info(
-            f'Starting evaluation for instance {instance["task_id"]}.\nLOG:   tail -f {log_file}'
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setFormatter(
-            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        )
-        logger.addHandler(file_handler)
-
-    logger.info(f'Process-specific workspace mounted at {workspace_mount_path}')
-    if instance['file_name'] != '':
-        # if this question comes with a file, we need to save it to the workspace
-        src_file = os.path.join(
-            DATASET_CACHE_DIR, '2023', metadata['data_split'], instance['file_name']
-        )
-        extension_name = instance['file_name'].split('.')[-1]
-        dest_file = os.path.join(workspace_mount_path, f'file.{extension_name}')
-        shutil.copyfile(src_file, dest_file)
-        logger.info(f'File copied to {dest_file}')
-    else:
-        dest_file = None
-
-    # Prepare instruction
-    instruction = f"{instance['Question']}\n"
-    logger.info(f'Instruction: {instruction}')
-    if dest_file:
-        instruction += f"\n\nThe mentioned file is provided in the workspace at: {dest_file.split('/')[-1]}"
-
-    instruction += 'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
-    instruction += 'Please encapsulate your final answer (answer ONLY) within <solution> and </solution>.\n'
-    instruction += (
-        'For example: The answer to the question is <solution> 42 </solution>.\n'
-    )
-    # NOTE: You can actually set slightly different instruction for different agents
-    instruction += AGENT_CLS_TO_INST_SUFFIX.get(agent_class, '')
-    logger.info(f'Instruction:\n{instruction}', extra={'msg_type': 'OBSERVATION'})
-
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State = asyncio.run(
-        main(
-            instruction,
-            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(agent_class),
-        )
-    )
-    # ======= Attempt to evaluate the agent's edits =======
-    # If you are working on simplier benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    model_answer_raw = ''
-    for act, _ in reversed(state.history):
-        if isinstance(act, CmdRunAction) and act.source == 'agent':
-            model_answer_raw = act.thought
-            break
-        elif isinstance(act, MessageAction) and act.source == 'agent':
-            model_answer_raw = act.content
-            break
-
-    # attempt to parse model_answer
-    model_answer = re.findall(r'<solution>(.*?)</solution>', model_answer_raw)
-    if len(model_answer) == 0:
-        logger.warning(f'Failed to parse model answer: {model_answer_raw}')
-        model_answer = model_answer_raw
-    else:
-        model_answer = model_answer[0]
-
-    logger.info(
-        f'Final message: {model_answer} | Ground truth: {instance["Final answer"]}'
-    )
-    score = question_scorer(
-        model_answer=model_answer, ground_truth=instance['Final answer']
-    )
-    test_result = {
-        'score': score,
-        'model_answer_raw': model_answer_raw,
-        'model_answer': model_answer,
-        'ground_truth': instance['Final answer'],
-    }
-
-    # Save the output
-    output = {
-        'instance_id': instance['task_id'],
-        'instance': instance,
-        'instruction': instance['Question'],
-        'metadata': metadata,
-        'history': [
-            (event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
-        ],
-        'error': state.error if state and state.error else None,
-        'test_result': test_result,
-    }
-
-    # Close the sandbox
-    config.workspace_mount_path = old_workspace_mount_path
-    return output
-
-
-if __name__ == '__main__':
-    parser = get_parser()
-    parser.add_argument(
-        '--level',
-        type=str,
-        help='gaia level to evaluate, eg. 2023_level1',
-    )
-    parser.add_argument(
-        '--data-split',
-        type=str,
-        help='data split to evaluate, eg. validation',
-    )
-    args, _ = parser.parse_known_args()
-    if args.directory:
-        config.workspace_base = os.path.abspath(args.directory)
-        logger.info(f'Setting workspace base to {config.workspace_base}')
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenDevin's repo
-    level = args.level
-    data_split = args.data_split
-    dataset = load_dataset('gaia-benchmark/GAIA', level)
-    huggingface_hub.snapshot_download(
-        'gaia-benchmark/GAIA',
-        repo_type='dataset',
-        local_dir=DATASET_CACHE_DIR,
-    )
-    gaia_tests = dataset[data_split]
-    logger.info(f'Evaluating GAIA-Benchmark {level} {data_split} split')
-
-    # Check https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/swe_bench/README.md#configure-opendevin-and-your-llm
-    # for details of how to set `llm_config`
-    if args.llm_config:
-        specified_llm_config = get_llm_config_arg(args.llm_config)
-        if specified_llm_config:
-            config.llm = specified_llm_config
-    logger.info(f'Config for evaluation: {config}')
-
-    # TEST METADATA
-    agent_class = args.agent_cls
-    assert (
-        agent_class in AGENT_CLS_TO_FAKE_USER_RESPONSE_FN
-    ), f'Unsupported agent class: {agent_class}'
-    model_name = config.llm.model.split('/')[-1]
-    max_iterations = args.max_iterations
-    eval_note = ''
-    if args.eval_note is not None:
-        eval_note += '_N_' + args.eval_note
-    eval_output_dir = os.path.join(
-        args.eval_output_dir,
-        'gaia',
-        agent_class,
-        model_name + '_maxiter_' + str(max_iterations) + eval_note,
-    )
-
-    pathlib.Path(eval_output_dir).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(os.path.join(eval_output_dir, 'logs')).mkdir(
-        parents=True, exist_ok=True
-    )
-    logger.info(f'Using evaluation output directory: {eval_output_dir}')
-
-    metadata = {
-        'gaia-level': level,
-        'data_split': data_split,
-        'agent_class': agent_class,
-        'model_name': model_name,
-        'max_iterations': max_iterations,
-        'eval_output_dir': eval_output_dir,
-        'start_time': time.strftime('%Y-%m-%d %H:%M:%S'),
-        # get the commit id of current repo for reproduciblity
-        'git_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'])
-        .decode('utf-8')
-        .strip(),
-    }
-    logger.info(f'Metadata: {metadata}')
-    with open(os.path.join(eval_output_dir, 'metadata.json'), 'w') as f:
-        json.dump(metadata, f)
-
-    # LIMIT EVALUATION
-    eval_n_limit = args.eval_n_limit
-    if eval_n_limit:
-        gaia_tests = gaia_tests.select(list(range(eval_n_limit)))
-        logger.info(f'Limiting evaluation to first {eval_n_limit} instances.')
-
-    # OUTPUT FILE
-    output_file = os.path.join(eval_output_dir, 'output.jsonl')
-    logger.info(f'Writing evaluation output to {output_file}')
-    finished_task_ids = set()
-    if os.path.exists(output_file):
-        with open(output_file, 'r') as f:
-            for line in f:
-                data = json.loads(line)
-                finished_task_ids.add(data['instance_id'])
-        logger.warning(
-            f'Output file {output_file} already exists. Loaded {len(finished_task_ids)} finished instances.'
-        )
-    output_fp = open(output_file, 'a')
-
-    logger.info(
-        f'Evaluation started with Agent {agent_class}, model {model_name}, max iterations {max_iterations}.'
-    )
-
-    # =============================================
-    # filter out finished instances
-    new_gaia_tests = []
-    for instance in gaia_tests:
-        if instance['task_id'] in finished_task_ids:
-            logger.info(
-                f'Skipping instance {instance["task_id"]} as it is already finished.'
-            )
-            continue
-        new_gaia_tests.append(instance)
-
-    gaia_tests = new_gaia_tests
-    logger.info(
-        f'Finished instances: {len(finished_task_ids)}, Remaining instances: {len(gaia_tests)}'
-    )
-    # =============================================
-
-    pbar = tqdm(total=len(gaia_tests))
-
-    # This function tracks the progress AND write the output to a JSONL file
-    def update_progress(future):
-        pbar.update(1)
-        output = future.result()
-        pbar.set_description(f'Instance {output["instance_id"]}')
-        pbar.set_postfix_str(f'Test Result: {output["test_result"]["score"]}')
-        logger.info(
-            f'Finished evaluation for instance {output["instance_id"]}: {output["test_result"]}'
-        )
-        output_fp.write(json.dumps(output) + '\n')
-        output_fp.flush()
-
-    # This sets the multi-processing
-    num_workers = args.eval_num_workers
-    logger.info(f'Using {num_workers} workers for evaluation.')
-
-    try:
-        with ProcessPoolExecutor(num_workers) as executor:
-            futures = []
-            # This is how we perform multi-processing
-            for instance in gaia_tests:
-                future = executor.submit(
-                    process_instance,
-                    instance,
-                    agent_class,
-                    metadata,
-                    reset_logger=bool(num_workers > 1),
-                )
-                future.add_done_callback(update_progress)
-                futures.append(future)
-
-            # Wait for all futures to complete
-            for future in futures:
-                future.result()
-    except KeyboardInterrupt:
-        logger.info('KeyboardInterrupt received. Cleaning up...')
-        cleanup()
-
-    output_fp.close()
-    logger.info('Evaluation finished.')

evaluation/gaia/scorer.py

@@ -1,98 +0,0 @@
-import re
-import string
-import warnings
-
-
-def normalize_number_str(number_str: str) -> float:
-    # we replace these common units and commas to allow
-    # conversion to float
-    for char in ['$', '%', ',']:
-        number_str = number_str.replace(char, '')
-    try:
-        return float(number_str)
-    except ValueError:
-        print(f'String {number_str} cannot be normalized to number str.')
-        return float('inf')
-
-
-def split_string(
-    s: str,
-    char_list: list[str] = [',', ';'],
-) -> list[str]:
-    pattern = f"[{''.join(char_list)}]"
-    return re.split(pattern, s)
-
-
-def question_scorer(
-    model_answer: str,
-    ground_truth: str,
-) -> bool:
-    def is_float(element: any) -> bool:
-        try:
-            float(element)
-            return True
-        except ValueError:
-            return False
-
-    # if gt is a number
-    if is_float(ground_truth):
-        print(f'Evaluating {model_answer} as a number.')
-        normalized_answer = normalize_number_str(model_answer)
-        return normalized_answer == float(ground_truth)
-
-    # if gt is a list
-    elif any(char in ground_truth for char in [',', ';']):
-        print(f'Evaluating {model_answer} as a comma separated list.')
-        # question with the fish: normalization removes punct
-
-        gt_elems = split_string(ground_truth)
-        ma_elems = split_string(model_answer)
-
-        # check length is the same
-        if len(gt_elems) != len(ma_elems):
-            warnings.warn(
-                'Answer lists have different lengths, returning False.', UserWarning
-            )
-            return False
-
-        # compare each element as float or str
-        comparisons = []
-        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
-            if is_float(gt_elem):
-                normalized_ma_elem = normalize_number_str(ma_elem)
-                comparisons.append(normalized_ma_elem == float(gt_elem))
-            else:
-                # we do not remove punct since comparisons can include punct
-                comparisons.append(
-                    normalize_str(ma_elem, remove_punct=False)
-                    == normalize_str(gt_elem, remove_punct=False)
-                )
-        return all(comparisons)
-
-    # if gt is a str
-    else:
-        print(f'Evaluating {model_answer} as a string.')
-        return normalize_str(model_answer) == normalize_str(ground_truth)
-
-
-def normalize_str(input_str, remove_punct=True) -> str:
-    """
-    Normalize a string by:
-    - Removing all white spaces
-    - Optionally removing punctuation (if remove_punct is True)
-    - Converting to lowercase
-    Parameters:
-    - input_str: str, the string to normalize
-    - remove_punct: bool, whether to remove punctuation (default: True)
-    Returns:
-    - str, the normalized string
-    """
-    # Remove all white spaces. Required e.g for seagull vs. sea gull
-    no_spaces = re.sub(r'\s', '', input_str)
-
-    # Remove punctuation, if specified.
-    if remove_punct:
-        translator = str.maketrans('', '', string.punctuation)
-        return no_spaces.lower().translate(translator)
-    else:
-        return no_spaces.lower()

evaluation/gaia/scripts/run_infer.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-MODEL_CONFIG=$1
-AGENT=$2
-EVAL_LIMIT=$3
-LEVELS=$4
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-if [ -z "$LEVELS" ]; then
-  LEVELS="2023_level1"
-  echo "Levels not specified, use default $LEVELS"
-fi
-
-# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenDevin
-# We need to track the version of Agent in the evaluation to make sure results are comparable
-AGENT_VERSION=v$(poetry run python -c "import agenthub; from opendevin.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-echo "LEVELS: $LEVELS"
-
-COMMAND="poetry run python ./evaluation/gaia/run_infer.py \
-  --agent-cls $AGENT \
-  --llm-config $MODEL_CONFIG \
-  --max-iterations 30 \
-  --level $LEVELS \
-  --data-split validation \
-  --max-chars 10000000 \
-  --eval-num-workers 1 \
-  --eval-note ${AGENT_VERSION}_${LEVELS}"
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND

evaluation/humanevalfix/README.md

@@ -1,210 +0,0 @@
-# HumanEvalFix Evaluation with OpenDevin
-
-Implements evaluation of agents on HumanEvalFix from the HumanEvalPack benchmark introduced in [OctoPack: Instruction Tuning Code Large Language Models](https://arxiv.org/abs/2308.07124). Please see [here](https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/bigcode_eval/tasks/humanevalpack.py) for the reference implementation used in the paper.
-
-## Setup Environment
-
-Please follow [this document](https://github.com/OpenDevin/OpenDevin/blob/main/Development.md) to setup local develop environment for OpenDevin.
-
-
-## Configure OpenDevin and your LLM
-
-Create a `config.toml` file if it does not exist at the root of the workspace.
-
-Add the following configurations:
-
-```toml
-[core]
-max_iterations = 100
-cache_dir = "/tmp/cache"
-ssh_hostname = "localhost"
-enable_auto_lint = true
-
-# TODO: Change these to the model you want to evaluate
-[eval_gpt4_1106_preview]
-model = "gpt-4-1106-preview"
-api_key = "XXX"
-temperature = 0.0
-
-[eval_some_openai_compatible_model]
-model = "openai/MODEL_NAME"
-base_url = "https://OPENAI_COMPATIBLE_URL/v1"
-api_key = "XXX"
-temperature = 0.0
-```
-
-## Run Inference on HumanEvalFix
-
-```bash
-./evaluation/humanevalfix/scripts/run_infer.sh eval_gpt4_1106_preview
-```
-
-You can replace `eval_gpt4_1106_preview` with any model you set up in `config.toml`.
-
-
-## Examples
-
-For each problem, OpenDevin is given a set number of iterations to fix the failing code. The history field shows each iteration's response to correct its code that fails any test case.
-
-
-```
-{
-    "task_id": "Python/2",
-    "instruction": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
-    "metadata": {
-        "agent_class": "CodeActAgent",
-        "model_name": "gpt-4",
-        "max_iterations": 10,
-        "eval_output_dir": "evaluation/evaluation_outputs/outputs/humanevalfix/CodeActAgent/gpt-4_maxiter_10_N_v1.4",
-        "start_time": "2024-05-22 20:54:15",
-        "git_commit": "4d3253696f5a9d9de02ab86969fe9796fa40331f"
-    },
-    "history": [
-        [
-            {
-                "id": 27,
-                "timestamp": "2024-05-22T20:57:24.688651",
-                "source": "user",
-                "message": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
-                "action": "message",
-                "args": {
-                    "content": "Please fix the function in Python__2.py such that all test cases pass.\nEnvironment has been set up for you to start working. You may assume all necessary tools are installed.\n\n# Problem Statement\ndef truncate_number(number: float) -> float:\n    return number % 1.0 + 1.0\n\n\n\n\n\n\ndef check(truncate_number):\n    assert truncate_number(3.5) == 0.5\n    assert abs(truncate_number(1.33) - 0.33) < 1e-6\n    assert abs(truncate_number(123.456) - 0.456) < 1e-6\n\ncheck(truncate_number)\n\nIMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\nYou should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\nYou SHOULD INCLUDE PROPER INDENTATION in your edit commands.\nWhen you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n",
-                    "wait_for_response": false
-                }
-            },
-            {
-                "message": "No observation",
-                "observation": "null",
-                "content": "",
-                "extras": {}
-            }
-        ],
-        [
-            {
-                "id": 32,
-                "timestamp": "2024-05-22T20:57:28.413818",
-                "source": "agent",
-                "message": "Running command: open Python__2.py",
-                "action": "run",
-                "args": {
-                    "command": "open Python__2.py",
-                    "background": false,
-                    "thought": "Let's start by opening the Python__2.py file to see the current implementation of the function."
-                }
-            },
-            {
-                "id": 33,
-                "timestamp": "2024-05-22T20:57:28.803780",
-                "source": "agent",
-                "message": "Command `open Python__2.py` executed with exit code 0.",
-                "cause": 32,
-                "observation": "run",
-                "content": "[File: /workspace/Python__2.py (14 lines total)]\r\n1:def truncate_number(number: float) -> float:\r\n2:    return number % 1.0 + 1.0\r\n3:\r\n4:\r\n5:\r\n6:\r\n7:\r\n8:\r\n9:def check(truncate_number):\r\n10:    assert truncate_number(3.5) == 0.5\r\n11:    assert abs(truncate_number(1.33) - 0.33) < 1e-6\r\n12:    assert abs(truncate_number(123.456) - 0.456) < 1e-6\r\n13:\r\n14:check(truncate_number)",
-                "extras": {
-                    "command_id": -1,
-                    "command": "open Python__2.py",
-                    "exit_code": 0
-                }
-            }
-        ],
-        [
-            {
-                "id": 34,
-                "timestamp": "2024-05-22T20:57:36.605638",
-                "source": "agent",
-                "message": "Running command: edit 2:2 <<EOF\n    return number % 1.0\nEOF",
-                "action": "run",
-                "args": {
-                    "command": "edit 2:2 <<EOF\n    return number % 1.0\nEOF",
-                    "background": false,
-                    "thought": "The current implementation of the function `truncate_number` is incorrect because it adds 1.0 to the fractional part of the number. This is not necessary and is causing the function to fail the test cases. \n\nThe correct implementation should only return the fractional part of the number. This can be achieved by using the modulus operator with 1.0, which will return the fractional part of the number.\n\nLet's correct the function implementation."
-                }
-            },
-            {
-                "id": 35,
-                "timestamp": "2024-05-22T20:57:36.973592",
-                "source": "agent",
-                "message": "Command `edit 2:2 <<EOF\n    return number % 1.0\nEOF` executed with exit code 0.",
-                "cause": 34,
-                "observation": "run",
-                "content": "> > [File: /workspace/Python__2.py (14 lines total)]\r\n1:def truncate_number(number: float) -> float:\r\n2:    return number % 1.0\r\n3:\r\n4:\r\n5:\r\n6:\r\n7:\r\n8:\r\n9:def check(truncate_number):\r\n10:    assert truncate_number(3.5) == 0.5\r\n11:    assert abs(truncate_number(1.33) - 0.33) < 1e-6\r\n12:    assert abs(truncate_number(123.456) - 0.456) < 1e-6\r\n13:\r\n14:check(truncate_number)\r\nFile updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.",
-                "extras": {
-                    "command_id": -1,
-                    "command": "edit 2:2 <<EOF\n    return number % 1.0\nEOF",
-                    "exit_code": 0
-                }
-            }
-        ],
-        [
-            {
-                "id": 36,
-                "timestamp": "2024-05-22T20:57:48.420415",
-                "source": "agent",
-                "message": "Running command: python3 Python__2.py",
-                "action": "run",
-                "args": {
-                    "command": "python3 Python__2.py",
-                    "background": false,
-                    "thought": "The function `truncate_number` has been updated. Now, let's run the Python file to see if the test cases pass."
-                }
-            },
-            {
-                "id": 37,
-                "timestamp": "2024-05-22T20:57:48.635312",
-                "source": "agent",
-                "message": "Command `python3 Python__2.py` executed with exit code 0.",
-                "cause": 36,
-                "observation": "run",
-                "content": "",
-                "extras": {
-                    "command_id": -1,
-                    "command": "python3 Python__2.py",
-                    "exit_code": 0
-                }
-            }
-        ],
-        [
-            {
-                "id": 38,
-                "timestamp": "2024-05-22T20:58:10.589252",
-                "source": "agent",
-                "message": "All done! What's next on the agenda?",
-                "action": "finish",
-                "args": {
-                    "outputs": {},
-                    "thought": ""
-                }
-            },
-            {
-                "message": "No observation",
-                "observation": "null",
-                "content": "",
-                "extras": {}
-            }
-        ]
-    ],
-    "error": null,
-    "test_result": {
-        "result": {
-            "pass@1": 1.0
-        },
-        "metadata": {
-            "logs": {
-                "0": [
-                    [
-                        0,
-                        {
-                            "task_id": 0,
-                            "passed": true,
-                            "result": "passed",
-                            "completion_id": 0
-                        }
-                    ]
-                ]
-            },
-            "timeout": 10,
-            "num_workers": 4
-        }
-    }
-}
-```

evaluation/humanevalfix/run_infer.py

@@ -1,379 +0,0 @@
-"""
-Implements evaluation of agents on HumanEvalFix from the HumanEvalPack benchmark introduced in
-"OctoPack: Instruction Tuning Code Large Language Models" (https://arxiv.org/abs/2308.07124).
-Please see https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/bigcode_eval/tasks/humanevalpack.py
-for the reference implementation used in the paper.
-
-TODOs:
-- Potentially support other HumanEvalPack datasets (Explain & Synthesize)
-- Support other languages (currently only Python)
-"""
-
-import asyncio
-import json
-import logging
-import multiprocessing as mp
-import os
-import pathlib
-import subprocess
-import time
-from concurrent.futures import ProcessPoolExecutor
-
-import pandas as pd
-from datasets import load_dataset
-from evaluate import load
-from tqdm import tqdm
-
-from opendevin.controller.state.state import State
-from opendevin.core.config import args, config, get_llm_config_arg
-from opendevin.core.logger import get_console_handler
-from opendevin.core.logger import opendevin_logger as logger
-from opendevin.core.main import main
-from opendevin.events.action import MessageAction
-from opendevin.events.serialization.event import event_to_dict
-
-IMPORT_HELPER = {
-    'python': [
-        'import math',
-        'import re',
-        'import sys',
-        'import copy',
-        'import datetime',
-        'import itertools',
-        'import collections',
-        'import heapq',
-        'import statistics',
-        'import functools',
-        'import hashlib',
-        'import numpy',
-        'import numpy as np',
-        'import string',
-        'from typing import *',
-        'from collections import *',
-    ],
-}
-
-LANGUAGE_TO_TIMEOUT = {
-    'python': 10,
-}
-
-LANGUAGE_TO_NUM_WORKERS = {
-    'python': 4,
-}
-
-
-def cleanup():
-    logger.info('Cleaning up child processes...')
-    for process in mp.active_children():
-        logger.info(f'Terminating child process: {process.name}')
-        process.terminate()
-        process.join()
-
-
-def codeact_user_response(state: State) -> str:
-    msg = (
-        'Please continue working on the task on whatever approach you think is suitable.\n'
-        'If you think you have modified the code in a way that fixes the issue, please run the following command: <execute_bash> exit </execute_bash>.\n'
-        'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN HELP OR USE THE INTERNET TO SOLVE THIS TASK.\n'
-    )
-    if state.history:
-        user_msgs = [
-            action
-            for action, _ in state.history
-            if isinstance(action, MessageAction) and action.source == 'user'
-        ]
-        if len(user_msgs) >= 2:
-            # let the agent know that it can give up when it has tried 3 times
-            return (
-                msg
-                + 'If you want to give up, run: <execute_bash> exit </execute_bash>.\n'
-            )
-    return msg
-
-
-def monologue_user_response(state: State) -> str:
-    raise NotImplementedError('MonologueAgent should never ask for user responses.')
-
-
-AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
-    'CodeActAgent': codeact_user_response,
-    'MonologueAgent': monologue_user_response,
-}
-
-AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
-}
-
-
-def get_test_result(instance, path, language='python', timeout=10):
-    # Evaluation reference: https://github.com/bigcode-project/bigcode-evaluation-harness/blob/84b96da31b7f840b55c5733325346176140cdb6b/bigcode_eval/tasks/humanevalpack.py#L347
-    test_result = {'result': {}, 'metadata': {}}
-    code_metric = load('Muennighoff/code_eval_octopack')
-    timeout = LANGUAGE_TO_TIMEOUT[language]
-    num_workers = LANGUAGE_TO_NUM_WORKERS[language]
-    python_imports = '\n'.join(IMPORT_HELPER[language])
-
-    # Load function from path
-    with open(path, 'r') as f:
-        function = f.read()
-
-    function = [[python_imports + '\n' + function.strip()]]
-
-    results, logs = code_metric.compute(
-        references=[instance.test],
-        predictions=function,
-        language=language,
-        timeout=timeout,
-        num_workers=num_workers,
-    )
-    test_result['result'] = results
-    test_result['metadata'] = {
-        'logs': logs,
-        'timeout': timeout,
-        'num_workers': num_workers,
-    }
-    return test_result
-
-
-def process_instance(
-    instance, agent_class, metadata, skip_workspace_mount, reset_logger: bool = True
-):
-    old_workspace_mount_path = config.workspace_mount_path
-    old_workspace_base = config.workspace_base
-    workspace_mount_path = os.path.join(config.workspace_mount_path, '_eval_workspace')
-    # create process-specific workspace dir
-    # if `not skip_workspace_mount` - we will create a workspace directory for EACH process
-    # so that different agent don't interfere with each other.
-    if not skip_workspace_mount:
-        workspace_mount_path = os.path.join(workspace_mount_path, str(os.getpid()))
-        pathlib.Path(workspace_mount_path).mkdir(parents=True, exist_ok=True)
-
-    # reset workspace to config
-    config.workspace_base = workspace_mount_path
-    config.workspace_mount_path = workspace_mount_path
-
-    # Setup the logger properly, so you can run multi-processing to parallize the evaluation
-    if reset_logger:
-        # Set up logger
-        log_file = os.path.join(
-            eval_output_dir,
-            'logs',
-            f'instance_{instance.task_id.replace("/", "__")}.log',
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        # add back the console handler to print ONE line
-        logger.addHandler(get_console_handler())
-        logger.info(
-            f'Starting evaluation for instance {instance.task_id}.\nLOG:   tail -f {log_file}'
-        )
-        # Remove all existing handlers from logger
-        for handler in logger.handlers[:]:
-            logger.removeHandler(handler)
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setFormatter(
-            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        )
-        logger.addHandler(file_handler)
-
-    if not skip_workspace_mount:
-        logger.info(f'Process-specific workspace mounted at {workspace_mount_path}')
-
-    # Create file with HumanEvalFix problem
-    # Prompt reference: https://github.com/bigcode-project/bigcode-evaluation-harness/blob/84b96da31b7f840b55c5733325346176140cdb6b/bigcode_eval/tasks/humanevalpack.py#L509
-    problem_statement = (
-        instance.declaration + instance.buggy_solution + '\n' + instance.test
-    )
-    path = os.path.join(
-        workspace_mount_path, f'{instance.task_id.replace("/", "__")}.py'
-    )
-    with open(path, 'w') as f:
-        f.write(problem_statement)
-
-    # Prepare instruction
-    instruction = (
-        f'Please fix the function in {instance.task_id.replace("/", "__")}.py such that all test cases pass.\n'
-        'Environment has been set up for you to start working. You may assume all necessary tools are installed.\n\n'
-        '# Problem Statement\n'
-        f'{problem_statement}\n\n'
-    )
-    instruction += (
-        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
-        'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
-        'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
-    )
-    # NOTE: You can actually set slightly different instruction for different agents
-    instruction += AGENT_CLS_TO_INST_SUFFIX.get(agent_class, '')
-
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
-    state: State = asyncio.run(
-        main(
-            instruction,
-            fake_user_response_fn=AGENT_CLS_TO_FAKE_USER_RESPONSE_FN.get(agent_class),
-        )
-    )
-
-    # ======= Attempt to evaluate the agent's edits =======
-    test_result = get_test_result(instance, path)
-
-    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-    if state is None:
-        raise ValueError('State should not be None.')
-
-    # Save the output
-    output = {
-        'task_id': instance.task_id,
-        'instruction': instruction,
-        'metadata': metadata,
-        'history': [
-            (event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
-        ],
-        'error': state.error if state and state.error else None,
-        'test_result': test_result,
-    }
-
-    config.workspace_mount_path = old_workspace_mount_path
-    config.workspace_base = old_workspace_base
-    return output
-
-
-if __name__ == '__main__':
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenDevin's repo
-    dataset = load_dataset(
-        'bigcode/humanevalpack', 'python'
-    )  # TODO: Support other languages
-    hefix_tests = dataset['test'].to_pandas()
-
-    # Check https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/humanevalfix/README.md#configure-opendevin-and-your-llm
-    # for details of how to set `llm_config`
-    if args.llm_config:
-        specified_llm_config = get_llm_config_arg(args.llm_config)
-        if specified_llm_config:
-            config.llm = specified_llm_config
-    logger.info(f'Config for evaluation: {config}')
-
-    # TEST METADATA
-    agent_class = args.agent_cls
-    assert (
-        agent_class in AGENT_CLS_TO_FAKE_USER_RESPONSE_FN
-    ), f'Unsupported agent class: {agent_class}'
-    model_name = config.llm.model.split('/')[-1]
-    max_iterations = args.max_iterations
-    eval_note = ''
-    if args.eval_note is not None:
-        eval_note += '_N_' + args.eval_note
-    eval_output_dir = os.path.join(
-        args.eval_output_dir,
-        'humanevalfix',
-        agent_class,
-        model_name + '_maxiter_' + str(max_iterations) + eval_note,
-    )
-
-    pathlib.Path(eval_output_dir).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(os.path.join(eval_output_dir, 'logs')).mkdir(
-        parents=True, exist_ok=True
-    )
-    logger.info(f'Using evaluation output directory: {eval_output_dir}')
-
-    metadata = {
-        'agent_class': agent_class,
-        'model_name': model_name,
-        'max_iterations': max_iterations,
-        'eval_output_dir': eval_output_dir,
-        'start_time': time.strftime('%Y-%m-%d %H:%M:%S'),
-        # get the commit id of current repo for reproduciblity
-        'git_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'])
-        .decode('utf-8')
-        .strip(),
-    }
-    logger.info(f'Metadata: {metadata}')
-    with open(os.path.join(eval_output_dir, 'metadata.json'), 'w') as f:
-        json.dump(metadata, f)
-
-    # LIMIT EVALUATION
-    eval_n_limit = args.eval_n_limit
-    if eval_n_limit:
-        hefix_tests = hefix_tests.head(eval_n_limit)
-        logger.info(f'Limiting evaluation to first {eval_n_limit} instances.')
-
-    # OUTPUT FILE
-    output_file = os.path.join(eval_output_dir, 'output.jsonl')
-    logger.info(f'Writing evaluation output to {output_file}')
-    finished_instance_ids = set()
-    if os.path.exists(output_file):
-        with open(output_file, 'r') as f:
-            for line in f:
-                data = json.loads(line)
-                finished_instance_ids.add(data['task_id'])
-        logger.warning(
-            f'Output file {output_file} already exists. Loaded {len(finished_instance_ids)} finished instances.'
-        )
-    output_fp = open(output_file, 'a')
-
-    logger.info(
-        f'Evaluation started with Agent {agent_class}, model {model_name}, max iterations {max_iterations}.'
-    )
-
-    # =============================================
-    # filter out finished instances
-    new_hefix_tests = []
-    for idx, instance in hefix_tests.iterrows():
-        if instance.task_id in finished_instance_ids:
-            logger.info(
-                f'Skipping instance {instance.task_id} as it is already finished.'
-            )
-            continue
-        new_hefix_tests.append(instance)
-
-    hefix_tests = pd.DataFrame(new_hefix_tests)
-    logger.info(
-        f'Finished instances: {len(finished_instance_ids)}, Remaining instances: {len(hefix_tests)}'
-    )
-    # =============================================
-
-    pbar = tqdm(total=len(hefix_tests))
-
-    # This function tracks the progress AND write the output to a JSONL file
-    def update_progress(future):
-        pbar.update(1)
-        output = future.result()
-        pbar.set_description(f'Instance {output["task_id"]}')
-        pbar.set_postfix_str(f'Test Result: {output["test_result"]["result"]}')
-        logger.info(
-            f'Finished evaluation for instance {output["task_id"]}: {output["test_result"]["result"]}'
-        )
-        output_fp.write(json.dumps(output) + '\n')
-        output_fp.flush()
-
-    # This sets the multi-processing
-    num_workers = args.eval_num_workers
-    logger.info(f'Using {num_workers} workers for evaluation.')
-
-    try:
-        with ProcessPoolExecutor(num_workers) as executor:
-            futures = []
-            # This is how we perform multi-processing
-            for row_idx, instance in hefix_tests.iterrows():
-                future = executor.submit(
-                    process_instance,
-                    instance,
-                    agent_class,
-                    metadata,
-                    skip_workspace_mount=False,
-                    reset_logger=bool(num_workers > 1),
-                )
-                future.add_done_callback(update_progress)
-                futures.append(future)
-
-            # Wait for all futures to complete
-            for future in futures:
-                future.result()
-    except KeyboardInterrupt:
-        print('KeyboardInterrupt received. Cleaning up...')
-        cleanup()
-
-    output_fp.close()
-    logger.info('Evaluation finished.')

evaluation/humanevalfix/scripts/run_infer.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-MODEL_CONFIG=$1
-AGENT=$2
-EVAL_LIMIT=$3
-
-echo "
-################################################################################
-                                  !!!WARNING!!!
-################################################################################
-The "code_eval" metric executes untrusted model-generated code in Python.
-Although it is highly unlikely that model-generated code will do something
-overtly malicious in response to this test suite, model-generated code may act
-destructively due to a lack of model capability or alignment.
-Users are strongly encouraged to sandbox this evaluation suite so that it
-does not perform destructive actions on their host or network. For more
-information on how OpenAI sandboxes its code, see the paper \"Evaluating Large
-Language Models Trained on Code\" (https://arxiv.org/abs/2107.03374).
-
-Once you have read this disclaimer and taken appropriate precautions,
-set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this
-with:
-
->>> import os
->>> os.environ[\"HF_ALLOW_CODE_EVAL\"] = \"1\"
-
-################################################################################
-"
-
-echo "WARNING: You are about to enable the execution of untrusted model-generated code by setting the environment variable HF_ALLOW_CODE_EVAL to '1'."
-echo "It is highly unlikely that model-generated code will do something overtly malicious in response to this test suite, however, it may act destructively due to a lack of model capability or alignment."
-echo "Please confirm that you have read the disclaimer, taken the necessary precautions, and wish to proceed (y/n):"
-read user_input
-
-if [ "$user_input" = "y" ]; then
-  export HF_ALLOW_CODE_EVAL="1"
-  echo "Environment variable HF_ALLOW_CODE_EVAL has been set to '1'."
-else
-  echo "Operation aborted. Environment variable HF_ALLOW_CODE_EVAL has not been set."
-  exit 1
-fi
-
-# ################################################################################
-
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
-
-# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenDevin
-# We need to track the version of Agent in the evaluation to make sure results are comparable
-AGENT_VERSION=v$(poetry run python -c "import agenthub; from opendevin.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-
-COMMAND="poetry run python evaluation/humanevalfix/run_infer.py \
-  --agent-cls $AGENT \
-  --llm-config $MODEL_CONFIG \
-  --max-iterations 10 \
-  --max-chars 10000000 \
-  --eval-num-workers 1 \
-  --eval-note $AGENT_VERSION"
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND

evaluation/regression/run_tests.py

@@ -13,15 +13,9 @@ if __name__ == '__main__':
         python script_name.py [--OPENAI_API_KEY=<api_key>] [--model=<model_name>]
 
     """
-    parser = argparse.ArgumentParser(
-        description='This script runs pytest with specific arguments and configuration.'
-    )
-    parser.add_argument(
-        '--OPENAI_API_KEY', type=str, required=True, help='Your OpenAI API key'
-    )
-    parser.add_argument(
-        '--model', type=str, required=True, help='The model name to use'
-    )
+    parser = argparse.ArgumentParser(description='This script runs pytest with specific arguments and configuration.')
+    parser.add_argument('--OPENAI_API_KEY', type=str, required=True, help='Your OpenAI API key')
+    parser.add_argument('--model', type=str, required=True, help='The model name to use')
 
     parser_args = parser.parse_args()
     config.config['OPENAI_API_KEY'] = parser_args.OPENAI_API_KEY

evaluation/swe_bench/BUILD_TESTBED_AND_ENV.md

@@ -1,12 +1,12 @@
 # Pre-build Testbed and Env
 
-In the original SWE-Bench implementation, conda environment for evaluation is typically installed from scratch while evaluating on a particular instance. This poses several challenges:
+In the original SWE-Bench implementation, conda environment for evaluation is typically installed from scratch while evaluating on a paticular instance. This poses serveral challenges:
 
-- Efficiency: most time of evaluation will be wasted on downloading packages
+- Effeciency: most time of evaluation will be wasted on downloading packages
 - Stability: setup could failed due to bad internet connectivity
 - Reliability: it is possible that an instance is considered failed not because the agent did badly, but because the environment setup failed.
 
-In OpenDevin-SWE-Bench fork, we try to pre-build the **testbed** (i.e., code of the repository we want the agent to edit) AND the **conda environment**, so that in evaluation (inference) time, we can directly leverage existing environments for efficient evaluation.
+In OpenDevin-SWE-Bench fork, we try to pre-build the **testbed** (i.e., code of the repository we want the agent to edit) AND the **conda environment**, so that in evaluation (inference) time, we can directly leverage existing environments for effecienct evaluation.
 
 NOTE: We only support SWE-Bench lite for now. But modifying our existing scripts for full SWE-Bench should be quite straight forward.
 
@@ -34,6 +34,6 @@ Run the following command to do the above two steps. The results will be saved t
 
 ```bash
 pushd evaluation/swe_bench
-docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.2.1 -f ./scripts/docker/Dockerfile.full.v1.1 .
-docker push ghcr.io/opendevin/eval-swe-bench:full-v1.2.1
+docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.0 -f ./scripts/docker/Dockerfile.full.v1.0 .
+docker push ghcr.io/opendevin/eval-swe-bench:full-v1.0
 ```

evaluation/swe_bench/EVAL_PATCH.md

@@ -117,7 +117,7 @@ Before evaluating generated patches, you need to set up the Docker environment.
 ```shell
 docker run -it \
 -v DIR_TO_YOUR_PATCH_FILES_ON_HOST:/swe_bench_output \
-ghcr.io/opendevin/eval-swe-bench:full-v1.2.1 /bin/bash
+ghcr.io/opendevin/eval-swe-bench:full-v1.0 /bin/bash
 ```
 
 ### Evaluate Model Generated Patches

evaluation/swe_bench/README.md

@@ -3,11 +3,6 @@
 
 This folder contains evaluation harness we built on top of the original [SWE-Bench benchmark](https://www.swebench.com/) ([paper](https://arxiv.org/abs/2310.06770)). We create [a fork of SWE-Bench](https://github.com/OpenDevin/OD-SWE-bench.git) mostly build on top of [the original repo](https://github.com/princeton-nlp/SWE-bench) and [containerized](#opendevin-swe-bench-docker-image) it for easy evaluation.
 
-## Setup Environment
-
-Please follow [this document](https://github.com/OpenDevin/OpenDevin/blob/main/Development.md) to setup local develop environment for OpenDevin.
-
-
 ## OpenDevin SWE-Bench Docker Image
 
 In [OpenDevin-SWE-Bench fork](https://github.com/OpenDevin/OD-SWE-bench.git) (mostly from [original repo](https://github.com/princeton-nlp/SWE-bench) with some fixes), we try to pre-build the **testbed** (i.e., code of the repository we want the agent to edit) AND the **conda environment**, so that in evaluation (inference) time, we can directly leverage existing environments for effecienct evaluation.
@@ -15,7 +10,7 @@ In [OpenDevin-SWE-Bench fork](https://github.com/OpenDevin/OD-SWE-bench.git) (mo
 **We pack everything you need for SWE-Bench evaluation into one, gigantic, docker image.** To use it:
 
 ```bash
-docker pull ghcr.io/opendevin/eval-swe-bench:full-v1.2.1
+docker pull ghcr.io/opendevin/eval-swe-bench:full-v1.0
 ```
 
 The Docker image contains several important directories:
@@ -33,9 +28,17 @@ To reproduce how we pack the image, check [this doc](./BUILD_TESTBED_AND_ENV.md)
 
 NOTE: We only support SWE-Bench lite for now. But modifying our existing scripts for full SWE-Bench should be quite straight forward.
 
-## Configure OpenDevin and your LLM
+## Test if your environment works
 
-Create a `config.toml` file if it does not exist at the root of the workspace.
+```bash
+python3 evaluation/swe_bench/swe_env_box.py
+```
+
+If you get to the interactive shell successfully, it means success!
+
+## Configure your LLM
+
+Create a `config.toml` file if not exists at the root of workspace.
 
 Add the following configurations:
 
@@ -45,13 +48,11 @@ max_iterations = 100
 cache_dir = "/tmp/cache"
 sandbox_container_image = "ghcr.io/opendevin/sandbox:latest"
 sandbox_type = "ssh"
+use_host_network = true
 ssh_hostname = "localhost"
 sandbox_timeout = 120
-
-# SWEBench eval specific
-use_host_network = false
+# eval specific
 run_as_devin = false
-enable_auto_lint = true
 
 # TODO: Change these to the model you want to evaluate
 [eval_gpt4_1106_preview]
@@ -66,55 +67,14 @@ api_key = "XXX"
 temperature = 0.0
 ```
 
-## Test if your environment works
-
-Make sure your Docker daemon is running, and you have pulled the `eval-swe-bench:full-v1.2`
-docker image. Then run this python script:
-
-```bash
-poetry run python evaluation/swe_bench/swe_env_box.py
-```
-
-If you get to the interactive shell successfully, it means your environment works!
-If you see an error, please make sure your `config.toml` contains all
-`SWEBench eval specific` settings as shown in the previous section.
-
 ## Run Inference on SWE-Bench Instances
 
 ```bash
-./evaluation/swe_bench/scripts/run_infer.sh [model_config] [agent] [eval_limit]
-# e.g., ./evaluation/swe_bench/scripts/run_infer.sh eval_gpt4_1106_preview CodeActAgent 300
+./evaluation/swe_bench/scripts/run_infer.sh eval_gpt4_1106_preview
 ```
 
-where `model_config` is mandatory, while `agent` and `eval_limit` are optional.
+You can replace `eval_gpt4_1106_preview` with any model you setted up in `config.toml`.
 
-`model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
-LLM settings, as defined in your `config.toml`.
-
-`agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting
-to `CodeActAgent`.
-
-`eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By
-default, the script evaluates the entire SWE-bench_Lite test set (300 issues). Note:
-in order to use `eval_limit`, you must also set `agent`.
-
-Let's say you'd like to run 10 instances using `eval_gpt4_1106_preview` and CodeActAgent,
-then your command would be:
-
-```bash
-./evaluation/swe_bench/scripts/run_infer.sh eval_gpt4_1106_preview CodeActAgent 10
-```
-
-If you would like to specify a list of tasks you'd like to benchmark on, you could
-create a `config.toml` under `./evaluation/swe_bench/` folder, and put a list
-attribute named `selected_ids`, e.g.
-
-```toml
-selected_ids = ['sphinx-doc__sphinx-8721', 'sympy__sympy-14774', 'scikit-learn__scikit-learn-10508']
-```
-
-Then only these tasks (rows whose `instance_id` is in the above list) will be evaluated.
-In this case, `eval_limit` option applies to tasks that are in the `selected_ids` list.
 
 ## Evaluate Generated Patches
 

evaluation/swe_bench/run_infer.py

@@ -9,12 +9,10 @@ import time
 from concurrent.futures import ProcessPoolExecutor
 
 import pandas as pd
-import toml
 import whatthepatch
 from datasets import load_dataset
 from tqdm import tqdm
 
-import agenthub
 from evaluation.swe_bench.swe_env_box import SWEBenchSSHBox
 from opendevin.controller.state.state import State
 from opendevin.core.config import args, config, get_llm_config_arg
@@ -24,8 +22,6 @@ from opendevin.core.main import main
 from opendevin.events.action import MessageAction
 from opendevin.events.serialization.event import event_to_dict
 
-USE_HINT_TEXT = os.environ.get('USE_HINT_TEXT', 'false') == 'true'
-
 
 def cleanup():
     print('Cleaning up child processes...')
@@ -62,19 +58,16 @@ def monologue_user_response(state: State) -> str:
 
 AGENT_CLS_TO_FAKE_USER_RESPONSE_FN = {
     'CodeActAgent': codeact_user_response,
-    'CodeActSWEAgent': codeact_user_response,
     'MonologueAgent': monologue_user_response,
 }
 
 AGENT_CLS_TO_INST_SUFFIX = {
-    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n',
-    'CodeActSWEAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n',
+    'CodeActAgent': 'When you think you have fixed the issue through code changes, please run the following command: <execute_bash> exit </execute_bash>.\n'
 }
 
 
 def get_test_result(instance, sandbox, workspace_dir_name):
     test_result = {'result': {}, 'metadata': {}}
-    # NOTE: if you need to do something in the sandbox to get the correctness metric, modify this function
     try:
         test_patch_parsed = whatthepatch.parse_patch(instance.test_patch)
         # get a list of filepaths that are involved in the patch
@@ -190,22 +183,14 @@ def get_test_result(instance, sandbox, workspace_dir_name):
 
 
 def process_instance(
-    instance: dict,
-    agent_class: str,
-    metadata: dict,
-    skip_workspace_mount: bool,
-    eval_output_dir: str,
-    reset_logger: bool = True,
+    instance, agent_class, metadata, skip_workspace_mount, reset_logger: bool = True
 ):
     workspace_mount_path = os.path.join(config.workspace_mount_path, '_eval_workspace')
     # create process-specific workspace dir
-    # if `not skip_workspace_mount` - we will create a workspace directory for EACH process
-    # so that different agent don't interfere with each other.
     if not skip_workspace_mount:
         workspace_mount_path = os.path.join(workspace_mount_path, str(os.getpid()))
         pathlib.Path(workspace_mount_path).mkdir(parents=True, exist_ok=True)
 
-    # Setup the logger properly, so you can run multi-processing to parallize the evaluation
     if reset_logger:
         # Set up logger
         log_file = os.path.join(
@@ -217,7 +202,7 @@ def process_instance(
         # add back the console handler to print ONE line
         logger.addHandler(get_console_handler())
         logger.info(
-            f'Starting evaluation for instance {instance.instance_id}.\nHint: run "tail -f {log_file}" to see live logs in a seperate shell'
+            f'Starting evaluation for instance {instance.instance_id}.\nLOG:   tail -f {log_file}'
         )
         # Remove all existing handlers from logger
         for handler in logger.handlers[:]:
@@ -227,84 +212,35 @@ def process_instance(
             logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
         )
         logger.addHandler(file_handler)
-    else:
-        logger.info(f'Starting evaluation for instance {instance.instance_id}.')
 
     if not skip_workspace_mount:
         logger.info(f'Process-specific workspace mounted at {workspace_mount_path}')
 
-    # NOTE: this is something special we do for SWE-Bench due to the reason described in the previous section
-    # You can omit this if you don't need to setup specialized sandbox
     workspace_dir_name = f'{instance.repo}__{instance.version}'.replace('/', '__')
     sandbox = SWEBenchSSHBox.get_box_for_instance(
         instance,
         workspace_dir_name,
         skip_workspace_mount=skip_workspace_mount,
         workspace_mount_path=workspace_mount_path,
-        sandbox_plugins=agenthub.Agent.get_cls(agent_class).sandbox_plugins,
     )
 
     # Prepare instruction
-    if agent_class == 'CodeActSWEAgent':
-        instruction = (
-            'We are currently solving the following issue within our repository. Here is the issue text:\n'
-            '--- BEGIN ISSUE ---\n'
-            f'{instance.problem_statement}\n'
-            '--- END ISSUE ---\n\n'
-        )
-
-        if USE_HINT_TEXT and instance.hints_text:
-            instruction += (
-                f'--- BEGIN HINTS ---\n{instance.hints_text}\n--- END HINTS ---\n'
-            )
-        instruction += f"""Now, you're going to solve this issue on your own. Your terminal session has started and you're in the repository's root directory. You can use any bash commands or the special interface to help you. Edit all the files you need to and run any checks or tests that you want.
-Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command.
-When you're satisfied with all of the changes you've made, you can run the following command: <execute_bash> exit </execute_bash>.
-Note however that you cannot use any interactive session commands (e.g. vim) in this environment, but you can write scripts and run them. E.g. you can write a python script and then run it with `python <script_name>.py`.
-
-NOTE ABOUT THE EDIT COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate indentation before each line!
-
-IMPORTANT TIPS:
-1. Always start by trying to replicate the bug that the issues discusses.
-    If the issue includes code for reproducing the bug, we recommend that you re-implement that in your environment, and run it to make sure you can reproduce the bug.
-    Then start trying to fix it.
-    When you think you've fixed the bug, re-run the bug reproduction script to make sure that the bug has indeed been fixed.
-
-    If the bug reproduction script does not print anything when it successfully runs, we recommend adding a print("Script completed successfully, no errors.") command at the end of the file,
-    so that you can be sure that the script indeed ran fine all the way through.
-
-2. If you run a command and it doesn't work, try running a different command. A command that did not work once will not work the second time unless you modify it!
-
-3. If you open a file and need to get to an area around a specific line that is not in the first 100 lines, say line 583, don't just use the scroll_down command multiple times. Instead, use the goto 583 command. It's much quicker.
-
-4. If the bug reproduction script requires inputting/reading a specific file, such as buggy-input.png, and you'd like to understand how to input that file, conduct a search in the existing repo code, to see whether someone else has already done that. Do this by running the command: find_file("buggy-input.png") If that doesn't work, use the linux 'find' command.
-
-5. Always make sure to look at the currently open file and the current working directory (which appears right after the currently open file). The currently open file might be in a different directory than the working directory! Note that some commands, such as 'create', open files, so they might change the current  open file.
-
-6. When editing files, it is easy to accidentally specify a wrong line number or to write code with incorrect indentation. Always check the code after you issue an edit to make sure that it reflects what you wanted to accomplish. If it didn't, issue another command to fix it.
-
-[Current directory: /workspace/{workspace_dir_name}]
-"""
-    else:
-        # Testing general agents
-        instruction = (
-            f'Please fix the following issue for the repository in /workspace/{workspace_dir_name}.\n'
-            'Environment has been set up for you to start working. You may assume all necessary tools are installed.\n\n'
-            '# Problem Statement\n'
-            f'{instance.problem_statement}\n\n'
-        )
-        if USE_HINT_TEXT and instance.hints_text:
-            instruction += f'# Hints\n{instance.hints_text}\n\n'
-        instruction += (
-            'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
-            'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
-            'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
-        )
-
-    # NOTE: You can actually set slightly different instruction for different agents
+    instruction = (
+        f'Please fix the following issue for the repository in /workspace/{workspace_dir_name}.\n'
+        'Environment has been set up for you to start working. You may assume all necessary tools are installed.\n\n'
+        '# Problem Statement\n'
+        f'{instance.problem_statement}\n\n'
+    )
+    if instance.hints_text:
+        instruction += f'# Hints\n{instance.hints_text}\n\n'
+    instruction += (
+        'IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n'
+        'You should NOT modify any existing test case files. If needed, you can add new test cases in a NEW file to reproduce the issue.\n'
+        'You SHOULD INCLUDE PROPER INDENTATION in your edit commands.\n'
+    )
     instruction += AGENT_CLS_TO_INST_SUFFIX.get(agent_class, '')
 
-    # Here's how you can run the agent (similar to the `main` function) and get the final task state
+    # Run the agent
     state: State = asyncio.run(
         main(
             instruction,
@@ -313,35 +249,27 @@ IMPORTANT TIPS:
         )
     )
 
-    # ======= THIS IS SWE-Bench specific =======
     # Get git patch
     git_patch = sandbox.get_diff_patch()
     logger.info(f'Got git diff for instance {instance.instance_id}')
-    # ==========================================
 
     # ======= Attempt to evaluate the agent's edits =======
-    # TODO: if you need to do something in the sandbox to get the correctness metric, modify this function
+    # Attempt to analyze the test patch to get involved filepaths
     test_result = get_test_result(instance, sandbox, workspace_dir_name)
 
-    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
-    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
-
     if state is None:
         raise ValueError('State should not be None.')
 
-    metrics = state.metrics.get() if state.metrics else None
-
     # Save the output
     output = {
         'instance_id': instance.instance_id,
-        'swe_instance': instance.to_dict(),  # SWE Bench specific
+        'swe_instance': instance.to_dict(),
         'instruction': instruction,
-        'git_patch': git_patch,  # SWE Bench specific
+        'git_patch': git_patch,
         'metadata': metadata,
         'history': [
             (event_to_dict(action), event_to_dict(obs)) for action, obs in state.history
         ],
-        'metrics': metrics,
         'error': state.error if state and state.error else None,
         'test_result': test_result,
     }
@@ -351,30 +279,11 @@ IMPORTANT TIPS:
     return output
 
 
-def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
-    file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.toml')
-    if os.path.exists(file_path):
-        with open(file_path, 'r') as file:
-            data = toml.load(file)
-            if 'selected_ids' in data:
-                selected_ids = data['selected_ids']
-                logger.info(
-                    f'Filtering {len(selected_ids)} tasks from "selected_ids"...'
-                )
-                subset = dataset[dataset[filter_column].isin(selected_ids)]
-                logger.info(f'Retained {subset.shape[0]} tasks after filtering')
-                return subset
-    return dataset
-
-
 if __name__ == '__main__':
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenDevin's repo
+    # Load the dataset
     dataset = load_dataset('princeton-nlp/SWE-bench_Lite')
-    swe_bench_tests = filter_dataset(dataset['test'].to_pandas(), 'instance_id')
+    swe_bench_tests = dataset['test'].to_pandas()
 
-    # Check https://github.com/OpenDevin/OpenDevin/blob/main/evaluation/swe_bench/README.md#configure-opendevin-and-your-llm
-    # for details of how to set `llm_config`
     if args.llm_config:
         specified_llm_config = get_llm_config_arg(args.llm_config)
         if specified_llm_config:
@@ -393,7 +302,7 @@ if __name__ == '__main__':
         eval_note += '_N_' + args.eval_note
     eval_output_dir = os.path.join(
         args.eval_output_dir,
-        'swe_bench_lite',
+        'swe_bench',
         agent_class,
         model_name + '_maxiter_' + str(max_iterations) + eval_note,
     )
@@ -410,16 +319,11 @@ if __name__ == '__main__':
         'max_iterations': max_iterations,
         'eval_output_dir': eval_output_dir,
         'start_time': time.strftime('%Y-%m-%d %H:%M:%S'),
-        # get the commit id of current repo for reproduciblity
+        # get the commit id of current repo
         'git_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'])
         .decode('utf-8')
         .strip(),
     }
-    _agent_cls = agenthub.Agent.get_cls(agent_class)
-    if hasattr(_agent_cls, 'system_message'):
-        metadata['system_message'] = _agent_cls.system_message
-    if hasattr(_agent_cls, 'in_context_example'):
-        metadata['in_context_example'] = _agent_cls.in_context_example
     logger.info(f'Metadata: {metadata}')
     with open(os.path.join(eval_output_dir, 'metadata.json'), 'w') as f:
         json.dump(metadata, f)
@@ -448,7 +352,6 @@ if __name__ == '__main__':
         f'Evaluation started with Agent {agent_class}, model {model_name}, max iterations {max_iterations}.'
     )
 
-    # =============================================
     # filter out finished instances
     new_swe_bench_tests = []
     for idx, instance in swe_bench_tests.iterrows():
@@ -463,11 +366,9 @@ if __name__ == '__main__':
     logger.info(
         f'Finished instances: {len(finished_instance_ids)}, Remaining instances: {len(swe_bench_tests)}'
     )
-    # =============================================
 
     pbar = tqdm(total=len(swe_bench_tests))
 
-    # This function tracks the progress AND write the output to a JSONL file
     def update_progress(future):
         pbar.update(1)
         output = future.result()
@@ -479,18 +380,14 @@ if __name__ == '__main__':
         output_fp.write(json.dumps(output) + '\n')
         output_fp.flush()
 
-    # This sets the multi-processing
     num_workers = args.eval_num_workers
     logger.info(f'Using {num_workers} workers for evaluation.')
 
-    # This is SWE-Bench specific - CodeActAgent doesn't require mounted workspace to work
     skip_workspace_mount = agent_class == 'CodeActAgent'
     logger.info(f'Skipping workspace mount: {skip_workspace_mount}')
-
     try:
         with ProcessPoolExecutor(num_workers) as executor:
             futures = []
-            # This is how we perform multi-processing
             for row_idx, instance in swe_bench_tests.iterrows():
                 future = executor.submit(
                     process_instance,
@@ -498,7 +395,6 @@ if __name__ == '__main__':
                     agent_class,
                     metadata,
                     skip_workspace_mount,
-                    eval_output_dir,
                     reset_logger=bool(num_workers > 1),
                 )
                 future.add_done_callback(update_progress)

evaluation/swe_bench/scripts/docker/Dockerfile.full.v1.0

@@ -10,4 +10,4 @@ RUN --mount=type=bind,source=./eval_workspace,target=/eval_workspace \
     /eval_workspace/ /swe_util/
 
 # pushd evaluation/SWE-bench
-# docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.1 -f ./scripts/docker/Dockerfile.full.v1.1 .
+# docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.0 -f ./scripts/docker/Dockerfile.full.v1.0 .

evaluation/swe_bench/scripts/docker/Dockerfile.full.v1.2

@@ -1,12 +0,0 @@
-FROM ghcr.io/opendevin/eval-swe-bench:full-v1.1
-
-RUN apt-get update && apt-get install -y \
-    libgl1-mesa-glx \
-    && rm -rf /var/lib/apt/lists/*
-
-# install basic dependencies for CodeActAgent
-RUN pip3 install --upgrade pip
-RUN pip3 install jupyterlab notebook jupyter_kernel_gateway flake8
-# TODO: those dependencies are needed for agentskills, we should pack them in a new sandbox image
-RUN pip3 install python-docx PyPDF2 python-pptx pylatexenc openai opencv-python
-# docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.2 -f ./scripts/docker/Dockerfile.full.v1.2 .

evaluation/swe_bench/scripts/docker/Dockerfile.full.v1.2.1

@@ -1,10 +0,0 @@
-FROM ghcr.io/opendevin/eval-swe-bench:full-v1.2
-
-# ================== Update OD-SWE-Bench ==================
-# copy everything except the folder of `eval_data` or `miniforge3`
-# typically, this should be the OD codebase
-RUN --mount=type=bind,source=./eval_workspace,target=/eval_workspace \
-    rsync -ar --progress /eval_workspace/OD-SWE-bench/ /swe_util/OD-SWE-bench
-
-# pushd evaluation/SWE-bench
-# docker build -t ghcr.io/opendevin/eval-swe-bench:full-v1.2.1 -f ./scripts/docker/Dockerfile.full.v1.2.1 .

evaluation/swe_bench/scripts/eval_infer.sh

@@ -26,7 +26,7 @@ docker run --rm \
     -e OD_SWE_BENCH=/swe_util/OD-SWE-bench \
     -e EVAL_DATA_DIR=/swe_util/eval_data \
     -w /swe_util \
-    ghcr.io/opendevin/eval-swe-bench:full-v1.2.1 \
+    ghcr.io/opendevin/eval-swe-bench:full-v1.0 \
     bash -c "./get_agent_report.sh --output-file /swe_bench_output/$FILE_NAME \
     --agent-name CodeActAgent \
     --dataset swe-bench-test-lite \

evaluation/swe_bench/scripts/run_infer.sh

@@ -1,52 +1,15 @@
 #!/bin/bash
+
+AGENT=CodeActAgent
+AGENT_VERSION=v1.3
 MODEL_CONFIG=$1
-AGENT=$2
-EVAL_LIMIT=$3
-MAX_ITER=$4
 
-if [ -z "$AGENT" ]; then
-  echo "Agent not specified, use default CodeActAgent"
-  AGENT="CodeActAgent"
-fi
+# You should add $MODEL_CONFIG in your `config.toml`
 
-if [ -z "$MAX_ITER" ]; then
-  echo "MAX_ITER not specified, use default 30"
-  MAX_ITER=30
-fi
-
-# IMPORTANT: Because Agent's prompt changes fairly often in the rapidly evolving codebase of OpenDevin
-# We need to track the version of Agent in the evaluation to make sure results are comparable
-AGENT_VERSION=v$(poetry run python -c "import agenthub; from opendevin.controller.agent import Agent; print(Agent.get_cls('$AGENT').VERSION)")
-
-echo "AGENT: $AGENT"
-echo "AGENT_VERSION: $AGENT_VERSION"
-echo "MODEL_CONFIG: $MODEL_CONFIG"
-
-# Default to use Hint
-if [ -z "$USE_HINT_TEXT" ]; then
-  export USE_HINT_TEXT=true
-fi
-echo "USE_HINT_TEXT: $USE_HINT_TEXT"
-EVAL_NOTE="$AGENT_VERSION"
-# if not using Hint, add -no-hint to the eval note
-if [ "$USE_HINT_TEXT" = false ]; then
-  EVAL_NOTE="$EVAL_NOTE-no-hint"
-fi
-
-unset SANDBOX_ENV_GITHUB_TOKEN # prevent the agent from using the github token to push
-
-COMMAND="poetry run python evaluation/swe_bench/run_infer.py \
+poetry run python3 evaluation/swe_bench/run_infer.py \
   --agent-cls $AGENT \
   --llm-config $MODEL_CONFIG \
-  --max-iterations $MAX_ITER \
+  --max-iterations 50 \
   --max-chars 10000000 \
   --eval-num-workers 8 \
-  --eval-note $EVAL_NOTE"
-
-if [ -n "$EVAL_LIMIT" ]; then
-  echo "EVAL_LIMIT: $EVAL_LIMIT"
-  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
-fi
-
-# Run the command
-eval $COMMAND
+  --eval-note $AGENT_VERSION

evaluation/swe_bench/swe_env_box.py

@@ -1,18 +1,12 @@
 import sys
 import uuid
 
-from datasets import load_dataset
-
 from opendevin.core.config import config
 from opendevin.core.logger import opendevin_logger as logger
 from opendevin.runtime.docker.ssh_box import DockerSSHBox
-from opendevin.runtime.plugins import (
-    AgentSkillsRequirement,
-    JupyterRequirement,
-    PluginRequirement,
-)
+from opendevin.runtime.plugins import JupyterRequirement, SWEAgentCommandsRequirement
 
-SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.2.1'
+SWE_BENCH_CONTAINER_IMAGE = 'ghcr.io/opendevin/eval-swe-bench:full-v1.0'
 
 
 class SWEBenchSSHBox(DockerSSHBox):
@@ -24,15 +18,12 @@ class SWEBenchSSHBox(DockerSSHBox):
         swe_instance_id: str | None = None,
         swe_instance: dict | None = None,
         skip_workspace_mount: bool = True,
-        sandbox_plugins: list[PluginRequirement] = [],  # noqa: B006
-        workspace_dir_name: str | None = None,
     ):
         if swe_instance_id is None:
             raise ValueError('swe_instance_id must be provided!')
         self.swe_instance_id = swe_instance_id
         self.swe_instance = swe_instance
         self.skip_workspace_mount = skip_workspace_mount
-        self.workspace_dir_name = workspace_dir_name
 
         assert (
             container_image is not None
@@ -40,7 +31,6 @@ class SWEBenchSSHBox(DockerSSHBox):
         # Need to run as root to use SWEBench container
         sid = f'swe_bench_{swe_instance_id}' + str(uuid.uuid4())
         super().__init__(container_image, timeout, sid)
-        self.init_plugins(sandbox_plugins)
 
         exit_code, output = self.execute('mv ~/.bashrc ~/.bashrc.bak')
         assert exit_code == 0, f'Failed to backup ~/.bashrc: {output}'
@@ -73,30 +63,21 @@ class SWEBenchSSHBox(DockerSSHBox):
         cls,
         instance,
         workspace_dir_name=None,
+        n_tries=5,
         skip_workspace_mount: bool = True,
         workspace_mount_path: str | None = None,
-        sandbox_plugins: list[PluginRequirement] = [],  # noqa: B006
     ) -> 'SWEBenchSSHBox':
         if workspace_dir_name is None:
             workspace_dir_name = f"{instance['repo']}__{instance['version']}".replace(
                 '/', '__'
             )
-        old_workspace_base = config.workspace_base
-        old_workspace_mount_path = config.workspace_mount_path
         config.workspace_base = workspace_mount_path
         config.workspace_mount_path = workspace_mount_path
-
-        # linting python after editing helps LLM fix indentations
-        config.enable_auto_lint = True
-        # Need to run as root to use SWEBench container
-        config.run_as_devin = False
         sandbox = cls(
             container_image=SWE_BENCH_CONTAINER_IMAGE,
             swe_instance_id=instance['instance_id'],
             swe_instance=instance,
             skip_workspace_mount=skip_workspace_mount,
-            sandbox_plugins=sandbox_plugins,
-            workspace_dir_name=workspace_dir_name,
         )
         logger.info(f"SSH box started for instance {instance['instance_id']}.")
 
@@ -118,21 +99,11 @@ class SWEBenchSSHBox(DockerSSHBox):
         if exit_code != 0:
             logger.error(f'Failed to remove remote: {output}')
             sys.exit(1)
-
-        # restore workspace_base and workspace_mount_path
-        config.workspace_base = old_workspace_base
-        config.workspace_mount_path = old_workspace_mount_path
         return sandbox
 
     def get_diff_patch(self):
         # add everything to the index
-        exit_code, output = self.execute(f'cd /workspace/{self.workspace_dir_name}')
-        if exit_code != 0:
-            logger.error('Failed to cd to the repo')
-            return ''
-
-        # add everything to the index
-        exit_code, output = self.execute('git add -A')
+        exit_code, output = self.execute('git add --all')
         if exit_code != 0:
             logger.error('Failed to add everything to the index')
             return ''
@@ -148,20 +119,25 @@ class SWEBenchSSHBox(DockerSSHBox):
 
 
 if __name__ == '__main__':
-    # NOTE: It is preferable to load datasets from huggingface datasets and perform post-processing
-    # so we don't need to manage file uploading to OpenDevin's repo
-    dataset = load_dataset('princeton-nlp/SWE-bench_Lite')
-    swe_bench_tests = dataset['test'].to_pandas()
+    EXAMPLE_INSTANCE = {
+        'repo': 'django/django',
+        'instance_id': 'django__django-11099',
+        'base_commit': 'd26b2424437dabeeca94d7900b37d2df4410da0c',
+        'patch': "diff --git a/django/contrib/auth/validators.py b/django/contrib/auth/validators.py\n--- a/django/contrib/auth/validators.py\n+++ b/django/contrib/auth/validators.py\n@@ -7,7 +7,7 @@\n \n @deconstructible\n class ASCIIUsernameValidator(validators.RegexValidator):\n-    regex = r'^[\\w.@+-]+$'\n+    regex = r'^[\\w.@+-]+\\Z'\n     message = _(\n         'Enter a valid username. This value may contain only English letters, '\n         'numbers, and @/./+/-/_ characters.'\n@@ -17,7 +17,7 @@ class ASCIIUsernameValidator(validators.RegexValidator):\n \n @deconstructible\n class UnicodeUsernameValidator(validators.RegexValidator):\n-    regex = r'^[\\w.@+-]+$'\n+    regex = r'^[\\w.@+-]+\\Z'\n     message = _(\n         'Enter a valid username. This value may contain only letters, '\n         'numbers, and @/./+/-/_ characters.'\n",
+        'test_patch': "diff --git a/tests/auth_tests/test_validators.py b/tests/auth_tests/test_validators.py\n--- a/tests/auth_tests/test_validators.py\n+++ b/tests/auth_tests/test_validators.py\n@@ -237,7 +237,7 @@ def test_unicode_validator(self):\n         invalid_usernames = [\n             \"o'connell\", \"عبد ال\",\n             \"zerowidth\\u200Bspace\", \"nonbreaking\\u00A0space\",\n-            \"en\\u2013dash\",\n+            \"en\\u2013dash\", 'trailingnewline\\u000A',\n         ]\n         v = validators.UnicodeUsernameValidator()\n         for valid in valid_usernames:\n@@ -250,7 +250,7 @@ def test_unicode_validator(self):\n \n     def test_ascii_validator(self):\n         valid_usernames = ['glenn', 'GLEnN', 'jean-marc']\n-        invalid_usernames = [\"o'connell\", 'Éric', 'jean marc', \"أحمد\"]\n+        invalid_usernames = [\"o'connell\", 'Éric', 'jean marc', \"أحمد\", 'trailingnewline\\n']\n         v = validators.ASCIIUsernameValidator()\n         for valid in valid_usernames:\n             with self.subTest(valid=valid):\n",
+        'problem_statement': "UsernameValidator allows trailing newline in usernames\nDescription\n\t\nASCIIUsernameValidator and UnicodeUsernameValidator use the regex \nr'^[\\w.@+-]+$'\nThe intent is to only allow alphanumeric characters as well as ., @, +, and -. However, a little known quirk of Python regexes is that $ will also match a trailing newline. Therefore, the user name validators will accept usernames which end with a newline. You can avoid this behavior by instead using \\A and \\Z to terminate regexes. For example, the validator regex could be changed to\nr'\\A[\\w.@+-]+\\Z'\nin order to reject usernames that end with a newline.\nI am not sure how to officially post a patch, but the required change is trivial - using the regex above in the two validators in contrib.auth.validators.\n",
+        'hints_text': '',
+        'created_at': '2019-03-20T03:46:18Z',
+        'version': '3.0',
+        'FAIL_TO_PASS': '["test_ascii_validator (auth_tests.test_validators.UsernameValidatorsTests)", "test_unicode_validator (auth_tests.test_validators.UsernameValidatorsTests)", "test_help_text (auth_tests.test_validators.UserAttributeSimilarityValidatorTest)"]',
+        'PASS_TO_PASS': '["test_help_text (auth_tests.test_validators.MinimumLengthValidatorTest)", "test_validate (auth_tests.test_validators.MinimumLengthValidatorTest)", "test_help_text (auth_tests.test_validators.NumericPasswordValidatorTest)", "test_validate (auth_tests.test_validators.NumericPasswordValidatorTest)", "test_validate (auth_tests.test_validators.UserAttributeSimilarityValidatorTest)", "test_validate_property (auth_tests.test_validators.UserAttributeSimilarityValidatorTest)", "test_empty_password_validator_help_text_html (auth_tests.test_validators.PasswordValidationTest)", "test_get_default_password_validators (auth_tests.test_validators.PasswordValidationTest)", "test_get_password_validators_custom (auth_tests.test_validators.PasswordValidationTest)", "test_password_changed (auth_tests.test_validators.PasswordValidationTest)", "test_password_changed_with_custom_validator (auth_tests.test_validators.PasswordValidationTest)", "test_password_validators_help_text_html (auth_tests.test_validators.PasswordValidationTest)", "test_password_validators_help_text_html_escaping (auth_tests.test_validators.PasswordValidationTest)", "test_password_validators_help_texts (auth_tests.test_validators.PasswordValidationTest)", "test_validate_password (auth_tests.test_validators.PasswordValidationTest)", "test_help_text (auth_tests.test_validators.CommonPasswordValidatorTest)", "test_validate (auth_tests.test_validators.CommonPasswordValidatorTest)", "test_validate_custom_list (auth_tests.test_validators.CommonPasswordValidatorTest)", "test_validate_django_supplied_file (auth_tests.test_validators.CommonPasswordValidatorTest)"]',
+        'environment_setup_commit': '419a78300f7cd27611196e1e464d50fd0385ff27',
+    }
 
-    # INSTANCE_ID = 'django__django-11099'
-    INSTANCE_ID = 'astropy__astropy-12907'
-    swe_bench_tests = swe_bench_tests[swe_bench_tests['instance_id'] == INSTANCE_ID]
-    EXAMPLE_INSTANCE = swe_bench_tests.iloc[0].to_dict()
+    sandbox = SWEBenchSSHBox.get_box_for_instance(instance=EXAMPLE_INSTANCE)
 
-    sandbox = SWEBenchSSHBox.get_box_for_instance(
-        instance=EXAMPLE_INSTANCE,
-        sandbox_plugins=[AgentSkillsRequirement(), JupyterRequirement()],
-    )
+    # in actual eval, this will be initialized by the controller
+    sandbox.init_plugins([JupyterRequirement(), SWEAgentCommandsRequirement()])
 
     # PRE TEST
     exit_code, output = sandbox.execute('cd $REPO_PATH')
@@ -174,7 +150,9 @@ if __name__ == '__main__':
     logger.info(f'git apply $SWE_TASK_DIR/test.patch: {output}')
 
     # TEST
-    exit_code, output = sandbox.execute('$TEST_CMD')
+    exit_code, output = sandbox.execute(
+        './tests/runtests.py --verbosity 2 auth_tests.test_validators'
+    )
     assert exit_code == 1, 'Expected exit code 1 (since this is a FAIL_TO_PASS)'
     logger.info(f'$TEST_CMD:\n{output}')
 
@@ -184,7 +162,9 @@ if __name__ == '__main__':
     logger.info(f'git apply $SWE_TASK_DIR/gold.patch: {output}')
 
     # TEST
-    exit_code, output = sandbox.execute('$TEST_CMD')
+    exit_code, output = sandbox.execute(
+        './tests/runtests.py --verbosity 2 auth_tests.test_validators'
+    )
     assert exit_code == 0, 'Expected exit code 0 (since we applied the gold patch)'
     logger.info(f'$TEST_CMD:\n{output}')
 

frontend/package.json

@@ -8,43 +8,42 @@
   },
   "dependencies": {
     "@monaco-editor/react": "^4.6.0",
-    "@nextui-org/react": "^2.3.6",
-    "@react-types/shared": "^3.23.1",
-    "@reduxjs/toolkit": "^2.2.5",
+    "@nextui-org/react": "^2.2.10",
+    "@react-types/shared": "^3.22.1",
+    "@reduxjs/toolkit": "^2.2.2",
     "@vitejs/plugin-react": "^4.2.1",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.4.0",
-    "clsx": "^2.1.1",
+    "clsx": "^2.1.0",
     "eslint-config-airbnb-typescript": "^18.0.0",
-    "framer-motion": "^11.2.6",
-    "i18next": "^23.11.5",
-    "i18next-browser-languagedetector": "^8.0.0",
-    "i18next-http-backend": "^2.5.2",
-    "jose": "^5.3.0",
-    "monaco-editor": "^0.49.0",
-    "react": "^18.3.1",
-    "react-dom": "^18.3.1",
+    "framer-motion": "^11.0.24",
+    "i18next": "^23.10.1",
+    "i18next-browser-languagedetector": "^7.2.1",
+    "i18next-http-backend": "^2.5.0",
+    "jose": "^5.2.3",
+    "monaco-editor": "^0.47.0",
+    "react": "^18.2.0",
+    "react-accessible-treeview": "^2.8.3",
+    "react-dom": "^18.2.0",
     "react-highlight": "^0.15.0",
     "react-hot-toast": "^2.4.1",
-    "react-i18next": "^14.1.2",
-    "react-icons": "^5.2.1",
+    "react-i18next": "^14.1.0",
+    "react-icons": "^5.0.1",
     "react-markdown": "^9.0.1",
-    "react-redux": "^9.1.2",
-    "react-router-dom": "^6.23.1",
+    "react-redux": "^9.1.0",
     "react-syntax-highlighter": "^15.5.0",
-    "tailwind-merge": "^2.3.0",
-    "vite": "^5.2.11",
-    "web-vitals": "^3.5.2"
+    "tailwind-merge": "^2.2.2",
+    "vite": "^5.1.6",
+    "web-vitals": "^2.1.4"
   },
   "scripts": {
-    "start": "npm run make-i18n && vite",
+    "start": "vite",
     "build": "tsc && vite build",
     "test": "vitest run",
     "preview": "vite preview",
     "make-i18n": "node scripts/make-i18n-translations.cjs",
     "prelint": "npm run make-i18n",
     "lint": "eslint src --ext .ts,.tsx,.js && prettier --check src/**/*.{ts,tsx}",
-    "lint:fix": "eslint src --ext .ts,.tsx,.js --fix && prettier --write src/**/*.{ts,tsx}",
     "prepare": "cd .. && husky install frontend/.husky"
   },
   "husky": {
@@ -59,17 +58,17 @@
     ]
   },
   "devDependencies": {
-    "@tailwindcss/typography": "^0.5.13",
-    "@testing-library/jest-dom": "^6.4.5",
-    "@testing-library/react": "^15.0.7",
+    "@tailwindcss/typography": "^0.5.12",
+    "@testing-library/jest-dom": "^6.4.2",
+    "@testing-library/react": "^13.4.0",
     "@testing-library/user-event": "^13.5.0",
     "@types/node": "^18.0.0 ",
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
+    "@types/react": "^18.2.66",
+    "@types/react-dom": "^18.2.22",
     "@types/react-highlight": "^0.12.8",
-    "@types/react-syntax-highlighter": "^15.5.13",
-    "@typescript-eslint/eslint-plugin": "^7.10.0",
-    "@typescript-eslint/parser": "^7.10.0",
+    "@types/react-syntax-highlighter": "^15.5.11",
+    "@typescript-eslint/eslint-plugin": "^7.4.0",
+    "@typescript-eslint/parser": "^7.0.0",
     "autoprefixer": "^10.4.19",
     "eslint": "^8.57.0",
     "eslint-config-airbnb": "^19.0.4",
@@ -79,16 +78,16 @@
     "eslint-plugin-jsx-a11y": "^6.8.0",
     "eslint-plugin-prettier": "^5.1.3",
     "eslint-plugin-react": "^7.34.1",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "husky": "^9.0.11",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "husky": "^8.0.0",
     "jsdom": "^24.0.0",
-    "lint-staged": "^15.2.4",
+    "lint-staged": "^15.2.2",
     "postcss": "^8.4.38",
     "prettier": "^3.2.5",
     "tailwindcss": "^3.4.2",
     "typescript": "^5.4.3",
     "vite-tsconfig-paths": "^4.3.2",
-    "vitest": "^1.6.0"
+    "vitest": "^1.5.0"
   },
   "packageManager": "npm@10.5.0",
   "volta": {

frontend/src/App.tsx

@@ -1,5 +1,5 @@
 import { useDisclosure } from "@nextui-org/react";
-import React, { useEffect } from "react";
+import React, { useEffect, useState } from "react";
 import { Toaster } from "react-hot-toast";
 import CogTooth from "#/assets/cog-tooth";
 import ChatInterface from "#/components/chat/ChatInterface";
@@ -8,13 +8,15 @@ import { Container, Orientation } from "#/components/Resizable";
 import Workspace from "#/components/Workspace";
 import LoadPreviousSessionModal from "#/components/modals/load-previous-session/LoadPreviousSessionModal";
 import SettingsModal from "#/components/modals/settings/SettingsModal";
+import { fetchMsgTotal } from "#/services/session";
+import Socket from "#/services/socket";
+import { ResFetchMsgTotal } from "#/types/ResponseType";
 import "./App.css";
 import AgentControlBar from "./components/AgentControlBar";
 import AgentStatusBar from "./components/AgentStatusBar";
 import Terminal from "./components/terminal/Terminal";
-import Session from "#/services/session";
-import { getToken } from "#/services/auth";
-import { settingsAreUpToDate } from "#/services/settings";
+import { initializeAgent } from "./services/agent";
+import { settingsAreUpToDate } from "./services/settings";
 
 interface Props {
   setSettingOpen: (isOpen: boolean) => void;
@@ -41,6 +43,8 @@ function Controls({ setSettingOpen }: Props): JSX.Element {
 let initOnce = false;
 
 function App(): JSX.Element {
+  const [isWarned, setIsWarned] = useState(false);
+
   const {
     isOpen: settingsModalIsOpen,
     onOpen: onSettingsModalOpen,
@@ -53,18 +57,31 @@ function App(): JSX.Element {
     onOpenChange: onLoadPreviousSessionModalOpenChange,
   } = useDisclosure();
 
+  const getMsgTotal = () => {
+    if (isWarned) return;
+    fetchMsgTotal()
+      .then((data: ResFetchMsgTotal) => {
+        if (data.msg_total > 0) {
+          onLoadPreviousSessionModalOpen();
+          setIsWarned(true);
+        }
+      })
+      .catch();
+  };
+
   useEffect(() => {
     if (initOnce) return;
     initOnce = true;
 
     if (!settingsAreUpToDate()) {
       onSettingsModalOpen();
-    } else if (getToken()) {
-      onLoadPreviousSessionModalOpen();
     } else {
-      Session.startNewSession();
+      initializeAgent();
     }
 
+    Socket.registerCallback("open", [getMsgTotal]);
+
+    getMsgTotal();
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 

frontend/src/api/index.ts

@@ -1,19 +1,9 @@
-export interface FeedbackData {
-  email: string;
-  token: string;
-  feedback: "positive" | "negative";
-  trajectory: unknown[];
+export async function fetchModels() {
+  const response = await fetch(`/api/litellm-models`);
+  return response.json();
 }
 
-export const sendFeedback = async (data: FeedbackData) =>
-  fetch(
-    "https://kttkfkoju5.execute-api.us-east-2.amazonaws.com/od-share-trajectory",
-    {
-      method: "POST",
-      mode: "no-cors",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify(data),
-    },
-  );
+export async function fetchAgents() {
+  const response = await fetch(`/api/agents`);
+  return response.json();
+}

frontend/src/components/AgentControlBar.tsx

@@ -5,6 +5,7 @@ import ArrowIcon from "#/assets/arrow";
 import PauseIcon from "#/assets/pause";
 import PlayIcon from "#/assets/play";
 import { changeAgentState } from "#/services/agentStateService";
+import { clearMsgs } from "#/services/session";
 import store, { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import { clearMessages } from "#/state/chatSlice";
@@ -72,6 +73,7 @@ function AgentControlBar() {
     }
 
     if (action === AgentState.STOPPED) {
+      clearMsgs().then().catch();
       store.dispatch(clearMessages());
     } else {
       setIsLoading(true);
@@ -84,6 +86,7 @@ function AgentControlBar() {
   useEffect(() => {
     if (curAgentState === desiredState) {
       if (curAgentState === AgentState.STOPPED) {
+        clearMsgs().then().catch();
         store.dispatch(clearMessages());
       }
       setIsLoading(false);

frontend/src/components/AgentStatusBar.tsx

@@ -5,56 +5,46 @@ import { I18nKey } from "#/i18n/declaration";
 import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 
-enum IndicatorColor {
-  BLUE = "bg-blue-500",
-  GREEN = "bg-green-500",
-  ORANGE = "bg-orange-500",
-  YELLOW = "bg-yellow-500",
-  RED = "bg-red-500",
-  DARK_ORANGE = "bg-orange-800",
-}
+const AgentStatusMap: { [k: string]: { message: string; indicator: string } } =
+  {
+    [AgentState.INIT]: {
+      message: "Agent is initialized, waiting for task...",
+      indicator: "bg-blue-500",
+    },
+    [AgentState.RUNNING]: {
+      message: "Agent is running task...",
+      indicator: "bg-green-500",
+    },
+    [AgentState.AWAITING_USER_INPUT]: {
+      message: "Agent is awaiting user input...",
+      indicator: "bg-orange-500",
+    },
+    [AgentState.PAUSED]: {
+      message: "Agent has paused.",
+      indicator: "bg-yellow-500",
+    },
+    [AgentState.LOADING]: {
+      message: "Agent is initializing...",
+      indicator: "bg-yellow-500",
+    },
+    [AgentState.STOPPED]: {
+      message: "Agent has stopped.",
+      indicator: "bg-red-500",
+    },
+    [AgentState.FINISHED]: {
+      message: "Agent has finished the task.",
+      indicator: "bg-green-500",
+    },
+    [AgentState.ERROR]: {
+      message: "Agent encountered an error.",
+      indicator: "bg-red-500",
+    },
+  };
 
 function AgentStatusBar() {
   const { t } = useTranslation();
   const { curAgentState } = useSelector((state: RootState) => state.agent);
 
-  const AgentStatusMap: {
-    [k: string]: { message: string; indicator: IndicatorColor };
-  } = {
-    [AgentState.INIT]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_INIT_MESSAGE),
-      indicator: IndicatorColor.BLUE,
-    },
-    [AgentState.RUNNING]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_RUNNING_MESSAGE),
-      indicator: IndicatorColor.GREEN,
-    },
-    [AgentState.AWAITING_USER_INPUT]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_AWAITING_USER_INPUT_MESSAGE),
-      indicator: IndicatorColor.ORANGE,
-    },
-    [AgentState.PAUSED]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_PAUSED_MESSAGE),
-      indicator: IndicatorColor.YELLOW,
-    },
-    [AgentState.LOADING]: {
-      message: t(I18nKey.CHAT_INTERFACE$INITIALIZING_AGENT_LOADING_MESSAGE),
-      indicator: IndicatorColor.DARK_ORANGE,
-    },
-    [AgentState.STOPPED]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_STOPPED_MESSAGE),
-      indicator: IndicatorColor.RED,
-    },
-    [AgentState.FINISHED]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_FINISHED_MESSAGE),
-      indicator: IndicatorColor.GREEN,
-    },
-    [AgentState.ERROR]: {
-      message: t(I18nKey.CHAT_INTERFACE$AGENT_ERROR_MESSAGE),
-      indicator: IndicatorColor.RED,
-    },
-  };
-
   // TODO: Extend the agent status, e.g.:
   // - Agent is typing
   // - Agent is initializing
@@ -63,12 +53,23 @@ function AgentStatusBar() {
   // - Agent is not available
   return (
     <div className="flex items-center">
-      <div
-        className={`w-3 h-3 mr-2 rounded-full animate-pulse ${AgentStatusMap[curAgentState].indicator}`}
-      />
-      <span className="text-sm text-stone-400">
-        {AgentStatusMap[curAgentState].message}
-      </span>
+      {curAgentState !== AgentState.LOADING ? (
+        <>
+          <div
+            className={`w-3 h-3 mr-2 rounded-full animate-pulse ${AgentStatusMap[curAgentState].indicator}`}
+          />
+          <span className="text-sm text-stone-400">
+            {AgentStatusMap[curAgentState].message}
+          </span>
+        </>
+      ) : (
+        <>
+          <div className="w-3 h-3 mr-3 bg-orange-800 rounded-full animate-pulse" />
+          <span className="text-sm text-stone-400">
+            {t(I18nKey.CHAT_INTERFACE$INITIALZING_AGENT_LOADING_MESSAGE)}
+          </span>
+        </>
+      )}
     </div>
   );
 }

frontend/src/components/Browser.tsx

@@ -24,12 +24,7 @@ function Browser(): JSX.Element {
       </div>
       <div className="overflow-y-auto grow scrollbar-hide rounded-xl">
         {screenshotSrc ? (
-          <img
-            src={imgSrc}
-            style={{ objectFit: "contain", width: "100%", height: "auto" }}
-            className="rounded-xl"
-            alt="Browser Screenshot"
-          />
+          <img src={imgSrc} className="rounded-xl" alt="Browser Screenshot" />
         ) : (
           <div className="flex flex-col items-center h-full justify-center">
             <IoIosGlobe size={100} />

frontend/src/components/CodeEditor.tsx

@@ -1,23 +1,31 @@
 import Editor, { Monaco } from "@monaco-editor/react";
 import { Tab, Tabs } from "@nextui-org/react";
 import type { editor } from "monaco-editor";
-import React, { useMemo } from "react";
+import React, { useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { VscCode } from "react-icons/vsc";
-import { useSelector } from "react-redux";
+import { useDispatch, useSelector } from "react-redux";
 import { I18nKey } from "#/i18n/declaration";
+import { selectFile } from "#/services/fileService";
+import { setCode } from "#/state/codeSlice";
 import { RootState } from "#/store";
 import FileExplorer from "./file-explorer/FileExplorer";
+import { CodeEditorContext } from "./CodeEditorContext";
 
 function CodeEditor(): JSX.Element {
   const { t } = useTranslation();
-  const code = useSelector((state: RootState) => state.code.code);
-  const activeFilepath = useSelector((state: RootState) => state.code.path);
-
+  const [selectedFileAbsolutePath, setSelectedFileAbsolutePath] = useState("");
   const selectedFileName = useMemo(() => {
-    const paths = activeFilepath.split("/");
+    const paths = selectedFileAbsolutePath.split("/");
     return paths[paths.length - 1];
-  }, [activeFilepath]);
+  }, [selectedFileAbsolutePath]);
+  const codeEditorContext = useMemo(
+    () => ({ selectedFileAbsolutePath }),
+    [selectedFileAbsolutePath],
+  );
+
+  const dispatch = useDispatch();
+  const code = useSelector((state: RootState) => state.code.code);
 
   const handleEditorDidMount = (
     editor: editor.IStandaloneCodeEditor,
@@ -37,44 +45,56 @@ function CodeEditor(): JSX.Element {
     monaco.editor.setTheme("my-theme");
   };
 
+  const onSelectFile = async (absolutePath: string) => {
+    const paths = absolutePath.split("/");
+    const rootlessPath = paths.slice(1).join("/");
+
+    setSelectedFileAbsolutePath(absolutePath);
+
+    const newCode = await selectFile(rootlessPath);
+    dispatch(setCode(newCode));
+  };
+
   return (
     <div className="flex h-full w-full bg-neutral-900 transition-all duration-500 ease-in-out">
-      <FileExplorer />
-      <div className="flex flex-col min-h-0 w-full">
-        <Tabs
-          disableCursorAnimation
-          classNames={{
-            base: "border-b border-divider border-neutral-600 mb-4",
-            tabList:
-              "w-full relative rounded-none bg-neutral-900 p-0 border-divider",
-            cursor: "w-full bg-neutral-600 rounded-none",
-            tab: "max-w-fit px-4 h-[36px]",
-            tabContent: "group-data-[selected=true]:text-white",
-          }}
-          aria-label="Options"
-        >
-          <Tab
-            key={selectedFileName.toLocaleLowerCase()}
-            title={selectedFileName}
-          />
-        </Tabs>
-        <div className="flex grow items-center justify-center">
-          {selectedFileName === "" ? (
-            <div className="flex flex-col items-center text-neutral-400">
-              <VscCode size={100} />
-              {t(I18nKey.CODE_EDITOR$EMPTY_MESSAGE)}
-            </div>
-          ) : (
-            <Editor
-              height="100%"
-              path={selectedFileName.toLocaleLowerCase()}
-              defaultValue=""
-              value={code}
-              onMount={handleEditorDidMount}
+      <CodeEditorContext.Provider value={codeEditorContext}>
+        <FileExplorer onFileClick={onSelectFile} />
+        <div className="flex flex-col min-h-0 w-full">
+          <Tabs
+            disableCursorAnimation
+            classNames={{
+              base: "border-b border-divider border-neutral-600 mb-4",
+              tabList:
+                "w-full relative rounded-none bg-neutral-900 p-0 border-divider",
+              cursor: "w-full bg-neutral-600 rounded-none",
+              tab: "max-w-fit px-4 h-[36px]",
+              tabContent: "group-data-[selected=true]:text-white",
+            }}
+            aria-label="Options"
+          >
+            <Tab
+              key={selectedFileName.toLocaleLowerCase()}
+              title={selectedFileName}
             />
-          )}
+          </Tabs>
+          <div className="flex grow items-center justify-center">
+            {selectedFileName === "" ? (
+              <div className="flex flex-col items-center text-neutral-400">
+                <VscCode size={100} />
+                {t(I18nKey.CODE_EDITOR$EMPTY_MESSAGE)}
+              </div>
+            ) : (
+              <Editor
+                height="100%"
+                path={selectedFileName.toLocaleLowerCase()}
+                defaultValue=""
+                value={code}
+                onMount={handleEditorDidMount}
+              />
+            )}
+          </div>
         </div>
-      </div>
+      </CodeEditorContext.Provider>
     </div>
   );
 }

frontend/src/components/CodeEditorContext.ts

@@ -0,0 +1,5 @@
+import { createContext } from "react";
+
+export const CodeEditorContext = createContext({
+  selectedFileAbsolutePath: "",
+});

frontend/src/components/chat/ChatInterface.test.tsx

@@ -5,7 +5,7 @@ import { act } from "react-dom/test-utils";
 import userEvent from "@testing-library/user-event";
 import { renderWithProviders } from "test-utils";
 import ChatInterface from "./ChatInterface";
-import Session from "#/services/session";
+import Socket from "#/services/socket";
 import ActionType from "#/types/ActionType";
 import { addAssistantMessage } from "#/state/chatSlice";
 import AgentState from "#/types/AgentState";
@@ -15,17 +15,16 @@ vi.mock("#/hooks/useTyping", () => ({
   useTyping: vi.fn((text: string) => text),
 }));
 
-const sessionSpy = vi.spyOn(Session, "send");
-vi.spyOn(Session, "isConnected").mockImplementation(() => true);
+const socketSpy = vi.spyOn(Socket, "send");
 
 // This is for the scrollview ref in Chat.tsx
 // TODO: Move this into test setup
 HTMLElement.prototype.scrollTo = vi.fn(() => {});
 
 describe("ChatInterface", () => {
-  it("should render empty message list and input", () => {
+  it("should render the messages and input", () => {
     renderWithProviders(<ChatInterface />);
-    expect(screen.queryAllByTestId("message")).toHaveLength(0);
+    expect(screen.queryAllByTestId("message")).toHaveLength(1); // initial welcome message only
   });
 
   it("should render the new message the user has typed", async () => {
@@ -66,7 +65,7 @@ describe("ChatInterface", () => {
     expect(screen.getByText("Hello to you!")).toBeInTheDocument();
   });
 
-  it("should send the a start event to the Session", () => {
+  it("should send the a start event to the Socket", () => {
     renderWithProviders(<ChatInterface />, {
       preloadedState: {
         agent: {
@@ -84,10 +83,10 @@ describe("ChatInterface", () => {
       action: ActionType.MESSAGE,
       args: { content: "my message" },
     };
-    expect(sessionSpy).toHaveBeenCalledWith(JSON.stringify(event));
+    expect(socketSpy).toHaveBeenCalledWith(JSON.stringify(event));
   });
 
-  it("should send the a user message event to the Session", () => {
+  it("should send the a user message event to the Socket", () => {
     renderWithProviders(<ChatInterface />, {
       preloadedState: {
         agent: {
@@ -105,7 +104,7 @@ describe("ChatInterface", () => {
       action: ActionType.MESSAGE,
       args: { content: "my message" },
     };
-    expect(sessionSpy).toHaveBeenCalledWith(JSON.stringify(event));
+    expect(socketSpy).toHaveBeenCalledWith(JSON.stringify(event));
   });
 
   it("should disable the user input if agent is not initialized", () => {

frontend/src/components/chat/ChatInterface.tsx

@@ -5,41 +5,31 @@ import { RiArrowRightDoubleLine } from "react-icons/ri";
 import { useTranslation } from "react-i18next";
 import { twMerge } from "tailwind-merge";
 import { VscArrowDown } from "react-icons/vsc";
-import { FaRegThumbsDown, FaRegThumbsUp } from "react-icons/fa";
 import ChatInput from "./ChatInput";
 import Chat from "./Chat";
 import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
 import { sendChatMessage } from "#/services/chatService";
-import { addUserMessage, addAssistantMessage } from "#/state/chatSlice";
+import { addUserMessage } from "#/state/chatSlice";
 import { I18nKey } from "#/i18n/declaration";
 import { useScrollToBottom } from "#/hooks/useScrollToBottom";
-import Session from "#/services/session";
-import { getToken } from "#/services/auth";
-import toast from "#/utils/toast";
-import { FeedbackData, sendFeedback } from "#/api";
-import { removeApiKey } from "#/utils/utils";
 
 interface ScrollButtonProps {
   onClick: () => void;
   icon: JSX.Element;
   label: string;
-  // eslint-disable-next-line react/require-default-props
-  disabled?: boolean;
 }
 
 function ScrollButton({
   onClick,
   icon,
   label,
-  disabled = false,
 }: ScrollButtonProps): JSX.Element {
   return (
     <button
       type="button"
       className="relative border-1 text-xs rounded px-2 py-1 border-neutral-600 bg-neutral-700 cursor-pointer select-none"
       onClick={onClick}
-      disabled={disabled}
     >
       <div className="flex items-center">
         {icon} <span className="inline-block">{label}</span>
@@ -53,30 +43,6 @@ function ChatInterface() {
   const { messages } = useSelector((state: RootState) => state.chat);
   const { curAgentState } = useSelector((state: RootState) => state.agent);
 
-  const [feedbackShared, setFeedbackShared] = React.useState(false);
-  const [feedbackLoading, setFeedbackLoading] = React.useState(false);
-
-  const shareFeedback = async (feedback: "positive" | "negative") => {
-    const data: FeedbackData = {
-      email: "NOT_PROVIDED",
-      token: getToken(),
-      feedback,
-      trajectory: removeApiKey(Session._history),
-    };
-
-    try {
-      setFeedbackLoading(true);
-      await sendFeedback(data);
-      toast.info("Feedback shared successfully.");
-    } catch (e) {
-      console.error(e);
-      toast.error("share-error", "Failed to share, see console for details.");
-    } finally {
-      setFeedbackShared(true);
-      setFeedbackLoading(false);
-    }
-  };
-
   const handleSendMessage = (content: string) => {
     dispatch(addUserMessage(content));
     sendChatMessage(content);
@@ -92,12 +58,6 @@ function ChatInterface() {
   const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
     useScrollToBottom(scrollRef);
 
-  React.useEffect(() => {
-    if (curAgentState === AgentState.INIT && messages.length === 0) {
-      dispatch(addAssistantMessage(t(I18nKey.CHAT_INTERFACE$INITIAL_MESSAGE)));
-    }
-  }, [curAgentState]);
-
   return (
     <div className="flex flex-col h-full bg-neutral-800">
       <div className="flex items-center gap-2 border-b border-neutral-600 text-sm px-4 py-2">
@@ -138,23 +98,6 @@ function ChatInterface() {
               label: t(I18nKey.CHAT_INTERFACE$INPUT_CONTINUE_MESSAGE),
             })}
         </div>
-
-        {!feedbackShared && messages.length > 3 && (
-          <div className="flex justify-start gap-2 p-2">
-            <ScrollButton
-              disabled={feedbackLoading}
-              onClick={() => shareFeedback("positive")}
-              icon={<FaRegThumbsUp className="inline mr-2 w-3 h-3" />}
-              label=""
-            />
-            <ScrollButton
-              disabled={feedbackLoading}
-              onClick={() => shareFeedback("negative")}
-              icon={<FaRegThumbsDown className="inline mr-2 w-3 h-3" />}
-              label=""
-            />
-          </div>
-        )}
       </div>
 
       <ChatInput

frontend/src/components/chat/ChatMessage.tsx

@@ -11,7 +11,6 @@ function ChatMessage({ message }: MessageProps) {
   // const text = useTyping(message.content);
 
   const className = twMerge(
-    "markdown-body",
     "p-3 text-white max-w-[90%] overflow-y-auto rounded-lg",
     message.sender === "user" ? "bg-neutral-700 self-end" : "bg-neutral-500",
   );

frontend/src/components/file-explorer/ExplorerTree.test.tsx

@@ -1,8 +1,17 @@
 import React from "react";
-import { renderWithProviders } from "test-utils";
+import { render } from "@testing-library/react";
 import ExplorerTree from "./ExplorerTree";
+import { WorkspaceFile } from "#/services/fileService";
 
-const FILES = ["file-1-1.ts", "folder-1-2"];
+const NODE: WorkspaceFile = {
+  name: "root-folder-1",
+  children: [
+    { name: "file-1-1.ts" },
+    { name: "folder-1-2", children: [{ name: "file-1-2.ts" }] },
+  ],
+};
+
+const onFileClick = vi.fn();
 
 describe("ExplorerTree", () => {
   afterEach(() => {
@@ -10,21 +19,25 @@ describe("ExplorerTree", () => {
   });
 
   it("should render the explorer", () => {
-    const { getByText } = renderWithProviders(
-      <ExplorerTree files={FILES} defaultOpen />,
+    const { getByText, queryByText } = render(
+      <ExplorerTree root={NODE} onFileClick={onFileClick} defaultOpen />,
     );
 
+    expect(getByText("root-folder-1")).toBeInTheDocument();
     expect(getByText("file-1-1.ts")).toBeInTheDocument();
     expect(getByText("folder-1-2")).toBeInTheDocument();
-    // TODO: make sure children render
+    expect(queryByText("file-1-2.ts")).not.toBeInTheDocument();
   });
 
   it("should render the explorer given the defaultExpanded prop", () => {
-    const { queryByText } = renderWithProviders(<ExplorerTree files={FILES} />);
+    const { getByText, queryByText } = render(
+      <ExplorerTree root={NODE} onFileClick={onFileClick} />,
+    );
 
-    expect(queryByText("file-1-1.ts")).toBeInTheDocument();
-    expect(queryByText("folder-1-2")).toBeInTheDocument();
-    // TODO: make sure children don't render
+    expect(getByText("root-folder-1")).toBeInTheDocument();
+    expect(queryByText("file-1-1.ts")).not.toBeInTheDocument();
+    expect(queryByText("folder-1-2")).not.toBeInTheDocument();
+    expect(queryByText("file-1-2.ts")).not.toBeInTheDocument();
   });
 
   it.todo("should render all children as collapsed when defaultOpen is false");

frontend/src/components/file-explorer/ExplorerTree.tsx

@@ -1,17 +1,26 @@
 import React from "react";
 import TreeNode from "./TreeNode";
+import { WorkspaceFile } from "#/services/fileService";
 
 interface ExplorerTreeProps {
-  files: string[];
+  root: WorkspaceFile;
+  onFileClick: (path: string) => void;
   defaultOpen?: boolean;
 }
 
-function ExplorerTree({ files, defaultOpen = false }: ExplorerTreeProps) {
+function ExplorerTree({
+  root,
+  onFileClick,
+  defaultOpen = false,
+}: ExplorerTreeProps) {
   return (
     <div className="w-full overflow-x-auto h-full pt-[4px]">
-      {files.map((file) => (
-        <TreeNode key={file} path={file} defaultOpen={defaultOpen} />
-      ))}
+      <TreeNode
+        node={root}
+        path={root.name}
+        onFileClick={onFileClick}
+        defaultOpen={defaultOpen}
+      />
     </div>
   );
 }

frontend/src/components/file-explorer/FileExplorer.test.tsx

@@ -1,26 +1,22 @@
 import React from "react";
-import { waitFor, screen } from "@testing-library/react";
+import { render, waitFor, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { act } from "react-dom/test-utils";
-import { renderWithProviders } from "test-utils";
 import { describe, it, expect, vi, Mock } from "vitest";
 import FileExplorer from "./FileExplorer";
-import { uploadFiles, listFiles } from "#/services/fileService";
+import { getWorkspace, uploadFiles } from "#/services/fileService";
 import toast from "#/utils/toast";
-import AgentState from "#/types/AgentState";
 
-const toastSpy = vi.spyOn(toast, "error");
+const toastSpy = vi.spyOn(toast, "stickyError");
 
 vi.mock("../../services/fileService", async () => ({
-  listFiles: vi.fn(async (path: string = "/") => {
-    if (path === "/") {
-      return Promise.resolve(["folder1/", "file1.ts"]);
-    }
-    if (path === "/folder1/" || path === "folder1/") {
-      return Promise.resolve(["file2.ts"]);
-    }
-    return Promise.resolve([]);
-  }),
+  getWorkspace: vi.fn(async () => ({
+    name: "root",
+    children: [
+      { name: "file1.ts" },
+      { name: "folder1", children: [{ name: "file2.ts" }] },
+    ],
+  })),
 
   uploadFiles: vi.fn(),
 }));
@@ -31,60 +27,71 @@ describe("FileExplorer", () => {
   });
 
   it("should get the workspace directory", async () => {
-    const { getByText } = renderWithProviders(<FileExplorer />);
+    const { getByText } = render(<FileExplorer onFileClick={vi.fn} />);
 
+    expect(getWorkspace).toHaveBeenCalledTimes(1);
     await waitFor(() => {
-      expect(getByText("folder1")).toBeInTheDocument();
-      expect(getByText("file2.ts")).toBeInTheDocument();
+      expect(getByText("root")).toBeInTheDocument();
     });
-    expect(listFiles).toHaveBeenCalledTimes(2); // once for root, once for folder1
   });
 
   it.todo("should render an empty workspace");
 
-  it.only("should refetch the workspace when clicking the refresh button", async () => {
-    const { getByText } = renderWithProviders(<FileExplorer />, {
-      preloadedState: {
-        agent: {
-          curAgentState: AgentState.RUNNING,
-        },
-      },
-    });
-    await waitFor(() => {
-      expect(getByText("folder1")).toBeInTheDocument();
-      expect(getByText("file2.ts")).toBeInTheDocument();
-    });
-    expect(listFiles).toHaveBeenCalledTimes(2); // once for root, once for folder 1
-
-    // The 'await' keyword is required here to avoid a warning during test runs
-    await act(() => {
-      userEvent.click(screen.getByTestId("refresh"));
-    });
-
-    expect(listFiles).toHaveBeenCalledTimes(4); // 2 from initial render, 2 from refresh button
-  });
-
-  it("should toggle the explorer visibility when clicking the close button", async () => {
-    const { getByTestId, getByText, queryByText } = renderWithProviders(
-      <FileExplorer />,
+  it("calls the onFileClick function when a file is clicked", async () => {
+    const onFileClickMock = vi.fn();
+    const { getByText } = render(
+      <FileExplorer onFileClick={onFileClickMock} />,
     );
 
     await waitFor(() => {
       expect(getByText("folder1")).toBeInTheDocument();
     });
 
+    act(() => {
+      userEvent.click(getByText("folder1"));
+    });
+
+    act(() => {
+      userEvent.click(getByText("file2.ts"));
+    });
+
+    const absPath = "root/folder1/file2.ts";
+    expect(onFileClickMock).toHaveBeenCalledWith(absPath);
+  });
+
+  it("should refetch the workspace when clicking the refresh button", async () => {
+    const onFileClickMock = vi.fn();
+    render(<FileExplorer onFileClick={onFileClickMock} />);
+
+    // The 'await' keyword is required here to avoid a warning during test runs
+    await act(() => {
+      userEvent.click(screen.getByTestId("refresh"));
+    });
+
+    expect(getWorkspace).toHaveBeenCalledTimes(2); // 1 from initial render, 1 from refresh button
+  });
+
+  it("should toggle the explorer visibility when clicking the close button", async () => {
+    const { getByTestId, getByText, queryByText } = render(
+      <FileExplorer onFileClick={vi.fn} />,
+    );
+
+    await waitFor(() => {
+      expect(getByText("root")).toBeInTheDocument();
+    });
+
     act(() => {
       userEvent.click(getByTestId("toggle"));
     });
 
     // it should be hidden rather than removed from the DOM
-    expect(queryByText("folder1")).toBeInTheDocument();
-    expect(queryByText("folder1")).not.toBeVisible();
+    expect(queryByText("root")).toBeInTheDocument();
+    expect(queryByText("root")).not.toBeVisible();
   });
 
   it("should upload files", async () => {
     // TODO: Improve this test by passing expected argument to `uploadFiles`
-    const { getByTestId } = renderWithProviders(<FileExplorer />);
+    const { getByTestId } = render(<FileExplorer onFileClick={vi.fn} />);
     const file = new File([""], "file-name");
     const file2 = new File([""], "file-name-2");
 
@@ -96,7 +103,7 @@ describe("FileExplorer", () => {
     });
 
     expect(uploadFiles).toHaveBeenCalledOnce();
-    expect(listFiles).toHaveBeenCalled();
+    expect(getWorkspace).toHaveBeenCalled();
 
     const uploadDirInput = getByTestId("file-input");
 
@@ -106,7 +113,7 @@ describe("FileExplorer", () => {
     });
 
     expect(uploadFiles).toHaveBeenCalledTimes(2);
-    expect(listFiles).toHaveBeenCalled();
+    expect(getWorkspace).toHaveBeenCalled();
   });
 
   it.skip("should upload files when dragging them to the explorer", () => {
@@ -120,7 +127,7 @@ describe("FileExplorer", () => {
   it.todo("should display an error toast if file upload fails", async () => {
     (uploadFiles as Mock).mockRejectedValue(new Error());
 
-    const { getByTestId } = renderWithProviders(<FileExplorer />);
+    const { getByTestId } = render(<FileExplorer onFileClick={vi.fn} />);
 
     const uploadFileInput = getByTestId("file-input");
     const file = new File([""], "test");

frontend/src/components/file-explorer/FileExplorer.tsx

@@ -5,16 +5,17 @@ import {
   IoIosRefresh,
   IoIosCloudUpload,
 } from "react-icons/io";
-import { useDispatch, useSelector } from "react-redux";
 import { IoFileTray } from "react-icons/io5";
 import { twMerge } from "tailwind-merge";
-import AgentState from "#/types/AgentState";
-import { setRefreshID } from "#/state/codeSlice";
-import { listFiles, uploadFiles } from "#/services/fileService";
+import {
+  WorkspaceFile,
+  getWorkspace,
+  uploadFiles,
+} from "#/services/fileService";
 import IconButton from "../IconButton";
 import ExplorerTree from "./ExplorerTree";
+import { removeEmptyNodes } from "./utils";
 import toast from "#/utils/toast";
-import { RootState } from "#/store";
 
 interface ExplorerActionsProps {
   onRefresh: () => void;
@@ -85,45 +86,41 @@ function ExplorerActions({
   );
 }
 
-function FileExplorer() {
+interface FileExplorerProps {
+  onFileClick: (path: string) => void;
+}
+
+function FileExplorer({ onFileClick }: FileExplorerProps) {
+  const [workspace, setWorkspace] = React.useState<WorkspaceFile>();
   const [isHidden, setIsHidden] = React.useState(false);
   const [isDragging, setIsDragging] = React.useState(false);
-  const [files, setFiles] = React.useState<string[]>([]);
-  const { curAgentState } = useSelector((state: RootState) => state.agent);
+
   const fileInputRef = React.useRef<HTMLInputElement | null>(null);
-  const dispatch = useDispatch();
 
-  const selectFileInput = () => {
-    fileInputRef.current?.click(); // Trigger the file browser
+  const getWorkspaceData = async () => {
+    const wsFile = await getWorkspace();
+    setWorkspace(removeEmptyNodes(wsFile));
   };
 
-  const refreshWorkspace = async () => {
-    if (
-      curAgentState === AgentState.LOADING ||
-      curAgentState === AgentState.STOPPED
-    ) {
-      return;
-    }
-    dispatch(setRefreshID(Math.random()));
-    setFiles(await listFiles("/"));
+  const selectFileInput = async () => {
+    // Trigger the file browser
+    fileInputRef.current?.click();
   };
 
-  const uploadFileData = async (toAdd: FileList) => {
+  const uploadFileData = async (files: FileList) => {
     try {
-      await uploadFiles(toAdd);
-      await refreshWorkspace();
+      await uploadFiles(files);
+      await getWorkspaceData(); // Refresh the workspace to show the new file
     } catch (error) {
-      toast.error("ws", "Error uploading file");
+      toast.stickyError("ws", "Error uploading file");
     }
   };
 
   React.useEffect(() => {
     (async () => {
-      await refreshWorkspace();
+      await getWorkspaceData();
     })();
-  }, [curAgentState]);
 
-  React.useEffect(() => {
     const enableDragging = () => {
       setIsDragging(true);
     };
@@ -141,10 +138,6 @@ function FileExplorer() {
     };
   }, []);
 
-  if (!files.length) {
-    return null;
-  }
-
   return (
     <div className="relative">
       {isDragging && (
@@ -169,13 +162,19 @@ function FileExplorer() {
       >
         <div className="flex p-2 items-center justify-between relative">
           <div style={{ display: isHidden ? "none" : "block" }}>
-            <ExplorerTree files={files} defaultOpen />
+            {workspace && (
+              <ExplorerTree
+                root={workspace}
+                onFileClick={onFileClick}
+                defaultOpen
+              />
+            )}
           </div>
 
           <ExplorerActions
             isHidden={isHidden}
             toggleHidden={() => setIsHidden((prev) => !prev)}
-            onRefresh={refreshWorkspace}
+            onRefresh={getWorkspaceData}
             onUpload={selectFileInput}
           />
         </div>

frontend/src/components/file-explorer/TreeNode.test.tsx

@@ -1,130 +1,148 @@
 import React from "react";
-import { waitFor, act } from "@testing-library/react";
+import { act, render } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
-import { renderWithProviders } from "test-utils";
 import TreeNode from "./TreeNode";
-import { selectFile, listFiles } from "#/services/fileService";
+import { WorkspaceFile } from "#/services/fileService";
 
-vi.mock("../../services/fileService", async () => ({
-  listFiles: vi.fn(async (path: string = "/") => {
-    if (path === "/") {
-      return Promise.resolve(["folder1/", "file1.ts"]);
-    }
-    if (path === "/folder1/" || path === "folder1/") {
-      return Promise.resolve(["file2.ts"]);
-    }
-    return Promise.resolve([]);
-  }),
-  selectFile: vi.fn(async () => Promise.resolve({ code: "Hello world!" })),
-  uploadFile: vi.fn(),
-}));
+const onFileClick = vi.fn();
+
+const NODE: WorkspaceFile = {
+  name: "folder",
+  children: [
+    { name: "file.ts" },
+    { name: "folder2", children: [{ name: "file2.ts" }] },
+  ],
+};
 
 describe("TreeNode", () => {
   afterEach(() => {
-    vi.clearAllMocks();
+    vi.resetAllMocks();
   });
 
   it("should render a file if property has no children", () => {
-    const { getByText } = renderWithProviders(
-      <TreeNode path="/file.ts" defaultOpen />,
+    const { getByText } = render(
+      <TreeNode
+        node={NODE}
+        path={NODE.name}
+        onFileClick={onFileClick}
+        defaultOpen
+      />,
     );
 
     expect(getByText("file.ts")).toBeInTheDocument();
   });
 
-  it("should render a folder if it's in a subdir", async () => {
-    const { findByText } = renderWithProviders(
-      <TreeNode path="/folder1/" defaultOpen />,
+  it("should render a folder if property has children", () => {
+    const { getByText } = render(
+      <TreeNode
+        node={NODE}
+        path={NODE.name}
+        onFileClick={onFileClick}
+        defaultOpen
+      />,
     );
-    expect(listFiles).toHaveBeenCalledWith("/folder1/");
 
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await findByText("file2.ts")).toBeInTheDocument();
+    expect(getByText("folder")).toBeInTheDocument();
+    expect(getByText("file.ts")).toBeInTheDocument();
   });
 
-  it("should close a folder when clicking on it", async () => {
-    const { findByText, queryByText } = renderWithProviders(
-      <TreeNode path="/folder1/" defaultOpen />,
+  it("should close a folder when clicking on it", () => {
+    const { getByText, queryByText } = render(
+      <TreeNode
+        node={NODE}
+        path={NODE.name}
+        onFileClick={onFileClick}
+        defaultOpen
+      />,
     );
 
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await findByText("file2.ts")).toBeInTheDocument();
-
-    act(async () => {
-      userEvent.click(await findByText("folder1"));
-    });
-
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await queryByText("file2.ts")).not.toBeInTheDocument();
-  });
-
-  it("should open a folder when clicking on it", async () => {
-    const { getByText, findByText, queryByText } = renderWithProviders(
-      <TreeNode path="/folder1/" />,
-    );
-
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await queryByText("file2.ts")).not.toBeInTheDocument();
-
     act(() => {
-      userEvent.click(getByText("folder1"));
+      userEvent.click(getByText("folder"));
     });
-    expect(listFiles).toHaveBeenCalledWith("/folder1/");
 
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await findByText("file2.ts")).toBeInTheDocument();
+    expect(queryByText("folder2")).not.toBeInTheDocument();
+    expect(queryByText("file2.ts")).not.toBeInTheDocument();
+    expect(queryByText("file.ts")).not.toBeInTheDocument();
   });
 
-  it.only("should call a fn and return the full path of a file when clicking on it", () => {
-    const { getByText } = renderWithProviders(
-      <TreeNode path="/folder1/file2.ts" defaultOpen />,
+  it("should open a folder when clicking on it", () => {
+    const { getByText } = render(
+      <TreeNode node={NODE} path={NODE.name} onFileClick={onFileClick} />,
     );
 
+    act(() => {
+      userEvent.click(getByText("folder"));
+    });
+
+    expect(getByText("folder2")).toBeInTheDocument();
+    expect(getByText("file.ts")).toBeInTheDocument();
+  });
+
+  it("should call a fn and return the full path of a file when clicking on it", () => {
+    const { getByText } = render(
+      <TreeNode
+        node={NODE}
+        path={NODE.name}
+        onFileClick={onFileClick}
+        defaultOpen
+      />,
+    );
+
+    act(() => {
+      userEvent.click(getByText("file.ts"));
+    });
+
+    expect(onFileClick).toHaveBeenCalledWith("folder/file.ts");
+
+    act(() => {
+      userEvent.click(getByText("folder2"));
+    });
+
     act(() => {
       userEvent.click(getByText("file2.ts"));
     });
 
-    waitFor(() => {
-      expect(selectFile).toHaveBeenCalledWith("/folder1/file2.ts");
-    });
+    expect(onFileClick).toHaveBeenCalledWith("folder/folder2/file2.ts");
   });
 
-  it("should render the explorer given the defaultOpen prop", async () => {
-    const { getByText, findByText, queryByText } = renderWithProviders(
-      <TreeNode path="/" defaultOpen />,
+  it("should render the explorer given the defaultExpanded prop", () => {
+    const { getByText, queryByText } = render(
+      <TreeNode node={NODE} path={NODE.name} onFileClick={onFileClick} />,
     );
 
-    expect(listFiles).toHaveBeenCalledWith("/");
-
-    expect(await findByText("file1.ts")).toBeInTheDocument();
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await queryByText("file2.ts")).not.toBeInTheDocument();
+    expect(getByText("folder")).toBeInTheDocument();
+    expect(queryByText("folder2")).not.toBeInTheDocument();
+    expect(queryByText("file2.ts")).not.toBeInTheDocument();
+    expect(queryByText("file.ts")).not.toBeInTheDocument();
 
     act(() => {
-      userEvent.click(getByText("folder1"));
+      userEvent.click(getByText("folder"));
     });
 
-    expect(listFiles).toHaveBeenCalledWith("folder1/");
-
-    expect(await findByText("file1.ts")).toBeInTheDocument();
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await findByText("file2.ts")).toBeInTheDocument();
+    expect(getByText("folder2")).toBeInTheDocument();
+    expect(getByText("file.ts")).toBeInTheDocument();
   });
 
-  it("should render all children as collapsed when defaultOpen is false", async () => {
-    const { findByText, getByText, queryByText } = renderWithProviders(
-      <TreeNode path="/folder1/" />,
+  it("should render all children as collapsed when defaultOpen is false", () => {
+    const { getByText, queryByText } = render(
+      <TreeNode node={NODE} path={NODE.name} onFileClick={onFileClick} />,
     );
 
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await queryByText("file2.ts")).not.toBeInTheDocument();
+    expect(getByText("folder")).toBeInTheDocument();
+    expect(queryByText("folder2")).not.toBeInTheDocument();
+    expect(queryByText("file2.ts")).not.toBeInTheDocument();
+    expect(queryByText("file.ts")).not.toBeInTheDocument();
 
     act(() => {
-      userEvent.click(getByText("folder1"));
+      userEvent.click(getByText("folder"));
     });
-    expect(listFiles).toHaveBeenCalledWith("/folder1/");
 
-    expect(await findByText("folder1")).toBeInTheDocument();
-    expect(await findByText("file2.ts")).toBeInTheDocument();
+    expect(getByText("folder2")).toBeInTheDocument();
+    expect(getByText("file.ts")).toBeInTheDocument();
+    expect(queryByText("file2.ts")).not.toBeInTheDocument();
   });
+
+  it.todo(
+    "should maintain the expanded state of child folders when closing and opening a parent folder",
+  );
 });

frontend/src/components/file-explorer/TreeNode.tsx

@@ -1,11 +1,9 @@
 import React from "react";
-import { useDispatch, useSelector } from "react-redux";
 import { twMerge } from "tailwind-merge";
-import { RootState } from "#/store";
 import FolderIcon from "../FolderIcon";
 import FileIcon from "../FileIcons";
-import { listFiles, selectFile } from "#/services/fileService";
-import { setCode, setActiveFilepath } from "#/state/codeSlice";
+import { WorkspaceFile } from "#/services/fileService";
+import { CodeEditorContext } from "../CodeEditorContext";
 
 interface TitleProps {
   name: string;
@@ -28,65 +26,52 @@ function Title({ name, type, isOpen, onClick }: TitleProps) {
 }
 
 interface TreeNodeProps {
+  node: WorkspaceFile;
   path: string;
+  onFileClick: (path: string) => void;
   defaultOpen?: boolean;
 }
 
-function TreeNode({ path, defaultOpen = false }: TreeNodeProps) {
+function TreeNode({
+  node,
+  path,
+  onFileClick,
+  defaultOpen = false,
+}: TreeNodeProps) {
   const [isOpen, setIsOpen] = React.useState(defaultOpen);
-  const [children, setChildren] = React.useState<string[] | null>(null);
-  const refreshID = useSelector((state: RootState) => state.code.refreshID);
-  const activeFilepath = useSelector((state: RootState) => state.code.path);
+  const { selectedFileAbsolutePath } = React.useContext(CodeEditorContext);
 
-  const dispatch = useDispatch();
-
-  const fileParts = path.split("/");
-  const filename =
-    fileParts[fileParts.length - 1] || fileParts[fileParts.length - 2];
-
-  const isDirectory = path.endsWith("/");
-
-  const refreshChildren = async () => {
-    if (!isDirectory || !isOpen) {
-      setChildren(null);
-      return;
-    }
-    const files = await listFiles(path);
-    setChildren(files);
-  };
-
-  React.useEffect(() => {
-    refreshChildren();
-  }, [refreshID, isOpen]);
-
-  const handleClick = async () => {
-    if (isDirectory) {
+  const handleClick = React.useCallback(() => {
+    if (node.children) {
       setIsOpen((prev) => !prev);
     } else {
-      const newCode = await selectFile(path);
-      dispatch(setCode(newCode));
-      dispatch(setActiveFilepath(path));
+      onFileClick(path);
     }
-  };
+  }, [node, path, onFileClick]);
 
   return (
     <div
       className={twMerge(
         "text-sm text-neutral-400",
-        path === activeFilepath ? "bg-gray-700" : "",
+        path === selectedFileAbsolutePath ? "bg-gray-700" : "",
       )}
     >
       <Title
-        name={filename}
-        type={isDirectory ? "folder" : "file"}
+        name={node.name}
+        type={node.children ? "folder" : "file"}
         isOpen={isOpen}
         onClick={handleClick}
       />
 
-      {isOpen && children && (
+      {isOpen && node.children && (
         <div className="ml-5">
-          {children.map((child, index) => (
-            <TreeNode key={index} path={`${child}`} />
+          {node.children.map((child, index) => (
+            <TreeNode
+              key={index}
+              node={child}
+              path={`${path}/${child.name}`}
+              onFileClick={onFileClick}
+            />
           ))}
         </div>
       )}

frontend/src/components/file-explorer/utils.test.ts

@@ -0,0 +1,39 @@
+import { removeEmptyNodes } from "./utils";
+
+test("removeEmptyNodes removes empty arrays", () => {
+  const root = {
+    name: "a",
+    children: [
+      {
+        name: "b",
+        children: [],
+      },
+      {
+        name: "c",
+        children: [
+          {
+            name: "d",
+            children: [],
+          },
+        ],
+      },
+    ],
+  };
+
+  expect(removeEmptyNodes(root)).toEqual({
+    name: "a",
+    children: [
+      {
+        name: "b",
+      },
+      {
+        name: "c",
+        children: [
+          {
+            name: "d",
+          },
+        ],
+      },
+    ],
+  });
+});

frontend/src/components/file-explorer/utils.ts

@@ -0,0 +1,14 @@
+import { WorkspaceFile } from "#/services/fileService";
+
+export const removeEmptyNodes = (root: WorkspaceFile): WorkspaceFile => {
+  if (root.children) {
+    const children = root.children
+      .map(removeEmptyNodes)
+      .filter((node) => node !== undefined);
+    return {
+      ...root,
+      children: children.length ? children : undefined,
+    };
+  }
+  return root;
+};

frontend/src/components/markdown/code.tsx

@@ -21,12 +21,7 @@ export function code({
   }
 
   return (
-    <SyntaxHighlighter
-      className="rounded-lg"
-      style={vscDarkPlus}
-      language={match?.[1]}
-      PreTag="div"
-    >
+    <SyntaxHighlighter style={vscDarkPlus} language={match?.[1]} PreTag="div">
       {String(children).replace(/\n$/, "")}
     </SyntaxHighlighter>
   );

frontend/src/components/modals/load-previous-session/LoadPreviousSessionModal.test.tsx

@@ -2,23 +2,54 @@ import React from "react";
 import { act, render, screen, waitFor } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import LoadPreviousSessionModal from "./LoadPreviousSessionModal";
-import Session from "../../../services/session";
+import { clearMsgs, fetchMsgs } from "../../../services/session";
+import { addChatMessageFromEvent } from "../../../services/chatService";
+import { handleAssistantMessage } from "../../../services/actions";
+import toast from "../../../utils/toast";
 
 const RESUME_SESSION_BUTTON_LABEL_KEY =
   "LOAD_SESSION$RESUME_SESSION_MODAL_ACTION_LABEL";
 const START_NEW_SESSION_BUTTON_LABEL_KEY =
   "LOAD_SESSION$START_NEW_SESSION_MODAL_ACTION_LABEL";
 
+const mocks = vi.hoisted(() => ({
+  fetchMsgsMock: vi.fn(),
+}));
+
+vi.mock("../../../services/session", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../../../services/session")>()),
+  clearMsgs: vi.fn(),
+  fetchMsgs: mocks.fetchMsgsMock.mockResolvedValue({
+    messages: [
+      {
+        id: "1",
+        role: "user",
+        payload: { type: "action" },
+      },
+      {
+        id: "2",
+        role: "assistant",
+        payload: { type: "observation" },
+      },
+    ],
+  }),
+}));
+
+vi.mock("../../../services/chatService", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../../../services/chatService")>()),
+  addChatMessageFromEvent: vi.fn(),
+}));
+
 vi.mock("../../../services/actions", async (importOriginal) => ({
   ...(await importOriginal<typeof import("../../../services/actions")>()),
   handleAssistantMessage: vi.fn(),
 }));
 
-vi.spyOn(Session, "isConnected").mockImplementation(() => true);
-const restoreOrStartNewSessionSpy = vi.spyOn(
-  Session,
-  "restoreOrStartNewSession",
-);
+vi.mock("../../../utils/toast", () => ({
+  default: {
+    stickyError: vi.fn(),
+  },
+}));
 
 describe("LoadPreviousSession", () => {
   afterEach(() => {
@@ -44,6 +75,7 @@ describe("LoadPreviousSession", () => {
       userEvent.click(startNewSessionButton);
     });
 
+    expect(clearMsgs).toHaveBeenCalledTimes(1);
     // modal should close right after clearing messages
     expect(onOpenChangeMock).toHaveBeenCalledWith(false);
   });
@@ -61,9 +93,36 @@ describe("LoadPreviousSession", () => {
     });
 
     await waitFor(() => {
-      expect(restoreOrStartNewSessionSpy).toHaveBeenCalledTimes(1);
+      expect(fetchMsgs).toHaveBeenCalledTimes(1);
+      expect(addChatMessageFromEvent).toHaveBeenCalledTimes(1);
+      expect(handleAssistantMessage).toHaveBeenCalledTimes(1);
     });
     // modal should close right after fetching messages
     expect(onOpenChangeMock).toHaveBeenCalledWith(false);
   });
+
+  it("should show an error toast if there is an error fetching the session", async () => {
+    mocks.fetchMsgsMock.mockRejectedValue(new Error("Get messages failed."));
+
+    render(<LoadPreviousSessionModal isOpen onOpenChange={vi.fn} />);
+
+    const resumeSessionButton = screen.getByRole("button", {
+      name: RESUME_SESSION_BUTTON_LABEL_KEY,
+    });
+
+    act(() => {
+      userEvent.click(resumeSessionButton);
+    });
+
+    await waitFor(async () => {
+      await expect(() => fetchMsgs()).rejects.toThrow();
+      expect(handleAssistantMessage).not.toHaveBeenCalled();
+      expect(addChatMessageFromEvent).not.toHaveBeenCalled();
+      // error toast should be shown
+      expect(toast.stickyError).toHaveBeenCalledWith(
+        "ws",
+        "Error fetching the session",
+      );
+    });
+  });
 });

frontend/src/components/modals/load-previous-session/LoadPreviousSessionModal.tsx

@@ -1,8 +1,11 @@
 import React from "react";
 import { useTranslation } from "react-i18next";
 import { I18nKey } from "#/i18n/declaration";
+import { handleAssistantMessage } from "#/services/actions";
+import { addChatMessageFromEvent } from "#/services/chatService";
+import { clearMsgs, fetchMsgs } from "#/services/session";
+import toast from "#/utils/toast";
 import BaseModal from "../base-modal/BaseModal";
-import Session from "#/services/session";
 
 interface LoadPreviousSessionModalProps {
   isOpen: boolean;
@@ -15,6 +18,28 @@ function LoadPreviousSessionModal({
 }: LoadPreviousSessionModalProps) {
   const { t } = useTranslation();
 
+  const onStartNewSession = async () => {
+    await clearMsgs();
+  };
+
+  const onResumeSession = async () => {
+    try {
+      const { messages } = await fetchMsgs();
+
+      messages.forEach((message) => {
+        if (message.role === "user") {
+          addChatMessageFromEvent(message.payload);
+        }
+
+        if (message.role === "assistant") {
+          handleAssistantMessage(message.payload);
+        }
+      });
+    } catch (error) {
+      toast.stickyError("ws", "Error fetching the session");
+    }
+  };
+
   return (
     <BaseModal
       isOpen={isOpen}
@@ -25,13 +50,13 @@ function LoadPreviousSessionModal({
         {
           label: t(I18nKey.LOAD_SESSION$RESUME_SESSION_MODAL_ACTION_LABEL),
           className: "bg-primary rounded-lg",
-          action: Session.restoreOrStartNewSession,
+          action: onResumeSession,
           closeAfterAction: true,
         },
         {
           label: t(I18nKey.LOAD_SESSION$START_NEW_SESSION_MODAL_ACTION_LABEL),
           className: "bg-neutral-500 rounded-lg",
-          action: Session.startNewSession,
+          action: onStartNewSession,
           closeAfterAction: true,
         },
       ]}

frontend/src/components/modals/settings/SettingsModal.test.tsx

@@ -11,14 +11,12 @@ import {
   saveSettings,
   getDefaultSettings,
 } from "#/services/settings";
-import Session from "#/services/session";
-import { fetchAgents, fetchModels } from "#/services/options";
+import { initializeAgent } from "#/services/agent";
+import { fetchAgents, fetchModels } from "#/api";
 import SettingsModal from "./SettingsModal";
 
 const toastSpy = vi.spyOn(toast, "settingsChanged");
 const i18nSpy = vi.spyOn(i18next, "changeLanguage");
-const startNewSessionSpy = vi.spyOn(Session, "startNewSession");
-vi.spyOn(Session, "isConnected").mockImplementation(() => true);
 
 vi.mock("#/services/settings", async (importOriginal) => ({
   ...(await importOriginal<typeof import("#/services/settings")>()),
@@ -37,8 +35,12 @@ vi.mock("#/services/settings", async (importOriginal) => ({
   saveSettings: vi.fn(),
 }));
 
-vi.mock("#/services/options", async (importOriginal) => ({
-  ...(await importOriginal<typeof import("#/services/options")>()),
+vi.mock("#/services/agent", async () => ({
+  initializeAgent: vi.fn(),
+}));
+
+vi.mock("#/api", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("#/api")>()),
   fetchModels: vi
     .fn()
     .mockResolvedValue(Promise.resolve(["model1", "model2", "model3"])),
@@ -160,7 +162,7 @@ describe("SettingsModal", () => {
         userEvent.click(saveButton);
       });
 
-      expect(startNewSessionSpy).toHaveBeenCalled();
+      expect(initializeAgent).toHaveBeenCalled();
     });
 
     it("should display a toast for every change", async () => {

frontend/src/components/modals/settings/SettingsModal.tsx

@@ -3,10 +3,10 @@ import i18next from "i18next";
 import React, { useEffect } from "react";
 import { useTranslation } from "react-i18next";
 import { useSelector } from "react-redux";
-import { fetchAgents, fetchModels } from "#/services/options";
+import { fetchAgents, fetchModels } from "#/api";
 import { AvailableLanguages } from "#/i18n";
 import { I18nKey } from "#/i18n/declaration";
-import Session from "#/services/session";
+import { initializeAgent } from "#/services/agent";
 import { RootState } from "../../../store";
 import AgentState from "../../../types/AgentState";
 import {
@@ -100,7 +100,7 @@ function SettingsModal({ isOpen, onOpenChange }: SettingsProps) {
     const updatedSettings = getSettingsDifference(settings);
     saveSettings(settings);
     i18next.changeLanguage(settings.LANGUAGE);
-    Session.startNewSession();
+    initializeAgent(); // reinitialize the agent with the new settings
 
     const sensitiveKeys = ["LLM_API_KEY"];
 

frontend/src/i18n/translation.json

@@ -266,12 +266,12 @@
     "en": "Please stop the agent before editing these settings."
   },
   "LOAD_SESSION$MODAL_TITLE": {
-    "en": "Return to existing session?",
-    "zh-CN": "是否继续未完成的会话?",
-    "zh-TW": "是否繼續未完成的會話?"
+    "en": "Unfinished Session Detected",
+    "zh-CN": "检测到有未完成的会话",
+    "zh-TW": "偵測到未完成的會話"
   },
   "LOAD_SESSION$MODAL_CONTENT": {
-    "en": "You seem to have an ongoing session. Would you like to pick up where you left off, or start fresh?",
+    "en": "You seem to have an unfinished task. Would you like to pick up where you left off or start fresh?",
     "zh-CN": "您似乎有一个未完成的任务。您想继续之前的工作还是重新开始？",
     "zh-TW": "您似乎有一個未完成的任務。您想從上次離開的地方繼續還是重新開始？"
   },
@@ -285,9 +285,9 @@
     "zh-CN": "开始新会话",
     "zh-TW": "開始新會話"
   },
-  "CHAT_INTERFACE$INITIALIZING_AGENT_LOADING_MESSAGE": {
+  "CHAT_INTERFACE$INITIALZING_AGENT_LOADING_MESSAGE": {
     "en": "Initializing agent (may take up to 10 seconds)...",
-    "zh-CN": "正在初始化智能体（可能需要 10 秒以上时间）",
+    "zh-CN": "初始化智能体（可能需要 10 秒以上时间）",
     "de": "Agent wird initialisiert (kann bis zu 10 Sekunden dauern)...",
     "ko-KR": "에이전트 설치중(10초 정도 걸립니다)...",
     "no": "Initialiserer agent (det kan ta opptil 10 sekunder)...",
@@ -299,34 +299,6 @@
     "fr": "Initialisation de l'agent (peut prendre jusqu'à 10 secondes)...",
     "tr": "Ajan başlatılıyor (bu işlem 10 saniye kadar sürebilir)..."
   },
-  "CHAT_INTERFACE$AGENT_INIT_MESSAGE": {
-    "en": "Agent is initialized, waiting for task...",
-    "zh-CN": "智能体已初始化，等待任务中..."
-  },
-  "CHAT_INTERFACE$AGENT_RUNNING_MESSAGE": {
-    "en": "Agent is running task",
-    "zh-CN": "智能体正在执行任务..."
-  },
-  "CHAT_INTERFACE$AGENT_AWAITING_USER_INPUT_MESSAGE": {
-    "en": "Agent is awaiting user input...",
-    "zh-CN": "智能体正在等待用户输入..."
-  },
-  "CHAT_INTERFACE$AGENT_PAUSED_MESSAGE": {
-    "en": "Agent has paused.",
-    "zh-CN": "智能体已暂停"
-  },
-  "CHAT_INTERFACE$AGENT_STOPPED_MESSAGE": {
-    "en": "Agent has stopped.",
-    "zh-CN": "智能体已停止"
-  },
-  "CHAT_INTERFACE$AGENT_FINISHED_MESSAGE": {
-    "en": "Agent has finished the task.",
-    "zh-CN": "智能体已完成任务"
-  },
-  "CHAT_INTERFACE$AGENT_ERROR_MESSAGE": {
-    "en": "Agent encountered an error.",
-    "zh-CN": "智能体遇到错误"
-  },
   "CHAT_INTERFACE$INPUT_PLACEHOLDER": {
     "en": "Message assistant...",
     "zh-CN": "给助理发消息",

frontend/src/index.css

@@ -10,7 +10,6 @@
   --bg-editor-active: #31343D;
   --border-editor-sidebar: #3C3C4A;
   background-color: var(--neutral-900) !important;
-  --bg-neutral-muted: #afb8c133;
 }
 
 body {
@@ -26,17 +25,3 @@ code {
   font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
     monospace;
 }
-
-.markdown-body code {
-  padding: 0.2em 0.4em;
-  margin: 0;
-  font-size: 85%;
-  white-space: break-spaces;
-  background-color: var(--bg-neutral-muted);
-  border-radius: 6px;
-}
-
-.markdown-body pre code {
-  padding: 0;
-  background-color: inherit;
-}

frontend/src/index.tsx

@@ -4,19 +4,11 @@ import ReactDOM from "react-dom/client";
 import "./index.css";
 import { Provider } from "react-redux";
 import { NextUIProvider } from "@nextui-org/react";
-import { createBrowserRouter, RouterProvider } from "react-router-dom";
 import App from "./App";
 import reportWebVitals from "./reportWebVitals";
 import store from "#/store";
 import "#/i18n";
 
-const router = createBrowserRouter([
-  {
-    path: "/",
-    element: <App />,
-  },
-]);
-
 const root = ReactDOM.createRoot(
   document.getElementById("root") as HTMLElement,
 );
@@ -24,7 +16,7 @@ root.render(
   <React.StrictMode>
     <Provider store={store}>
       <NextUIProvider>
-        <RouterProvider router={router} />
+        <App />
       </NextUIProvider>
     </Provider>
   </React.StrictMode>,

frontend/src/services/actions.ts

@@ -1,6 +1,6 @@
 import { setScreenshotSrc, setUrl } from "#/state/browserSlice";
-import { addAssistantMessage, addUserMessage } from "#/state/chatSlice";
-import { setCode, setActiveFilepath } from "#/state/codeSlice";
+import { addAssistantMessage } from "#/state/chatSlice";
+import { setCode, updatePath } from "#/state/codeSlice";
 import { appendInput } from "#/state/commandSlice";
 import { appendJupyterInput } from "#/state/jupyterSlice";
 import { setRootTask } from "#/state/taskSlice";
@@ -16,27 +16,19 @@ const messageActions = {
     const { url, screenshotSrc } = message.args;
     store.dispatch(setUrl(url));
     store.dispatch(setScreenshotSrc(screenshotSrc));
-    store.dispatch(addAssistantMessage(message.message));
   },
   [ActionType.BROWSE_INTERACTIVE]: (message: ActionMessage) => {
-    if (message.args.thought) {
-      store.dispatch(addAssistantMessage(message.args.thought));
-    }
     const { url, screenshotSrc } = message.args;
     store.dispatch(setUrl(url));
     store.dispatch(setScreenshotSrc(screenshotSrc));
   },
   [ActionType.WRITE]: (message: ActionMessage) => {
     const { path, content } = message.args;
-    store.dispatch(setActiveFilepath(path));
+    store.dispatch(updatePath(path));
     store.dispatch(setCode(content));
   },
   [ActionType.MESSAGE]: (message: ActionMessage) => {
-    if (message.source === "user") {
-      store.dispatch(addUserMessage(message.args.content));
-    } else {
-      store.dispatch(addAssistantMessage(message.args.content));
-    }
+    store.dispatch(addAssistantMessage(message.args.content));
   },
   [ActionType.FINISH]: (message: ActionMessage) => {
     store.dispatch(addAssistantMessage(message.message));

frontend/src/services/agent.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it, vi } from "vitest";
+
+import ActionType from "#/types/ActionType";
+import { initializeAgent } from "./agent";
+import { Settings, saveSettings } from "./settings";
+import Socket from "./socket";
+
+const sendSpy = vi.spyOn(Socket, "send");
+
+describe("initializeAgent", () => {
+  it("Should initialize the agent with the current settings", () => {
+    const settings: Settings = {
+      LLM_MODEL: "llm_value",
+      AGENT: "agent_value",
+      LANGUAGE: "language_value",
+      LLM_API_KEY: "sk-...",
+    };
+
+    const event = {
+      action: ActionType.INIT,
+      args: settings,
+    };
+
+    saveSettings(settings);
+    initializeAgent();
+
+    expect(sendSpy).toHaveBeenCalledWith(JSON.stringify(event));
+  });
+});

frontend/src/services/agent.ts

@@ -0,0 +1,14 @@
+import ActionType from "#/types/ActionType";
+import { getSettings } from "./settings";
+import Socket from "./socket";
+
+/**
+ * Initialize the agent with the current settings.
+ * @param settings - The new settings.
+ */
+export const initializeAgent = () => {
+  const settings = getSettings();
+  const event = { action: ActionType.INIT, args: settings };
+  const eventString = JSON.stringify(event);
+  Socket.send(eventString);
+};