From fb1822123a598e01d73bdfa26b5f35de1ae744f3 Mon Sep 17 00:00:00 2001 From: geohotstan <135171913+geohotstan@users.noreply.github.com> Date: Sun, 24 Mar 2024 07:41:49 +0800 Subject: [PATCH] Adding pre-commit and CI for ruff and mypy (#69) * don't modify directories * oops typo * dev_config/python * add config to CI * bump CI python to 3.10 * 3.11? * del actions/ * add suggestions * delete unused code * missed some * oops missed another one * remove a file --- .github/workflows/lint.yml | 20 +++++++++++++++-- agenthub/__init__.py | 2 ++ agenthub/codeact_agent/__init__.py | 20 +++++++++++++---- agenthub/langchains_agent/__init__.py | 6 ++--- agenthub/langchains_agent/utils/llm.py | 2 +- agenthub/langchains_agent/utils/memory.py | 7 +----- dev_config/python/.pre-commit-config.yaml | 17 ++++++++++++++ dev_config/python/mypy.ini | 11 +++++++++ dev_config/python/ruff.toml | 3 +++ opendevin/agent.py | 5 ++--- opendevin/controller.py | 1 - opendevin/lib/actions/__init__.py | 1 + opendevin/lib/command_manager.py | 4 +--- opendevin/lib/event.py | 2 -- opendevin/main.py | 4 ++-- opendevin/sandbox/sandbox.py | 27 +++++++++-------------- server/server.py | 3 --- 17 files changed, 88 insertions(+), 47 deletions(-) create mode 100644 dev_config/python/.pre-commit-config.yaml create mode 100644 dev_config/python/mypy.ini create mode 100644 dev_config/python/ruff.toml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4d9f658e24..71dbd313ff 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,7 +3,7 @@ name: Lint on: [push, pull_request] jobs: - lint: + lint-frontend: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -15,4 +15,20 @@ jobs: npm ci --legacy-peer-deps - run: | cd frontend - npm run lint \ No newline at end of file + npm run lint + + lint-python: + name: Lint python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: 3.11 + - name: Install dependencies + run: pip install ruff mypy types-requests + - name: Run ruff + run: ruff check --config dev_config/python/ruff.toml opendevin/ server/ agenthub/ + - name: Run mypy + run: mypy --config-file dev_config/python/mypy.ini opendevin/ server/ agenthub/ diff --git a/agenthub/__init__.py b/agenthub/__init__.py index 781c578b91..4678f96538 100644 --- a/agenthub/__init__.py +++ b/agenthub/__init__.py @@ -1,2 +1,4 @@ from . import langchains_agent from . import codeact_agent + +__all__ = ['langchains_agent', 'codeact_agent'] diff --git a/agenthub/codeact_agent/__init__.py b/agenthub/codeact_agent/__init__.py index 47db8f79de..829627d94d 100644 --- a/agenthub/codeact_agent/__init__.py +++ b/agenthub/codeact_agent/__init__.py @@ -1,11 +1,12 @@ import os import re -import argparse from litellm import completion from termcolor import colored from typing import List, Dict from opendevin.agent import Agent, Message, Role +from opendevin.lib.event import Event +from opendevin.lib.command_manager import CommandManager from opendevin.sandbox.sandbox import DockerInteractive assert ( @@ -93,12 +94,13 @@ class CodeActAgent(Agent): command = re.search(r"(.*)", action, re.DOTALL) if command is not None: # a command was found - command = command.group(1) - if command.strip() == "exit": + command_group = command.group(1) + if command_group.strip() == "exit": print(colored("Exit received. Exiting...", "red")) break # execute the code - observation = self.env.execute(command) + # TODO: does exit_code get loaded into Message? + exit_code, observation = self.env.execute(command_group) self._history.append(Message(Role.ASSISTANT, observation)) print(colored("===ENV OBSERVATION:===\n" + observation, "blue")) else: @@ -120,5 +122,15 @@ class CodeActAgent(Agent): """ raise NotImplementedError + # TODO: implement these abstract methods + def add_event(self, event: Event) -> None: + raise NotImplementedError("Implement this abstract method") + + def step(self, cmd_mgr: CommandManager) -> Event: + raise NotImplementedError("Implement this abstract method") + + def search_memory(self, query: str) -> List[str]: + raise NotImplementedError("Implement this abstract method") + Agent.register("CodeActAgent", CodeActAgent) diff --git a/agenthub/langchains_agent/__init__.py b/agenthub/langchains_agent/__init__.py index a5a680f09a..4296f89ced 100644 --- a/agenthub/langchains_agent/__init__.py +++ b/agenthub/langchains_agent/__init__.py @@ -1,8 +1,6 @@ -import os -import argparse -from typing import List, Dict, Type +from typing import List -from opendevin.agent import Agent, Message +from opendevin.agent import Agent from agenthub.langchains_agent.utils.agent import Agent as LangchainsAgentImpl from opendevin.lib.event import Event diff --git a/agenthub/langchains_agent/utils/llm.py b/agenthub/langchains_agent/utils/llm.py index 8450dcc927..98475e3ced 100644 --- a/agenthub/langchains_agent/utils/llm.py +++ b/agenthub/langchains_agent/utils/llm.py @@ -7,7 +7,7 @@ if os.getenv("DEBUG"): set_debug(True) from typing import List -from langchain_core.pydantic_v1 import BaseModel, Field +from langchain_core.pydantic_v1 import BaseModel from langchain.chains import LLMChain from langchain.prompts import PromptTemplate diff --git a/agenthub/langchains_agent/utils/memory.py b/agenthub/langchains_agent/utils/memory.py index ed0f1713d7..f8ec812ada 100644 --- a/agenthub/langchains_agent/utils/memory.py +++ b/agenthub/langchains_agent/utils/memory.py @@ -1,14 +1,10 @@ -import os from . import json import chromadb from llama_index.core import Document from llama_index.core.retrievers import VectorIndexRetriever -from llama_index.core import VectorStoreIndex, StorageContext, load_index_from_storage -from llama_index.core.storage.docstore import SimpleDocumentStore -from llama_index.core.vector_stores import SimpleVectorStore - +from llama_index.core import VectorStoreIndex from llama_index.vector_stores.chroma import ChromaVectorStore class LongTermMemory: @@ -16,7 +12,6 @@ class LongTermMemory: db = chromadb.Client() self.collection = db.create_collection(name="memories") vector_store = ChromaVectorStore(chroma_collection=self.collection) - storage_context = StorageContext.from_defaults(vector_store=vector_store) self.index = VectorStoreIndex.from_vector_store(vector_store) self.thought_idx = 0 diff --git a/dev_config/python/.pre-commit-config.yaml b/dev_config/python/.pre-commit-config.yaml new file mode 100644 index 0000000000..a0a0646814 --- /dev/null +++ b/dev_config/python/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.3 + hooks: + - id: ruff + entry: ruff check --config dev_config/python/ruff.toml opendevin/ server/ agenthub/ + always_run: true + pass_filenames: false + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + additional_dependencies: [types-requests, types-setuptools] + entry: mypy --config-file dev_config/python/mypy.ini opendevin/ server/ agenthub/ + always_run: true + pass_filenames: false diff --git a/dev_config/python/mypy.ini b/dev_config/python/mypy.ini new file mode 100644 index 0000000000..cdbe39699a --- /dev/null +++ b/dev_config/python/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +warn_unused_configs = True +ignore_missing_imports = True +check_untyped_defs = True +explicit_package_bases = True +warn_unreachable = True +warn_redundant_casts = True +no_implicit_optional = True +strict_optional = True + +exclude = agenthub/langchains_agent/regression \ No newline at end of file diff --git a/dev_config/python/ruff.toml b/dev_config/python/ruff.toml new file mode 100644 index 0000000000..c457587c9b --- /dev/null +++ b/dev_config/python/ruff.toml @@ -0,0 +1,3 @@ +exclude = [ + "agenthub/langchains_agent/regression/", +] \ No newline at end of file diff --git a/opendevin/agent.py b/opendevin/agent.py index 21025ac0e1..5274c9ed1b 100644 --- a/opendevin/agent.py +++ b/opendevin/agent.py @@ -5,7 +5,6 @@ from enum import Enum from .lib.event import Event from .lib.command_manager import CommandManager -from .controller import AgentController class Role(Enum): SYSTEM = "system" # system message for LLM @@ -79,7 +78,7 @@ class Agent(ABC): return self._complete @property - def history(self) -> List[str]: + def history(self) -> List[Message]: """ Provides the history of interactions or state changes since the instruction was initiated. @@ -125,7 +124,7 @@ class Agent(ABC): to prepare the agent for restarting the instruction or cleaning up before destruction. """ - self.instruction = None + self.instruction = '' self._complete = False self._history = [] diff --git a/opendevin/controller.py b/opendevin/controller.py index e9ffb93b4a..69ad529a8d 100644 --- a/opendevin/controller.py +++ b/opendevin/controller.py @@ -27,7 +27,6 @@ class AgentController: return out_event def start_loop(self): - output = None for i in range(self.max_iterations): print("STEP", i, flush=True) log_events = self.command_manager.get_background_events() diff --git a/opendevin/lib/actions/__init__.py b/opendevin/lib/actions/__init__.py index 500c8b1069..34f5ce61a7 100644 --- a/opendevin/lib/actions/__init__.py +++ b/opendevin/lib/actions/__init__.py @@ -2,3 +2,4 @@ from .browse import browse from .write import write from .read import read +__all__ = ['run', 'kill', 'browse', 'write', 'read'] diff --git a/opendevin/lib/command_manager.py b/opendevin/lib/command_manager.py index c525d18848..b3273369f1 100644 --- a/opendevin/lib/command_manager.py +++ b/opendevin/lib/command_manager.py @@ -1,5 +1,3 @@ -import subprocess -import select from typing import List from opendevin.lib.event import Event @@ -41,7 +39,7 @@ class CommandManager: self.background_commands[bg_cmd.id] = bg_cmd return "Background command started. To stop it, send a `kill` action with id " + str(bg_cmd.id) - def kill_command(self, id: int) -> str: + def kill_command(self, id: int): # TODO: get log events before killing self.background_commands[id].shell.close() del self.background_commands[id] diff --git a/opendevin/lib/event.py b/opendevin/lib/event.py index d0d3cc56e9..6a07edb622 100644 --- a/opendevin/lib/event.py +++ b/opendevin/lib/event.py @@ -1,5 +1,3 @@ -import os -import json import opendevin.lib.actions as actions ACTION_TYPES = ['run', 'kill', 'browse', 'read', 'write', 'recall', 'think', 'summarize', 'output', 'error', 'finish'] diff --git a/opendevin/main.py b/opendevin/main.py index cda559719c..c841b3dab4 100644 --- a/opendevin/main.py +++ b/opendevin/main.py @@ -1,6 +1,6 @@ +from typing import Type import argparse -import agenthub # for the agent registry from opendevin.agent import Agent from opendevin.controller import AgentController @@ -14,7 +14,7 @@ if __name__ == "__main__": print(f"Running agent {args.agent_cls} (model: {args.model_name}, directory: {args.directory}) with task: \"{args.task}\"") - AgentCls: Agent = Agent.get_cls(args.agent_cls) + AgentCls: Type[Agent] = Agent.get_cls(args.agent_cls) agent = AgentCls( instruction=args.task, workspace_dir=args.directory, diff --git a/opendevin/sandbox/sandbox.py b/opendevin/sandbox/sandbox.py index 9fb7a9756f..97d5fdf86f 100644 --- a/opendevin/sandbox/sandbox.py +++ b/opendevin/sandbox/sandbox.py @@ -1,19 +1,15 @@ import os -import pty import sys import uuid import time -import shlex import select -import subprocess import docker -import time -from typing import List, Tuple +from typing import Tuple from collections import namedtuple import atexit -InputType = namedtuple("InputDtype", ["content"]) -OutputType = namedtuple("OutputDtype", ["content"]) +InputType = namedtuple("InputType", ["content"]) +OutputType = namedtuple("OutputType", ["content"]) CONTAINER_IMAGE = os.getenv("SANDBOX_CONTAINER_IMAGE", "opendevin/sandbox:latest") @@ -21,15 +17,15 @@ class DockerInteractive: def __init__( self, - workspace_dir: str = None, - container_image: str = None, + workspace_dir: str | None = None, + container_image: str | None = None, timeout: int = 120, - id: str = None + id: str | None = None ): if id is not None: - self.instance_id: str = id + self.instance_id = id else: - self.instance_id: str = uuid.uuid4() + self.instance_id = str(uuid.uuid4()) if workspace_dir is not None: assert os.path.exists(workspace_dir), f"Directory {workspace_dir} does not exist." # expand to absolute path @@ -52,7 +48,6 @@ class DockerInteractive: self.container_name = f"sandbox-{self.instance_id}" self.restart_docker_container() - uid = os.getuid() self.execute('useradd --shell /bin/bash -u {uid} -o -c \"\" -m devin && su devin') # regester container cleanup function atexit.register(self.cleanup) @@ -62,9 +57,9 @@ class DockerInteractive: return "" logs = "" while True: - ready_to_read, _, _ = select.select([self.log_generator], [], [], .1) + ready_to_read, _, _ = select.select([self.log_generator], [], [], .1) # type: ignore[has-type] if ready_to_read: - data = self.log_generator.read(4096) + data = self.log_generator.read(4096) # type: ignore[has-type] if not data: break # FIXME: we're occasionally seeing some escape characters like `\x02` and `\x00` in the logs... @@ -171,7 +166,7 @@ if __name__ == "__main__": print("\nExiting...") break if user_input.lower() == "exit": - print(f"Exiting...") + print("Exiting...") break exit_code, output = docker_interactive.execute(user_input) print("exit code:", exit_code) diff --git a/server/server.py b/server/server.py index d5b3086b34..b4b91a86ea 100644 --- a/server/server.py +++ b/server/server.py @@ -1,4 +1,3 @@ -import asyncio import json import os from time import sleep @@ -56,7 +55,6 @@ async def websocket_endpoint(websocket: WebSocket): await send_message_to_client(get_error_payload("Failed to start container: " + str(e))) continue - agent_listener = asyncio.create_task(listen_for_agent_messages()) if action == "terminal": msg = { "action": "terminal", @@ -67,7 +65,6 @@ async def websocket_endpoint(websocket: WebSocket): if agent_websocket is None: await send_message_to_client(get_error_payload("Agent not connected")) continue - await send_message_to_agent(data) except WebSocketDisconnect: print("Client websocket disconnected")