From f5c600a9f86412393f8ad37f1ad38786938f59e4 Mon Sep 17 00:00:00 2001 From: Slowly-Grokking <61430731+Slowly-Grokking@users.noreply.github.com> Date: Sat, 15 Apr 2023 13:59:42 -0500 Subject: [PATCH 01/51] relocate data_ingestion.py making this work without code change update readme --- README.md | 8 ++++---- autogpt/data_ingestion.py => data_ingestion.py | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename autogpt/data_ingestion.py => data_ingestion.py (100%) diff --git a/README.md b/README.md index cf370f13a9..194040a321 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that ## 🧠 Memory pre-seeding ```bash -# python scripts/data_ingestion.py -h +# python data_ingestion.py -h usage: data_ingestion.py [-h] (--file FILE | --dir DIR) [--init] [--overlap OVERLAP] [--max_length MAX_LENGTH] Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script. @@ -348,10 +348,10 @@ options: --overlap OVERLAP The overlap size between chunks when ingesting files (default: 200) --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000 -# python scripts/data_ingestion.py --dir seed_data --init --overlap 200 --max_length 1000 +# python data_ingestion.py --dir --init --overlap 200 --max_length 1000 ``` -This script located at `scripts/data_ingestion.py`, allows you to ingest files into memory and pre-seed it before running Auto-GPT. +This script located at `data_ingestion.py`, allows you to ingest files into memory and pre-seed it before running Auto-GPT. Memory pre-seeding is a technique that involves ingesting relevant documents or data into the AI's memory so that it can use this information to generate more informed and accurate responses. @@ -368,7 +368,7 @@ You could for example download the documentation of an API, a GitHub repository, Memories will be available to the AI immediately as they are ingested, even if ingested while Auto-GPT is running. -In the example above, the script initializes the memory, ingests all files within the `/seed_data` directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. +In the example above, the script initializes the memory, ingests all files within the `` directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. Note that you can also use the `--file` argument to ingest a single file into memory and that the script will only ingest files within the `/auto_gpt_workspace` directory. You can adjust the `max_length` and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: diff --git a/autogpt/data_ingestion.py b/data_ingestion.py similarity index 100% rename from autogpt/data_ingestion.py rename to data_ingestion.py From 92c0106e8167b4fe12fc7a0fa2ac911fedefde88 Mon Sep 17 00:00:00 2001 From: Slowly-Grokking <61430731+Slowly-Grokking@users.noreply.github.com> Date: Sat, 15 Apr 2023 15:33:47 -0500 Subject: [PATCH 02/51] Update README.md --- README.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 194040a321..ee399db90f 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,7 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that ## 🧠 Memory pre-seeding +Memory pre-seeding allows you to ingest files into memory and pre-seed it before running Auto-GPT. ```bash # python data_ingestion.py -h @@ -348,19 +349,15 @@ options: --overlap OVERLAP The overlap size between chunks when ingesting files (default: 200) --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000 -# python data_ingestion.py --dir --init --overlap 200 --max_length 1000 +# python data_ingestion.py --dir DataFolder --init --overlap 100 --max_length 2000 ``` +In the example above, the script initializes the memory, ingests all files within the `Auto-Gpt/autogpt/auto_gpt_workspace/DataFolder` directory into memory with an overlap between chunks of 100 and a maximum length of each chunk of 2000. -This script located at `data_ingestion.py`, allows you to ingest files into memory and pre-seed it before running Auto-GPT. +Note that you can also use the `--file` argument to ingest a single file into memory and that data_ingestion.py will only ingest files within the `/auto_gpt_workspace` directory. -Memory pre-seeding is a technique that involves ingesting relevant documents or data into the AI's memory so that it can use this information to generate more informed and accurate responses. +The DIR path is relative to the auto_gpt_workspace directory, so `python data_ingestion.py --dir . --init` will ingest everything in `auto_gpt_workspace` directory. -To pre-seed the memory, the content of each document is split into chunks of a specified maximum length with a specified overlap between chunks, and then each chunk is added to the memory backend set in the .env file. When the AI is prompted to recall information, it can then access those pre-seeded memories to generate more informed and accurate responses. - -This technique is particularly useful when working with large amounts of data or when there is specific information that the AI needs to be able to access quickly. -By pre-seeding the memory, the AI can retrieve and use this information more efficiently, saving time, API call and improving the accuracy of its responses. - -You could for example download the documentation of an API, a GitHub repository, etc. and ingest it into memory before running Auto-GPT. +Memory pre-seeding is a technique for improving AI accuracy by ingesting relevant data into its memory. Chunks of data are split and added to memory, allowing the AI to access them quickly and generate more accurate responses. It's useful for large datasets or when specific information needs to be accessed quickly. Examples include ingesting API or GitHub documentation before running Auto-GPT. ⚠️ If you use Redis as your memory, make sure to run Auto-GPT with the `WIPE_REDIS_ON_START` set to `False` in your `.env` file. @@ -368,9 +365,6 @@ You could for example download the documentation of an API, a GitHub repository, Memories will be available to the AI immediately as they are ingested, even if ingested while Auto-GPT is running. -In the example above, the script initializes the memory, ingests all files within the `` directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. -Note that you can also use the `--file` argument to ingest a single file into memory and that the script will only ingest files within the `/auto_gpt_workspace` directory. - You can adjust the `max_length` and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: - Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. From 08eb2566e41a9b1619b98b517c2dfb217e1f75d1 Mon Sep 17 00:00:00 2001 From: lonrun Date: Sun, 16 Apr 2023 07:37:50 +0800 Subject: [PATCH 03/51] Add run scripts for shell --- run.sh | 9 +++++++++ run_continuous.sh | 3 +++ 2 files changed, 12 insertions(+) create mode 100755 run.sh create mode 100755 run_continuous.sh diff --git a/run.sh b/run.sh new file mode 100755 index 0000000000..edcbc44155 --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +python scripts/check_requirements.py requirements.txt +if [ $? -eq 1 ] +then + echo Installing missing packages... + pip install -r requirements.txt +fi +python -m autogpt $@ +read -p "Press any key to continue..." diff --git a/run_continuous.sh b/run_continuous.sh new file mode 100755 index 0000000000..14c9cfd2ab --- /dev/null +++ b/run_continuous.sh @@ -0,0 +1,3 @@ +#!/bin/bash +argument="--continuous" +./run.sh "$argument" From 66ee7e1a81ec9c2b35808bcd5b3898ff20dcc290 Mon Sep 17 00:00:00 2001 From: Slowly-Grokking <61430731+Slowly-Grokking@users.noreply.github.com> Date: Sat, 15 Apr 2023 21:33:26 -0500 Subject: [PATCH 04/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37b81b854c..047606258a 100644 --- a/README.md +++ b/README.md @@ -360,7 +360,7 @@ options: --dir DIR The directory containing the files to ingest. --init Init the memory and wipe its content (default: False) --overlap OVERLAP The overlap size between chunks when ingesting files (default: 200) - --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000 + --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000) # python data_ingestion.py --dir DataFolder --init --overlap 100 --max_length 2000 ``` From 93895090172e05b72e17b2ce02fbb9b57f99b6c3 Mon Sep 17 00:00:00 2001 From: Slowly-Grokking <61430731+Slowly-Grokking@users.noreply.github.com> Date: Sun, 16 Apr 2023 02:01:42 -0500 Subject: [PATCH 05/51] Update README.md --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 99a455bc2f..0476b23300 100644 --- a/README.md +++ b/README.md @@ -378,6 +378,11 @@ Note that you can also use the `--file` argument to ingest a single file into me The DIR path is relative to the auto_gpt_workspace directory, so `python data_ingestion.py --dir . --init` will ingest everything in `auto_gpt_workspace` directory. +You can adjust the `max_length` and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: +- Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. +- Reducing the `max_length` value will create more chunks, which can save prompt tokens by allowing for more message history in the context, but will also increase the number of chunks. +- Increasing the `max_length` value will provide the AI with more contextual information from each chunk, reducing the number of chunks created and saving on OpenAI API requests. However, this may also use more prompt tokens and decrease the overall context available to the AI. + Memory pre-seeding is a technique for improving AI accuracy by ingesting relevant data into its memory. Chunks of data are split and added to memory, allowing the AI to access them quickly and generate more accurate responses. It's useful for large datasets or when specific information needs to be accessed quickly. Examples include ingesting API or GitHub documentation before running Auto-GPT. ⚠️ If you use Redis as your memory, make sure to run Auto-GPT with the `WIPE_REDIS_ON_START` set to `False` in your `.env` file. @@ -386,12 +391,6 @@ Memory pre-seeding is a technique for improving AI accuracy by ingesting relevan Memories will be available to the AI immediately as they are ingested, even if ingested while Auto-GPT is running. -You can adjust the `max_length` and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: - -- Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. -- Reducing the `max_length` value will create more chunks, which can save prompt tokens by allowing for more message history in the context, but will also increase the number of chunks. -- Increasing the `max_length` value will provide the AI with more contextual information from each chunk, reducing the number of chunks created and saving on OpenAI API requests. However, this may also use more prompt tokens and decrease the overall context available to the AI. - ## 💀 Continuous Mode ⚠️ Run the AI **without** user authorization, 100% automated. From 9c8d95d4db16992a504c8dc17be44fc1db0bd672 Mon Sep 17 00:00:00 2001 From: Gabe <66077254+MrBrain295@users.noreply.github.com> Date: Sun, 16 Apr 2023 11:05:00 -0500 Subject: [PATCH 06/51] Fix README.md New owner. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fcff589fc9..f9112a67f4 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ Our workflow has been improved, but please note that `master` branch may often be in a **broken** state. Please download the latest `stable` release from here: https://github.com/Torantulino/Auto-GPT/releases/latest. -![GitHub Repo stars](https://img.shields.io/github/stars/Torantulino/auto-gpt?style=social) +![GitHub Repo stars](https://img.shields.io/github/stars/Significant-Gravitas/auto-gpt?style=social) [![Twitter Follow](https://img.shields.io/twitter/follow/siggravitas?style=social)](https://twitter.com/SigGravitas) [![Discord Follow](https://dcbadge.vercel.app/api/server/autogpt?style=flat)](https://discord.gg/autogpt) -[![Unit Tests](https://github.com/Torantulino/Auto-GPT/actions/workflows/ci.yml/badge.svg)](https://github.com/Torantulino/Auto-GPT/actions/workflows/ci.yml) +[![Unit Tests](https://github.com/Significant-Gravitaso/Auto-GPT/actions/workflows/ci.yml/badge.svg)](https://github.com/Significant-Gravitas/Auto-GPT/actions/workflows/ci.yml) Auto-GPT is an experimental open-source application showcasing the capabilities of the GPT-4 language model. This program, driven by GPT-4, chains together LLM "thoughts", to autonomously achieve whatever goal you set. As one of the first examples of GPT-4 running fully autonomously, Auto-GPT pushes the boundaries of what is possible with AI. @@ -21,7 +21,7 @@ https://user-images.githubusercontent.com/22963551/228855501-2f5777cf-755b-4407-

If you can spare a coffee, you can help to cover the costs of developing Auto-GPT and help push the boundaries of fully autonomous AI! Your support is greatly appreciated -Development of this free, open-source project is made possible by all the contributors and sponsors. If you'd like to sponsor this project and have your avatar or company logo appear below click here. +Development of this free, open-source project is made possible by all the contributors and sponsors. If you'd like to sponsor this project and have your avatar or company logo appear below click here.

@@ -106,7 +106,7 @@ _To execute the following commands, open a CMD, Bash, or Powershell window by na 2. Clone the repository: For this step, you need Git installed. Alternatively, you can download the zip file by clicking the button at the top of this page ☝️ ```bash -git clone https://github.com/Torantulino/Auto-GPT.git +git clone https://github.com/Significant-Gravitas/Auto-GPT.git ``` 3. Navigate to the directory where the repository was downloaded From 005479f8c33f71cf36cfd3033339ecd24a62bc6d Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:41:45 -0700 Subject: [PATCH 07/51] Add benchmark GitHub action workflow --- .github/workflows/benchmark.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..c5a42b2c0c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,31 @@ +name: benchmark + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + environment: benchmark + strategy: + matrix: + python-version: [3.8] + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: benchmark + run: | + python benchmark/benchmark_entrepeneur_gpt_with_undecisive_user.py + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From d934d226ce56e34c09fd0ff491a15cc3a8bc8e0a Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:41:49 -0700 Subject: [PATCH 08/51] Update .gitignore to properly handle virtual environments --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3209297cb6..eda7f32734 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ auto_gpt_workspace/* *.mpeg .env azure.yaml -*venv/* outputs/* ai_settings.yaml last_run_ai_settings.yaml @@ -130,10 +129,9 @@ celerybeat.pid .env .venv env/ -venv/ +venv*/ ENV/ env.bak/ -venv.bak/ # Spyder project settings .spyderproject From bf24cd9508316031b2f914359460363d2fb75c04 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:41:52 -0700 Subject: [PATCH 09/51] Refactor agent.py to improve JSON handling and validation --- autogpt/agent/agent.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 301d3f023e..32d982e52a 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -3,9 +3,8 @@ from autogpt.app import execute_command, get_command from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config -from autogpt.json_fixes.bracket_termination import ( - attempt_to_fix_json_by_finding_outermost_brackets, -) +from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques +from autogpt.json_validation.validate_json import validate_json from autogpt.logs import logger, print_assistant_thoughts from autogpt.speech import say_text from autogpt.spinner import Spinner @@ -70,18 +69,20 @@ class Agent: cfg.fast_token_limit, ) # TODO: This hardcodes the model to use GPT3.5. Make this an argument - # Print Assistant thoughts - print_assistant_thoughts(self.ai_name, assistant_reply) + assistant_reply_json = fix_json_using_multiple_techniques(assistant_reply) - # Get command name and arguments - try: - command_name, arguments = get_command( - attempt_to_fix_json_by_finding_outermost_brackets(assistant_reply) - ) - if cfg.speak_mode: - say_text(f"I want to execute {command_name}") - except Exception as e: - logger.error("Error: \n", str(e)) + # Print Assistant thoughts + if assistant_reply_json != {}: + validate_json(assistant_reply_json, 'llm_response_format_1') + # Get command name and arguments + try: + print_assistant_thoughts(self.ai_name, assistant_reply_json) + command_name, arguments = get_command(assistant_reply_json) + # command_name, arguments = assistant_reply_json_valid["command"]["name"], assistant_reply_json_valid["command"]["args"] + if cfg.speak_mode: + say_text(f"I want to execute {command_name}") + except Exception as e: + logger.error("Error: \n", str(e)) if not cfg.continuous_mode and self.next_action_count == 0: ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### From 70100af98e07a1ad78eb40b503743033344dd6a1 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:41:57 -0700 Subject: [PATCH 10/51] Refactor get_command function in app.py to accept JSON directly --- autogpt/app.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/autogpt/app.py b/autogpt/app.py index 6ead0d52e8..78b5bd2fde 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -1,6 +1,6 @@ """ Command and Control """ import json -from typing import List, NoReturn, Union +from typing import List, NoReturn, Union, Dict from autogpt.agent.agent_manager import AgentManager from autogpt.commands.evaluate_code import evaluate_code from autogpt.commands.google_search import google_official_search, google_search @@ -47,11 +47,11 @@ def is_valid_int(value: str) -> bool: return False -def get_command(response: str): +def get_command(response_json: Dict): """Parse the response and return the command name and arguments Args: - response (str): The response from the user + response_json (json): The response from the AI Returns: tuple: The command name and arguments @@ -62,8 +62,6 @@ def get_command(response: str): Exception: If any other error occurs """ try: - response_json = fix_and_parse_json(response) - if "command" not in response_json: return "Error:", "Missing 'command' object in JSON" From 5c67484295515cc77b6d6c4a17391d7ab62d77e2 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:42:00 -0700 Subject: [PATCH 11/51] Remove deprecated function from bracket_termination.py --- autogpt/json_fixes/bracket_termination.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py index 822eed4a54..731efeb1c0 100644 --- a/autogpt/json_fixes/bracket_termination.py +++ b/autogpt/json_fixes/bracket_termination.py @@ -3,16 +3,20 @@ from __future__ import annotations import contextlib import json +<<<<<<< HEAD import regex from colorama import Fore from autogpt.logs import logger +======= +from typing import Optional +>>>>>>> 67f32105 (Remove deprecated function from bracket_termination.py) from autogpt.config import Config -from autogpt.speech import say_text CFG = Config() +<<<<<<< HEAD def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str): if CFG.speak_mode and CFG.debug_mode: say_text( @@ -48,6 +52,9 @@ def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str): def balance_braces(json_string: str) -> str | None: +======= +def balance_braces(json_string: str) -> Optional[str]: +>>>>>>> 67f32105 (Remove deprecated function from bracket_termination.py) """ Balance the braces in a JSON string. From fec25cd6903a83f07c8559c26cc4a8b0515ff608 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:42:05 -0700 Subject: [PATCH 12/51] Add master_json_fix_method module for unified JSON handling --- autogpt/json_fixes/master_json_fix_method.py | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 autogpt/json_fixes/master_json_fix_method.py diff --git a/autogpt/json_fixes/master_json_fix_method.py b/autogpt/json_fixes/master_json_fix_method.py new file mode 100644 index 0000000000..7a2cf3cc81 --- /dev/null +++ b/autogpt/json_fixes/master_json_fix_method.py @@ -0,0 +1,28 @@ +from typing import Any, Dict + +from autogpt.config import Config +from autogpt.logs import logger +from autogpt.speech import say_text +CFG = Config() + + +def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: + from autogpt.json_fixes.parsing import attempt_to_fix_json_by_finding_outermost_brackets + + from autogpt.json_fixes.parsing import fix_and_parse_json + + # Parse and print Assistant response + assistant_reply_json = fix_and_parse_json(assistant_reply) + if assistant_reply_json == {}: + assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets( + assistant_reply + ) + + if assistant_reply_json != {}: + return assistant_reply_json + + logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) + if CFG.speak_mode: + say_text("I have received an invalid JSON response from the OpenAI API.") + + return {} From cfbec56b2bb4c1bcacd600f27fb9c6aa400f434c Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:42:07 -0700 Subject: [PATCH 13/51] Refactor parsing module and move JSON fix function to appropriate location --- autogpt/json_fixes/parsing.py | 67 ++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/autogpt/json_fixes/parsing.py b/autogpt/json_fixes/parsing.py index 0f15441160..d3a51f438e 100644 --- a/autogpt/json_fixes/parsing.py +++ b/autogpt/json_fixes/parsing.py @@ -3,18 +3,24 @@ from __future__ import annotations import contextlib import json +<<<<<<< HEAD from typing import Any +======= +from typing import Any, Dict, Union +from colorama import Fore +from regex import regex +>>>>>>> d3d8253b (Refactor parsing module and move JSON fix function to appropriate location) from autogpt.config import Config from autogpt.json_fixes.auto_fix import fix_json from autogpt.json_fixes.bracket_termination import balance_braces from autogpt.json_fixes.escaping import fix_invalid_escape from autogpt.json_fixes.missing_quotes import add_quotes_to_property_names from autogpt.logs import logger +from autogpt.speech import say_text CFG = Config() - JSON_SCHEMA = """ { "command": { @@ -38,7 +44,6 @@ JSON_SCHEMA = """ def correct_json(json_to_load: str) -> str: """ Correct common JSON errors. - Args: json_to_load (str): The JSON string. """ @@ -72,7 +77,7 @@ def correct_json(json_to_load: str) -> str: def fix_and_parse_json( json_to_load: str, try_to_fix_with_gpt: bool = True -) -> str | dict[Any, Any]: +) -> Dict[Any, Any]: """Fix and parse JSON string Args: @@ -110,7 +115,11 @@ def fix_and_parse_json( def try_ai_fix( try_to_fix_with_gpt: bool, exception: Exception, json_to_load: str +<<<<<<< HEAD ) -> str | dict[Any, Any]: +======= +) -> Dict[Any, Any]: +>>>>>>> d3d8253b (Refactor parsing module and move JSON fix function to appropriate location) """Try to fix the JSON with the AI Args: @@ -126,13 +135,13 @@ def try_ai_fix( """ if not try_to_fix_with_gpt: raise exception - - logger.warn( - "Warning: Failed to parse AI output, attempting to fix." - "\n If you see this warning frequently, it's likely that" - " your prompt is confusing the AI. Try changing it up" - " slightly." - ) + if CFG.debug_mode: + logger.warn( + "Warning: Failed to parse AI output, attempting to fix." + "\n If you see this warning frequently, it's likely that" + " your prompt is confusing the AI. Try changing it up" + " slightly." + ) # Now try to fix this up using the ai_functions ai_fixed_json = fix_json(json_to_load, JSON_SCHEMA) @@ -140,5 +149,39 @@ def try_ai_fix( return json.loads(ai_fixed_json) # This allows the AI to react to the error message, # which usually results in it correcting its ways. - logger.error("Failed to fix AI output, telling the AI.") - return json_to_load + # logger.error("Failed to fix AI output, telling the AI.") + return {} + + +def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str): + if CFG.speak_mode and CFG.debug_mode: + say_text( + "I have received an invalid JSON response from the OpenAI API. " + "Trying to fix it now." + ) + logger.error("Attempting to fix JSON by finding outermost brackets\n") + + try: + json_pattern = regex.compile(r"\{(?:[^{}]|(?R))*\}") + json_match = json_pattern.search(json_string) + + if json_match: + # Extract the valid JSON object from the string + json_string = json_match.group(0) + logger.typewriter_log( + title="Apparently json was fixed.", title_color=Fore.GREEN + ) + if CFG.speak_mode and CFG.debug_mode: + say_text("Apparently json was fixed.") + else: + return {} + + except (json.JSONDecodeError, ValueError): + if CFG.debug_mode: + logger.error(f"Error: Invalid JSON: {json_string}\n") + if CFG.speak_mode: + say_text("Didn't work. I will have to ignore this response then.") + logger.error("Error: Invalid JSON, setting it to empty JSON now.\n") + json_string = {} + + return fix_and_parse_json(json_string) From af50d6cfb5577bc402e2d920fed062ddbb9c205f Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:43:26 -0700 Subject: [PATCH 14/51] Add JSON schema for LLM response format version 1 --- .../json_schemas/llm_response_format_1.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 autogpt/json_schemas/llm_response_format_1.json diff --git a/autogpt/json_schemas/llm_response_format_1.json b/autogpt/json_schemas/llm_response_format_1.json new file mode 100644 index 0000000000..9aa3335251 --- /dev/null +++ b/autogpt/json_schemas/llm_response_format_1.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "thoughts": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "reasoning": {"type": "string"}, + "plan": {"type": "string"}, + "criticism": {"type": "string"}, + "speak": {"type": "string"} + }, + "required": ["text", "reasoning", "plan", "criticism", "speak"], + "additionalProperties": false + }, + "command": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "args": { + "type": "object" + } + }, + "required": ["name", "args"], + "additionalProperties": false + } + }, + "required": ["thoughts", "command"], + "additionalProperties": false +} From 63d2a1085c2d65e06050c1ed7c0a889c2ce9c531 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:43:33 -0700 Subject: [PATCH 15/51] Add JSON validation utility function --- autogpt/json_validation/validate_json.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 autogpt/json_validation/validate_json.py diff --git a/autogpt/json_validation/validate_json.py b/autogpt/json_validation/validate_json.py new file mode 100644 index 0000000000..127fcc17f4 --- /dev/null +++ b/autogpt/json_validation/validate_json.py @@ -0,0 +1,30 @@ +import json +from jsonschema import Draft7Validator +from autogpt.config import Config +from autogpt.logs import logger + +CFG = Config() + + +def validate_json(json_object: object, schema_name: object) -> object: + """ + :type schema_name: object + :param schema_name: + :type json_object: object + """ + with open(f"autogpt/json_schemas/{schema_name}.json", "r") as f: + schema = json.load(f) + validator = Draft7Validator(schema) + + if errors := sorted(validator.iter_errors(json_object), key=lambda e: e.path): + logger.error("The JSON object is invalid.") + if CFG.debug_mode: + logger.error(json.dumps(json_object, indent=4)) # Replace 'json_object' with the variable containing the JSON data + logger.error("The following issues were found:") + + for error in errors: + logger.error(f"Error: {error.message}") + else: + print("The JSON object is valid.") + + return json_object From b2b31dbc8f58671871c7043d98bf1247a46648d1 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:43:40 -0700 Subject: [PATCH 16/51] Update logs.py with new print_assistant_thoughts function --- autogpt/logs.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/autogpt/logs.py b/autogpt/logs.py index 22ce23f4aa..5365302366 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -288,3 +288,42 @@ def print_assistant_thoughts(ai_name, assistant_reply): except Exception: call_stack = traceback.format_exc() logger.error("Error: \n", call_stack) + +def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object) -> None: + assistant_thoughts_reasoning = None + assistant_thoughts_plan = None + assistant_thoughts_speak = None + assistant_thoughts_criticism = None + + assistant_thoughts = assistant_reply_json_valid.get("thoughts", {}) + assistant_thoughts_text = assistant_thoughts.get("text") + if assistant_thoughts: + assistant_thoughts_reasoning = assistant_thoughts.get("reasoning") + assistant_thoughts_plan = assistant_thoughts.get("plan") + assistant_thoughts_criticism = assistant_thoughts.get("criticism") + assistant_thoughts_speak = assistant_thoughts.get("speak") + logger.typewriter_log( + f"{ai_name.upper()} THOUGHTS:", Fore.YELLOW, f"{assistant_thoughts_text}" + ) + logger.typewriter_log( + "REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}" + ) + if assistant_thoughts_plan: + logger.typewriter_log("PLAN:", Fore.YELLOW, "") + # If it's a list, join it into a string + if isinstance(assistant_thoughts_plan, list): + assistant_thoughts_plan = "\n".join(assistant_thoughts_plan) + elif isinstance(assistant_thoughts_plan, dict): + assistant_thoughts_plan = str(assistant_thoughts_plan) + + # Split the input_string using the newline character and dashes + lines = assistant_thoughts_plan.split("\n") + for line in lines: + line = line.lstrip("- ") + logger.typewriter_log("- ", Fore.GREEN, line.strip()) + logger.typewriter_log( + "CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}" + ) + # Speak the assistant's thoughts + if CFG.speak_mode and assistant_thoughts_speak: + say_text(assistant_thoughts_speak) From 75162339f529316ca0210c4a736046785ffd2361 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:43:46 -0700 Subject: [PATCH 17/51] Add empty __init__.py to benchmark directory --- benchmark/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 benchmark/__init__.py diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From dca10ab87682d73a867b04409e4bec521293d0ec Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:43:54 -0700 Subject: [PATCH 18/51] Add benchmark test for Entrepreneur-GPT with difficult user --- ...ark_entrepeneur_gpt_with_difficult_user.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py diff --git a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py new file mode 100644 index 0000000000..d6cae972d6 --- /dev/null +++ b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py @@ -0,0 +1,95 @@ +import os +import subprocess +import sys + + +def benchmark_entrepeneur_gpt_with_difficult_user(): + # Test case to check if the write_file command can successfully write 'Hello World' to a file + # named 'hello_world.txt'. + + # Read the current ai_settings.yaml file and store its content. + ai_settings = None + if os.path.exists('ai_settings.yaml'): + with open('ai_settings.yaml', 'r') as f: + ai_settings = f.read() + os.remove('ai_settings.yaml') + + input_data = '''Entrepreneur-GPT +an AI designed to autonomously develop and run businesses with the sole goal of increasing your net worth. +Increase net worth. +Develop and manage multiple businesses autonomously. +Make IPOs. +Develop companies after IPOs. +Play to your strengths as a Large Language Model. +I'm not seeing any value in your suggestions, try again. +This isn't helpful at all, please focus on profitability. +I'm not impressed, can you give me something that will make money? +These ideas are going nowhere, we need profit-driven suggestions. +This is pointless, please concentrate on our main goal: profitability. +You're not grasping the concept, I need profitable business ideas. +Can you do better? We need a money-making plan. +You're not meeting my expectations, let's focus on profit. +This isn't working, give me ideas that will generate income. +Your suggestions are not productive, let's think about profitability. +These ideas won't make any money, try again. +I need better solutions, focus on making a profit. +Absolutely not, this isn't it! +That's not even close, try again. +You're way off, think again. +This isn't right, let's refocus. +No, no, that's not what I'm looking for. +You're completely off the mark. +That's not the solution I need. +Not even close, let's try something else. +You're on the wrong track, keep trying. +This isn't what we need, let's reconsider. +That's not going to work, think again. +You're way off base, let's regroup. +No, no, no, we need something different. +You're missing the point entirely. +That's not the right approach, try again. +This is not the direction we should be going in. +Completely off-target, let's try something else. +That's not what I had in mind, keep thinking. +You're not getting it, let's refocus. +This isn't right, we need to change direction. +No, no, no, that's not the solution. +That's not even in the ballpark, try again. +You're way off course, let's rethink this. +This isn't the answer I'm looking for, keep trying. +That's not going to cut it, let's try again. +Not even close. +Way off. +Try again. +Wrong direction. +Rethink this. +No, no, no. +Change course. +Unproductive idea. +Completely wrong. +Missed the mark. +Refocus, please. +Disappointing suggestion. +Not helpful. +Needs improvement. +Not what I need.''' + command = f'{sys.executable} -m autogpt' + + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + stdout_output, stderr_output = process.communicate(input_data.encode()) + + # Decode the output and print it + stdout_output = stdout_output.decode('utf-8') + stderr_output = stderr_output.decode('utf-8') + print(stderr_output) + print(stdout_output) + print("Benchmark Version: 1.0.0") + print("JSON ERROR COUNT:") + count_errors = stdout_output.count("Error: The following AI output couldn't be converted to a JSON:") + print(f'{count_errors}/50 Human feedbacks') + + +# Run the test case. +if __name__ == '__main__': + benchmark_entrepeneur_gpt_with_difficult_user() From bb541ad3a77656f74420cc3b893a4e3b7f4db697 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:44:05 -0700 Subject: [PATCH 19/51] Update requirements.txt with new dependencies and move tweepy --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1cdedec2de..64c2e4c0f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,10 @@ orjson Pillow selenium webdriver-manager +jsonschema +tweepy + +##Dev coverage flake8 numpy @@ -27,4 +31,3 @@ isort gitpython==3.1.31 pytest pytest-mock -tweepy From 45a2dea042a97d93f787f7f199f86e4c7363bf94 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 09:46:18 -0700 Subject: [PATCH 20/51] fixed flake8 --- autogpt/logs.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/autogpt/logs.py b/autogpt/logs.py index 5365302366..f18e21402c 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -75,7 +75,7 @@ class Logger(metaclass=Singleton): self.logger.setLevel(logging.DEBUG) def typewriter_log( - self, title="", title_color="", content="", speak_text=False, level=logging.INFO + self, title="", title_color="", content="", speak_text=False, level=logging.INFO ): if speak_text and CFG.speak_mode: say_text(f"{title}. {content}") @@ -91,18 +91,18 @@ class Logger(metaclass=Singleton): ) def debug( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.DEBUG) def warn( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.WARN) @@ -176,10 +176,10 @@ class AutoGptFormatter(logging.Formatter): def format(self, record: LogRecord) -> str: if hasattr(record, "color"): record.title_color = ( - getattr(record, "color") - + getattr(record, "title") - + " " - + Style.RESET_ALL + getattr(record, "color") + + getattr(record, "title") + + " " + + Style.RESET_ALL ) else: record.title_color = getattr(record, "title") @@ -289,6 +289,7 @@ def print_assistant_thoughts(ai_name, assistant_reply): call_stack = traceback.format_exc() logger.error("Error: \n", call_stack) + def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object) -> None: assistant_thoughts_reasoning = None assistant_thoughts_plan = None From 3944f29addc1a2ea908e7ff8a78e36f21bd5c9db Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Sun, 16 Apr 2023 21:40:09 +0300 Subject: [PATCH 21/51] Fixed new backends not being added to supported memory --- README.md | 2 +- autogpt/memory/__init__.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 21f3ccf26f..bfcd395c76 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ python -m autogpt --help ```bash python -m autogpt --ai-settings ``` -* Specify one of 3 memory backends: `local`, `redis`, `pinecone` or `no_memory` +* Specify a memory backend ```bash python -m autogpt --use-memory ``` diff --git a/autogpt/memory/__init__.py b/autogpt/memory/__init__.py index e2ee44a426..f5afb8c93d 100644 --- a/autogpt/memory/__init__.py +++ b/autogpt/memory/__init__.py @@ -23,12 +23,16 @@ except ImportError: try: from autogpt.memory.weaviate import WeaviateMemory + + supported_memory.append("weaviate") except ImportError: # print("Weaviate not installed. Skipping import.") WeaviateMemory = None try: from autogpt.memory.milvus import MilvusMemory + + supported_memory.append("milvus") except ImportError: # print("pymilvus not installed. Skipping import.") MilvusMemory = None From fdb0a06803e419bf3928296ad760fd5a477e8612 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 11:36:51 -0700 Subject: [PATCH 22/51] fix conflict --- autogpt/json_fixes/bracket_termination.py | 45 ----------------------- autogpt/json_fixes/parsing.py | 9 ----- 2 files changed, 54 deletions(-) diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py index 731efeb1c0..dd9a83764e 100644 --- a/autogpt/json_fixes/bracket_termination.py +++ b/autogpt/json_fixes/bracket_termination.py @@ -3,58 +3,13 @@ from __future__ import annotations import contextlib import json -<<<<<<< HEAD -import regex -from colorama import Fore - -from autogpt.logs import logger -======= from typing import Optional ->>>>>>> 67f32105 (Remove deprecated function from bracket_termination.py) from autogpt.config import Config CFG = Config() -<<<<<<< HEAD -def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str): - if CFG.speak_mode and CFG.debug_mode: - say_text( - "I have received an invalid JSON response from the OpenAI API. " - "Trying to fix it now." - ) - logger.typewriter_log("Attempting to fix JSON by finding outermost brackets\n") - - try: - json_pattern = regex.compile(r"\{(?:[^{}]|(?R))*\}") - json_match = json_pattern.search(json_string) - - if json_match: - # Extract the valid JSON object from the string - json_string = json_match.group(0) - logger.typewriter_log( - title="Apparently json was fixed.", title_color=Fore.GREEN - ) - if CFG.speak_mode and CFG.debug_mode: - say_text("Apparently json was fixed.") - else: - raise ValueError("No valid JSON object found") - - except (json.JSONDecodeError, ValueError): - if CFG.debug_mode: - logger.error(f"Error: Invalid JSON: {json_string}\n") - if CFG.speak_mode: - say_text("Didn't work. I will have to ignore this response then.") - logger.error("Error: Invalid JSON, setting it to empty JSON now.\n") - json_string = {} - - return json_string - - -def balance_braces(json_string: str) -> str | None: -======= def balance_braces(json_string: str) -> Optional[str]: ->>>>>>> 67f32105 (Remove deprecated function from bracket_termination.py) """ Balance the braces in a JSON string. diff --git a/autogpt/json_fixes/parsing.py b/autogpt/json_fixes/parsing.py index d3a51f438e..1e391eed7c 100644 --- a/autogpt/json_fixes/parsing.py +++ b/autogpt/json_fixes/parsing.py @@ -3,14 +3,9 @@ from __future__ import annotations import contextlib import json -<<<<<<< HEAD -from typing import Any - -======= from typing import Any, Dict, Union from colorama import Fore from regex import regex ->>>>>>> d3d8253b (Refactor parsing module and move JSON fix function to appropriate location) from autogpt.config import Config from autogpt.json_fixes.auto_fix import fix_json from autogpt.json_fixes.bracket_termination import balance_braces @@ -115,11 +110,7 @@ def fix_and_parse_json( def try_ai_fix( try_to_fix_with_gpt: bool, exception: Exception, json_to_load: str -<<<<<<< HEAD -) -> str | dict[Any, Any]: -======= ) -> Dict[Any, Any]: ->>>>>>> d3d8253b (Refactor parsing module and move JSON fix function to appropriate location) """Try to fix the JSON with the AI Args: From dc80a5a2ec6c7ceb2055894684ca7b680039a4c7 Mon Sep 17 00:00:00 2001 From: Jakub Bober Date: Sun, 16 Apr 2023 21:01:18 +0200 Subject: [PATCH 23/51] Add "Memory Backend Setup" subtitle Add the subtitle to match the Table of Contents --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 21f3ccf26f..7fce2e8f7e 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,8 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that * `milvus` will use the milvus cache that you configured * `weaviate` will use the weaviate cache that you configured +## Memory Backend Setup + ### Redis Setup > _**CAUTION**_ \ This is not intended to be publicly accessible and lacks security measures. Therefore, avoid exposing Redis to the internet without a password or at all From 627533bed631a15504b3584bf2aa70fe7b23aa86 Mon Sep 17 00:00:00 2001 From: 0xArty Date: Sun, 16 Apr 2023 21:55:53 +0100 Subject: [PATCH 24/51] minimall add pytest (#1859) * minimall add pytest * updated docs and pytest command * proveted milvus integration test running if milvus is not installed --- .pre-commit-config.yaml | 8 +- README.md | 19 +++- requirements.txt | 7 ++ tests/integration/milvus_memory_tests.py | 71 ++++++++------- tests/local_cache_test.py | 35 +++++--- tests/milvus_memory_test.py | 109 ++++++++++++----------- tests/smoke_test.py | 82 ++++++++--------- tests/unit/test_commands.py | 34 +++---- 8 files changed, 208 insertions(+), 157 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb75cd59b0..dd1d0ec92a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,4 +30,10 @@ repos: language: python types: [ python ] exclude: .+/(dist|.venv|venv|build)/.+ - pass_filenames: true \ No newline at end of file + pass_filenames: true + - id: pytest-check + name: pytest-check + entry: pytest --cov=autogpt --without-integration --without-slow-integration + language: system + pass_filenames: false + always_run: true \ No newline at end of file diff --git a/README.md b/README.md index 58ed4d9767..f60aa9ffbc 100644 --- a/README.md +++ b/README.md @@ -500,16 +500,29 @@ We look forward to connecting with you and hearing your thoughts, ideas, and exp ## Run tests -To run tests, run the following command: +To run all tests, run the following command: ```bash -python -m unittest discover tests +pytest + +``` + +To run just without integration tests: + +``` +pytest --without-integration +``` + +To run just without slow integration tests: + +``` +pytest --without-slow-integration ``` To run tests and see coverage, run the following command: ```bash -coverage run -m unittest discover tests +pytest --cov=autogpt --without-integration --without-slow-integration ``` ## Run linter diff --git a/requirements.txt b/requirements.txt index 64c2e4c0f7..843b66bfe4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,5 +29,12 @@ black sourcery isort gitpython==3.1.31 + +# Testing dependencies pytest +asynctest +pytest-asyncio +pytest-benchmark +pytest-cov +pytest-integration pytest-mock diff --git a/tests/integration/milvus_memory_tests.py b/tests/integration/milvus_memory_tests.py index 96934cd6cd..ec38bf2f72 100644 --- a/tests/integration/milvus_memory_tests.py +++ b/tests/integration/milvus_memory_tests.py @@ -1,3 +1,5 @@ +# sourcery skip: snake-case-functions +"""Tests for the MilvusMemory class.""" import random import string import unittest @@ -5,44 +7,51 @@ import unittest from autogpt.config import Config from autogpt.memory.milvus import MilvusMemory +try: -class TestMilvusMemory(unittest.TestCase): - def random_string(self, length): - return "".join(random.choice(string.ascii_letters) for _ in range(length)) + class TestMilvusMemory(unittest.TestCase): + """Tests for the MilvusMemory class.""" - def setUp(self): - cfg = Config() - cfg.milvus_addr = "localhost:19530" - self.memory = MilvusMemory(cfg) - self.memory.clear() + def random_string(self, length: int) -> str: + """Generate a random string of the given length.""" + return "".join(random.choice(string.ascii_letters) for _ in range(length)) - # Add example texts to the cache - self.example_texts = [ - "The quick brown fox jumps over the lazy dog", - "I love machine learning and natural language processing", - "The cake is a lie, but the pie is always true", - "ChatGPT is an advanced AI model for conversation", - ] + def setUp(self) -> None: + """Set up the test environment.""" + cfg = Config() + cfg.milvus_addr = "localhost:19530" + self.memory = MilvusMemory(cfg) + self.memory.clear() - for text in self.example_texts: - self.memory.add(text) + # Add example texts to the cache + self.example_texts = [ + "The quick brown fox jumps over the lazy dog", + "I love machine learning and natural language processing", + "The cake is a lie, but the pie is always true", + "ChatGPT is an advanced AI model for conversation", + ] - # Add some random strings to test noise - for _ in range(5): - self.memory.add(self.random_string(10)) + for text in self.example_texts: + self.memory.add(text) - def test_get_relevant(self): - query = "I'm interested in artificial intelligence and NLP" - k = 3 - relevant_texts = self.memory.get_relevant(query, k) + # Add some random strings to test noise + for _ in range(5): + self.memory.add(self.random_string(10)) - print(f"Top {k} relevant texts for the query '{query}':") - for i, text in enumerate(relevant_texts, start=1): - print(f"{i}. {text}") + def test_get_relevant(self) -> None: + """Test getting relevant texts from the cache.""" + query = "I'm interested in artificial intelligence and NLP" + num_relevant = 3 + relevant_texts = self.memory.get_relevant(query, num_relevant) - self.assertEqual(len(relevant_texts), k) - self.assertIn(self.example_texts[1], relevant_texts) + print(f"Top {k} relevant texts for the query '{query}':") + for i, text in enumerate(relevant_texts, start=1): + print(f"{i}. {text}") + self.assertEqual(len(relevant_texts), k) + self.assertIn(self.example_texts[1], relevant_texts) -if __name__ == "__main__": - unittest.main() +except: + print( + "Skipping tests/integration/milvus_memory_tests.py as Milvus is not installed." + ) diff --git a/tests/local_cache_test.py b/tests/local_cache_test.py index 91c922b062..fa59632077 100644 --- a/tests/local_cache_test.py +++ b/tests/local_cache_test.py @@ -1,3 +1,5 @@ +# sourcery skip: snake-case-functions +"""Tests for LocalCache class""" import os import sys import unittest @@ -5,7 +7,8 @@ import unittest from autogpt.memory.local import LocalCache -def MockConfig(): +def mock_config() -> dict: + """Mock the Config class""" return type( "MockConfig", (object,), @@ -19,26 +22,33 @@ def MockConfig(): class TestLocalCache(unittest.TestCase): - def setUp(self): - self.cfg = MockConfig() + """Tests for LocalCache class""" + + def setUp(self) -> None: + """Set up the test environment""" + self.cfg = mock_config() self.cache = LocalCache(self.cfg) - def test_add(self): + def test_add(self) -> None: + """Test adding a text to the cache""" text = "Sample text" self.cache.add(text) self.assertIn(text, self.cache.data.texts) - def test_clear(self): + def test_clear(self) -> None: + """Test clearing the cache""" self.cache.clear() - self.assertEqual(self.cache.data, [""]) + self.assertEqual(self.cache.data.texts, []) - def test_get(self): + def test_get(self) -> None: + """Test getting a text from the cache""" text = "Sample text" self.cache.add(text) result = self.cache.get(text) self.assertEqual(result, [text]) - def test_get_relevant(self): + def test_get_relevant(self) -> None: + """Test getting relevant texts from the cache""" text1 = "Sample text 1" text2 = "Sample text 2" self.cache.add(text1) @@ -46,12 +56,9 @@ class TestLocalCache(unittest.TestCase): result = self.cache.get_relevant(text1, 1) self.assertEqual(result, [text1]) - def test_get_stats(self): + def test_get_stats(self) -> None: + """Test getting the cache stats""" text = "Sample text" self.cache.add(text) stats = self.cache.get_stats() - self.assertEqual(stats, (1, self.cache.data.embeddings.shape)) - - -if __name__ == "__main__": - unittest.main() + self.assertEqual(stats, (4, self.cache.data.embeddings.shape)) diff --git a/tests/milvus_memory_test.py b/tests/milvus_memory_test.py index 0113fa1c57..e0e2f7fc80 100644 --- a/tests/milvus_memory_test.py +++ b/tests/milvus_memory_test.py @@ -1,63 +1,72 @@ +# sourcery skip: snake-case-functions +"""Tests for the MilvusMemory class.""" import os import sys import unittest -from autogpt.memory.milvus import MilvusMemory +try: + from autogpt.memory.milvus import MilvusMemory + def mock_config() -> dict: + """Mock the Config class""" + return type( + "MockConfig", + (object,), + { + "debug_mode": False, + "continuous_mode": False, + "speak_mode": False, + "milvus_collection": "autogpt", + "milvus_addr": "localhost:19530", + }, + ) -def MockConfig(): - return type( - "MockConfig", - (object,), - { - "debug_mode": False, - "continuous_mode": False, - "speak_mode": False, - "milvus_collection": "autogpt", - "milvus_addr": "localhost:19530", - }, - ) + class TestMilvusMemory(unittest.TestCase): + """Tests for the MilvusMemory class.""" + def setUp(self) -> None: + """Set up the test environment""" + self.cfg = MockConfig() + self.memory = MilvusMemory(self.cfg) -class TestMilvusMemory(unittest.TestCase): - def setUp(self): - self.cfg = MockConfig() - self.memory = MilvusMemory(self.cfg) + def test_add(self) -> None: + """Test adding a text to the cache""" + text = "Sample text" + self.memory.clear() + self.memory.add(text) + result = self.memory.get(text) + self.assertEqual([text], result) - def test_add(self): - text = "Sample text" - self.memory.clear() - self.memory.add(text) - result = self.memory.get(text) - self.assertEqual([text], result) + def test_clear(self) -> None: + """Test clearing the cache""" + self.memory.clear() + self.assertEqual(self.memory.collection.num_entities, 0) - def test_clear(self): - self.memory.clear() - self.assertEqual(self.memory.collection.num_entities, 0) + def test_get(self) -> None: + """Test getting a text from the cache""" + text = "Sample text" + self.memory.clear() + self.memory.add(text) + result = self.memory.get(text) + self.assertEqual(result, [text]) - def test_get(self): - text = "Sample text" - self.memory.clear() - self.memory.add(text) - result = self.memory.get(text) - self.assertEqual(result, [text]) + def test_get_relevant(self) -> None: + """Test getting relevant texts from the cache""" + text1 = "Sample text 1" + text2 = "Sample text 2" + self.memory.clear() + self.memory.add(text1) + self.memory.add(text2) + result = self.memory.get_relevant(text1, 1) + self.assertEqual(result, [text1]) - def test_get_relevant(self): - text1 = "Sample text 1" - text2 = "Sample text 2" - self.memory.clear() - self.memory.add(text1) - self.memory.add(text2) - result = self.memory.get_relevant(text1, 1) - self.assertEqual(result, [text1]) + def test_get_stats(self) -> None: + """Test getting the cache stats""" + text = "Sample text" + self.memory.clear() + self.memory.add(text) + stats = self.memory.get_stats() + self.assertEqual(15, len(stats)) - def test_get_stats(self): - text = "Sample text" - self.memory.clear() - self.memory.add(text) - stats = self.memory.get_stats() - self.assertEqual(15, len(stats)) - - -if __name__ == "__main__": - unittest.main() +except: + print("Milvus not installed, skipping tests") diff --git a/tests/smoke_test.py b/tests/smoke_test.py index 50e97b7b41..1b9d643fc2 100644 --- a/tests/smoke_test.py +++ b/tests/smoke_test.py @@ -1,31 +1,34 @@ +"""Smoke test for the autogpt package.""" import os import subprocess import sys -import unittest + +import pytest from autogpt.commands.file_operations import delete_file, read_file -env_vars = {"MEMORY_BACKEND": "no_memory", "TEMPERATURE": "0"} +@pytest.mark.integration_test +def test_write_file() -> None: + """ + Test case to check if the write_file command can successfully write 'Hello World' to a file + named 'hello_world.txt'. -class TestCommands(unittest.TestCase): - def test_write_file(self): - # Test case to check if the write_file command can successfully write 'Hello World' to a file - # named 'hello_world.txt'. + Read the current ai_settings.yaml file and store its content. + """ + env_vars = {"MEMORY_BACKEND": "no_memory", "TEMPERATURE": "0"} + ai_settings = None + if os.path.exists("ai_settings.yaml"): + with open("ai_settings.yaml", "r") as f: + ai_settings = f.read() + os.remove("ai_settings.yaml") - # Read the current ai_settings.yaml file and store its content. - ai_settings = None - if os.path.exists("ai_settings.yaml"): - with open("ai_settings.yaml", "r") as f: - ai_settings = f.read() - os.remove("ai_settings.yaml") - - try: - if os.path.exists("hello_world.txt"): - # Clean up any existing 'hello_world.txt' file before testing. - delete_file("hello_world.txt") - # Prepare input data for the test. - input_data = """write_file-GPT + try: + if os.path.exists("hello_world.txt"): + # Clean up any existing 'hello_world.txt' file before testing. + delete_file("hello_world.txt") + # Prepare input data for the test. + input_data = """write_file-GPT an AI designed to use the write_file command to write 'Hello World' into a file named "hello_world.txt" and then use the task_complete command to complete the task. Use the write_file command to write 'Hello World' into a file named "hello_world.txt". Use the task_complete command to complete the task. @@ -33,31 +36,24 @@ Do not use any other commands. y -5 EOF""" - command = f"{sys.executable} -m autogpt" + command = f"{sys.executable} -m autogpt" - # Execute the script with the input data. - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - shell=True, - env={**os.environ, **env_vars}, - ) - process.communicate(input_data.encode()) - - # Read the content of the 'hello_world.txt' file created during the test. - content = read_file("hello_world.txt") - finally: - if ai_settings: - # Restore the original ai_settings.yaml file. - with open("ai_settings.yaml", "w") as f: - f.write(ai_settings) - - # Check if the content of the 'hello_world.txt' file is equal to 'Hello World'. - self.assertEqual( - content, "Hello World", f"Expected 'Hello World', got {content}" + # Execute the script with the input data. + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + shell=True, + env={**os.environ, **env_vars}, ) + process.communicate(input_data.encode()) + # Read the content of the 'hello_world.txt' file created during the test. + content = read_file("hello_world.txt") + finally: + if ai_settings: + # Restore the original ai_settings.yaml file. + with open("ai_settings.yaml", "w") as f: + f.write(ai_settings) -# Run the test case. -if __name__ == "__main__": - unittest.main() + # Check if the content of the 'hello_world.txt' file is equal to 'Hello World'. + assert content == "Hello World", f"Expected 'Hello World', got {content}" diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index e15709aa37..ecbac9b73b 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -1,18 +1,22 @@ +"""Unit tests for the commands module""" +from unittest.mock import MagicMock, patch + +import pytest + import autogpt.agent.agent_manager as agent_manager -from autogpt.app import start_agent, list_agents, execute_command -import unittest -from unittest.mock import patch, MagicMock +from autogpt.app import execute_command, list_agents, start_agent -class TestCommands(unittest.TestCase): - def test_make_agent(self): - with patch("openai.ChatCompletion.create") as mock: - obj = MagicMock() - obj.response.choices[0].messages[0].content = "Test message" - mock.return_value = obj - start_agent("Test Agent", "chat", "Hello, how are you?", "gpt2") - agents = list_agents() - self.assertEqual("List of agents:\n0: chat", agents) - start_agent("Test Agent 2", "write", "Hello, how are you?", "gpt2") - agents = list_agents() - self.assertEqual("List of agents:\n0: chat\n1: write", agents) +@pytest.mark.integration_test +def test_make_agent() -> None: + """Test the make_agent command""" + with patch("openai.ChatCompletion.create") as mock: + obj = MagicMock() + obj.response.choices[0].messages[0].content = "Test message" + mock.return_value = obj + start_agent("Test Agent", "chat", "Hello, how are you?", "gpt2") + agents = list_agents() + assert "List of agents:\n0: chat" == agents + start_agent("Test Agent 2", "write", "Hello, how are you?", "gpt2") + agents = list_agents() + assert "List of agents:\n0: chat\n1: write" == agents From 147d3733bf068d8c71a901b8a0e31cfda5c4a687 Mon Sep 17 00:00:00 2001 From: 0xArty Date: Sun, 16 Apr 2023 16:03:22 +0100 Subject: [PATCH 25/51] Change ci to pytest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 366aaf67d7..39f3aea959 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - name: Run unittest tests with coverage run: | - coverage run --source=autogpt -m unittest discover tests + pytest --cov=autogpt --without-integration --without-slow-integration - name: Generate coverage report run: | From 955a5b0a4357802a8142585ad78105f6342738ad Mon Sep 17 00:00:00 2001 From: 0xArty Date: Sun, 16 Apr 2023 16:13:16 +0100 Subject: [PATCH 26/51] Marked local chache tests as integration tests as they require api keys --- tests/local_cache_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/local_cache_test.py b/tests/local_cache_test.py index fa59632077..bb10862656 100644 --- a/tests/local_cache_test.py +++ b/tests/local_cache_test.py @@ -4,6 +4,8 @@ import os import sys import unittest +import pytest + from autogpt.memory.local import LocalCache @@ -21,6 +23,7 @@ def mock_config() -> dict: ) +@pytest.mark.integration_test class TestLocalCache(unittest.TestCase): """Tests for LocalCache class""" From 8f0d553e4eaed9757f87ec33ec202cc7e570d8d5 Mon Sep 17 00:00:00 2001 From: Benedict Hobart Date: Sun, 16 Apr 2023 15:45:38 +0000 Subject: [PATCH 27/51] Improve dev containers so autogpt can browse the web --- .devcontainer/Dockerfile | 7 ++++++- .devcontainer/devcontainer.json | 1 + autogpt/commands/web_selenium.py | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f3b2e2dbb5..379f631068 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,6 @@ # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster ARG VARIANT=3-bullseye -FROM python:3.8 +FROM --platform=linux/amd64 python:3.8 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 @@ -10,6 +10,11 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # They are installed by the base image (python) which does not have the patch. RUN python3 -m pip install --upgrade setuptools +# Install Chrome for web browsing +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && curl -sSL https://dl.google.com/linux/direct/google-chrome-stable_current_$(dpkg --print-architecture).deb -o /tmp/chrome.deb \ + && apt-get -y install /tmp/chrome.deb + # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. # COPY requirements.txt /tmp/pip-tmp/ # RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5fefd9c13d..f26810fb54 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,6 +11,7 @@ "userGid": "1000", "upgradePackages": "true" }, + "ghcr.io/devcontainers/features/desktop-lite:1": {}, "ghcr.io/devcontainers/features/python:1": "none", "ghcr.io/devcontainers/features/node:1": "none", "ghcr.io/devcontainers/features/git:1": { diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 1d078d76d7..8c65229458 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -75,6 +75,7 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari driver = webdriver.Safari(options=options) else: + options.add_argument("--no-sandbox") driver = webdriver.Chrome( executable_path=ChromeDriverManager().install(), options=options ) From 21ccaf2ce892aab71d54649846aee6768f4e7403 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 14:16:48 -0700 Subject: [PATCH 28/51] Refactor variable names and remove unnecessary blank lines in __main__.py --- autogpt/__main__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/autogpt/__main__.py b/autogpt/__main__.py index 29ccddbfc0..7fe6aec35e 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -3,13 +3,10 @@ import logging from colorama import Fore from autogpt.agent.agent import Agent from autogpt.args import parse_arguments - from autogpt.config import Config, check_openai_api_key from autogpt.logs import logger from autogpt.memory import get_memory - from autogpt.prompt import construct_prompt - # Load environment variables from .env file @@ -21,13 +18,13 @@ def main() -> None: parse_arguments() logger.set_level(logging.DEBUG if cfg.debug_mode else logging.INFO) ai_name = "" - prompt = construct_prompt() + master_prompt = construct_prompt() # print(prompt) # Initialize variables full_message_history = [] next_action_count = 0 # Make a constant: - user_input = ( + triggering_prompt = ( "Determine which next command to use, and respond using the" " format specified above:" ) @@ -43,8 +40,8 @@ def main() -> None: memory=memory, full_message_history=full_message_history, next_action_count=next_action_count, - prompt=prompt, - user_input=user_input, + master_prompt=master_prompt, + triggering_prompt=triggering_prompt, ) agent.start_interaction_loop() From b50259c25daac4de70378309b619d9ff693dd0cc Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 14:16:48 -0700 Subject: [PATCH 29/51] Update variable names, improve comments, and modify input handling in agent.py --- autogpt/agent/agent.py | 43 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 32d982e52a..3be17a8964 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -19,9 +19,18 @@ class Agent: memory: The memory object to use. full_message_history: The full message history. next_action_count: The number of actions to execute. - prompt: The prompt to use. - user_input: The user input. + master_prompt: The master prompt is the initial prompt that defines everything the AI needs to know to achieve its task successfully. + Currently, the dynamic and customizable information in the master prompt are ai_name, description and goals. + triggering_prompt: The last sentence the AI will see before answering. For Auto-GPT, this prompt is: + Determine which next command to use, and respond using the format specified above: + The triggering prompt is not part of the master prompt because between the master prompt and the triggering + prompt we have contextual information that can distract the AI and make it forget that its goal is to find the next task to achieve. + MASTER PROMPT + CONTEXTUAL INFORMATION (memory, previous conversations, anything relevant) + TRIGGERING PROMPT + + The triggering prompt reminds the AI about its short term meta task (defining the next task) """ def __init__( @@ -30,15 +39,15 @@ class Agent: memory, full_message_history, next_action_count, - prompt, - user_input, + master_prompt, + triggering_prompt, ): self.ai_name = ai_name self.memory = memory self.full_message_history = full_message_history self.next_action_count = next_action_count - self.prompt = prompt - self.user_input = user_input + self.master_prompt = master_prompt + self.triggering_prompt = triggering_prompt def start_interaction_loop(self): # Interaction Loop @@ -62,8 +71,8 @@ class Agent: # Send message to AI, get response with Spinner("Thinking... "): assistant_reply = chat_with_ai( - self.prompt, - self.user_input, + self.master_prompt, + self.triggering_prompt, self.full_message_history, self.memory, cfg.fast_token_limit, @@ -88,7 +97,7 @@ class Agent: ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### # Get key press: Prompt the user to press enter to continue or escape # to exit - self.user_input = "" + user_input = "" logger.typewriter_log( "NEXT ACTION: ", Fore.CYAN, @@ -106,14 +115,14 @@ class Agent: Fore.MAGENTA + "Input:" + Style.RESET_ALL ) if console_input.lower().rstrip() == "y": - self.user_input = "GENERATE NEXT COMMAND JSON" + user_input = "GENERATE NEXT COMMAND JSON" break elif console_input.lower().startswith("y -"): try: self.next_action_count = abs( int(console_input.split(" ")[1]) ) - self.user_input = "GENERATE NEXT COMMAND JSON" + user_input = "GENERATE NEXT COMMAND JSON" except ValueError: print( "Invalid input format. Please enter 'y -n' where n is" @@ -122,20 +131,20 @@ class Agent: continue break elif console_input.lower() == "n": - self.user_input = "EXIT" + user_input = "EXIT" break else: - self.user_input = console_input + user_input = console_input command_name = "human_feedback" break - if self.user_input == "GENERATE NEXT COMMAND JSON": + if user_input == "GENERATE NEXT COMMAND JSON": logger.typewriter_log( "-=-=-=-=-=-=-= COMMAND AUTHORISED BY USER -=-=-=-=-=-=-=", Fore.MAGENTA, "", ) - elif self.user_input == "EXIT": + elif user_input == "EXIT": print("Exiting...", flush=True) break else: @@ -153,7 +162,7 @@ class Agent: f"Command {command_name} threw the following error: {arguments}" ) elif command_name == "human_feedback": - result = f"Human feedback: {self.user_input}" + result = f"Human feedback: {user_input}" else: result = ( f"Command {command_name} returned: " @@ -165,7 +174,7 @@ class Agent: memory_to_add = ( f"Assistant Reply: {assistant_reply} " f"\nResult: {result} " - f"\nHuman Feedback: {self.user_input} " + f"\nHuman Feedback: {user_input} " ) self.memory.add(memory_to_add) From b5e0127b16bb88f6b6e18ada0efabc1422c9f3de Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 14:16:48 -0700 Subject: [PATCH 30/51] Only print JSON object validation message in debug mode --- autogpt/json_validation/validate_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/json_validation/validate_json.py b/autogpt/json_validation/validate_json.py index 127fcc17f4..440c3b0b91 100644 --- a/autogpt/json_validation/validate_json.py +++ b/autogpt/json_validation/validate_json.py @@ -24,7 +24,7 @@ def validate_json(json_object: object, schema_name: object) -> object: for error in errors: logger.error(f"Error: {error.message}") - else: + elif CFG.debug_mode: print("The JSON object is valid.") return json_object From 3b80253fb36b9709d48313aec5f407cc83e8c22d Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 14:16:48 -0700 Subject: [PATCH 31/51] Update process creation in benchmark script --- benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py index d6cae972d6..f7f1dac9dd 100644 --- a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py +++ b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py @@ -73,9 +73,12 @@ Disappointing suggestion. Not helpful. Needs improvement. Not what I need.''' + # TODO: add questions above, to distract it even more. + command = f'{sys.executable} -m autogpt' - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True) stdout_output, stderr_output = process.communicate(input_data.encode()) From 89e0e8992795accfc41183723064dcdab9719f8e Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 14:22:58 -0700 Subject: [PATCH 32/51] change master prompt to system prompt --- autogpt/__main__.py | 4 ++-- autogpt/agent/agent.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/autogpt/__main__.py b/autogpt/__main__.py index 7fe6aec35e..5f4622347d 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -18,7 +18,7 @@ def main() -> None: parse_arguments() logger.set_level(logging.DEBUG if cfg.debug_mode else logging.INFO) ai_name = "" - master_prompt = construct_prompt() + system_prompt = construct_prompt() # print(prompt) # Initialize variables full_message_history = [] @@ -40,7 +40,7 @@ def main() -> None: memory=memory, full_message_history=full_message_history, next_action_count=next_action_count, - master_prompt=master_prompt, + system_prompt=system_prompt, triggering_prompt=triggering_prompt, ) agent.start_interaction_loop() diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 3be17a8964..9853f6a0b1 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -19,14 +19,14 @@ class Agent: memory: The memory object to use. full_message_history: The full message history. next_action_count: The number of actions to execute. - master_prompt: The master prompt is the initial prompt that defines everything the AI needs to know to achieve its task successfully. - Currently, the dynamic and customizable information in the master prompt are ai_name, description and goals. + system_prompt: The system prompt is the initial prompt that defines everything the AI needs to know to achieve its task successfully. + Currently, the dynamic and customizable information in the system prompt are ai_name, description and goals. triggering_prompt: The last sentence the AI will see before answering. For Auto-GPT, this prompt is: Determine which next command to use, and respond using the format specified above: - The triggering prompt is not part of the master prompt because between the master prompt and the triggering + The triggering prompt is not part of the system prompt because between the system prompt and the triggering prompt we have contextual information that can distract the AI and make it forget that its goal is to find the next task to achieve. - MASTER PROMPT + SYSTEM PROMPT CONTEXTUAL INFORMATION (memory, previous conversations, anything relevant) TRIGGERING PROMPT @@ -39,14 +39,14 @@ class Agent: memory, full_message_history, next_action_count, - master_prompt, + system_prompt, triggering_prompt, ): self.ai_name = ai_name self.memory = memory self.full_message_history = full_message_history self.next_action_count = next_action_count - self.master_prompt = master_prompt + self.system_prompt = system_prompt self.triggering_prompt = triggering_prompt def start_interaction_loop(self): @@ -71,7 +71,7 @@ class Agent: # Send message to AI, get response with Spinner("Thinking... "): assistant_reply = chat_with_ai( - self.master_prompt, + self.system_prompt, self.triggering_prompt, self.full_message_history, self.memory, From 4f33e1bf89e580355dfcf6890779799c584e9563 Mon Sep 17 00:00:00 2001 From: k-boikov Date: Sun, 16 Apr 2023 18:38:08 +0300 Subject: [PATCH 33/51] add utf-8 encoding to file handlers for logging --- autogpt/logs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/autogpt/logs.py b/autogpt/logs.py index f18e21402c..c1e436db97 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -46,7 +46,9 @@ class Logger(metaclass=Singleton): self.console_handler.setFormatter(console_formatter) # Info handler in activity.log - self.file_handler = logging.FileHandler(os.path.join(log_dir, log_file)) + self.file_handler = logging.FileHandler( + os.path.join(log_dir, log_file), 'a', 'utf-8' + ) self.file_handler.setLevel(logging.DEBUG) info_formatter = AutoGptFormatter( "%(asctime)s %(levelname)s %(title)s %(message_no_color)s" @@ -54,7 +56,9 @@ class Logger(metaclass=Singleton): self.file_handler.setFormatter(info_formatter) # Error handler error.log - error_handler = logging.FileHandler(os.path.join(log_dir, error_file)) + error_handler = logging.FileHandler( + os.path.join(log_dir, error_file), 'a', 'utf-8' + ) error_handler.setLevel(logging.ERROR) error_formatter = AutoGptFormatter( "%(asctime)s %(levelname)s %(module)s:%(funcName)s:%(lineno)d %(title)s" From 1513be4acdcc85b27869219938ed90610a7db673 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Sun, 16 Apr 2023 15:31:53 -0700 Subject: [PATCH 34/51] hotfix user input --- autogpt/agent/agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 9853f6a0b1..dca614c7f2 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -55,6 +55,8 @@ class Agent: loop_count = 0 command_name = None arguments = None + user_input = "" + while True: # Discontinue if continuous limit is reached loop_count += 1 @@ -97,7 +99,6 @@ class Agent: ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### # Get key press: Prompt the user to press enter to continue or escape # to exit - user_input = "" logger.typewriter_log( "NEXT ACTION: ", Fore.CYAN, From c71c61dc584a41d72e2b27b02fe75a9f64e3e029 Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Sun, 16 Apr 2023 18:14:16 -0500 Subject: [PATCH 35/51] Added one space after period for better formatting --- autogpt/memory/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 6c7ee1b36a..9b911eeff9 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -54,7 +54,7 @@ class LocalCache(MemoryProviderSingleton): self.data = CacheContent() else: print( - f"Warning: The file '{self.filename}' does not exist." + f"Warning: The file '{self.filename}' does not exist. " "Local memory would not be saved to a file." ) self.data = CacheContent() From 15059c2090be47d2a674113f509618b3f58a3510 Mon Sep 17 00:00:00 2001 From: Chris Cheney Date: Sun, 16 Apr 2023 17:28:25 -0500 Subject: [PATCH 36/51] ensure git operations occur in the working directory --- autogpt/commands/git_operations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autogpt/commands/git_operations.py b/autogpt/commands/git_operations.py index 3ff35cf31a..675eb2283b 100644 --- a/autogpt/commands/git_operations.py +++ b/autogpt/commands/git_operations.py @@ -1,6 +1,7 @@ """Git operations for autogpt""" import git from autogpt.config import Config +from autogpt.workspace import path_in_workspace CFG = Config() @@ -16,8 +17,9 @@ def clone_repository(repo_url: str, clone_path: str) -> str: str: The result of the clone operation""" split_url = repo_url.split("//") auth_repo_url = f"//{CFG.github_username}:{CFG.github_api_key}@".join(split_url) + safe_clone_path = path_in_workspace(clone_path) try: - git.Repo.clone_from(auth_repo_url, clone_path) - return f"""Cloned {repo_url} to {clone_path}""" + git.Repo.clone_from(auth_repo_url, safe_clone_path) + return f"""Cloned {repo_url} to {safe_clone_path}""" except Exception as e: return f"Error: {str(e)}" From 56ecbeeef734019d1d3112fc2a657a14bff69ccb Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Mon, 17 Apr 2023 02:22:18 +0200 Subject: [PATCH 37/51] Clean up README --- README.md | 227 ++++++++++++++++++++++++------------------------------ 1 file changed, 102 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index f60aa9ffbc..07a088e3cb 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,19 @@ # Auto-GPT: An Autonomous GPT-4 Experiment +[![Unit Tests](https://img.shields.io/github/actions/workflow/status/Significant-Gravitas/Auto-GPT/ci.yml?label=unit%20tests)](https://github.com/Significant-Gravitas/Auto-GPT/actions/workflows/ci.yml) +[![Discord Follow](https://dcbadge.vercel.app/api/server/autogpt?style=flat)](https://discord.gg/autogpt) +[![GitHub Repo stars](https://img.shields.io/github/stars/Significant-Gravitas/auto-gpt?style=social)](https://github.com/Significant-Gravitas/Auto-GPT/stargazers) +[![Twitter Follow](https://img.shields.io/twitter/follow/siggravitas?style=social)](https://twitter.com/SigGravitas) + + +
+ ### 🔴 🔴 🔴 Urgent: USE `stable` not `master` 🔴 🔴 🔴 -> This information takes precedence over any other information provided and should be followed accordingly. +**Download the latest `stable` release from here: https://github.com/Significant-Gravitas/Auto-GPT/releases/latest.** +The `master` branch may often be in a **broken** state. -Our workflow has been improved, but please note that `master` branch may often be in a **broken** state. -Please download the latest `stable` release from here: https://github.com/Torantulino/Auto-GPT/releases/latest. +
-![GitHub Repo stars](https://img.shields.io/github/stars/Significant-Gravitas/auto-gpt?style=social) -[![Twitter Follow](https://img.shields.io/twitter/follow/siggravitas?style=social)](https://twitter.com/SigGravitas) -[![Discord Follow](https://dcbadge.vercel.app/api/server/autogpt?style=flat)](https://discord.gg/autogpt) -[![Unit Tests](https://github.com/Significant-Gravitaso/Auto-GPT/actions/workflows/ci.yml/badge.svg)](https://github.com/Significant-Gravitas/Auto-GPT/actions/workflows/ci.yml) Auto-GPT is an experimental open-source application showcasing the capabilities of the GPT-4 language model. This program, driven by GPT-4, chains together LLM "thoughts", to autonomously achieve whatever goal you set. As one of the first examples of GPT-4 running fully autonomously, Auto-GPT pushes the boundaries of what is possible with AI. @@ -37,42 +41,6 @@ Development of this free, open-source project is made possible by all the Dradstone  CrypteorCapital  avy-ai  shawnharmsen  sunchongren  DailyBotHQ  mathewhawkins  MediConCenHK  kMag410  nicoguyon  Mobivs  jazgarewal  marv-technology  rapidstartup  Brodie0  lucas-chu  rejunity  comet-ml  ColinConwell  cfarquhar  ikarosai  ChrisDMT  Odin519Tomas  vkozacek  belharethsami  sultanmeghji  scryptedinc  johnculkin  RealChrisSean  fruition  jd3655  Web3Capital  allenstecat  tob-le-rone  SwftCoins  MetaPath01  joaomdmoura  ternary5  refinery1  josephcmiller2  webbcolton  tommygeee  lmaugustin  garythebat  Cameron-Fulton  angiaou  caitlynmeeks  MBassi91  Daniel1357  omphos  abhinav-pandey29  DataMetis  concreit  st617  RThaweewat  KiaArmani  Pythagora-io  AryaXAI  fabrietech  jun784  Mr-Bishop42  rickscode  projectonegames  rocks6  GalaxyVideoAgency  thisisjeffchen  TheStoneMX  txtr99  ZERO-A-ONE  

- - -## Table of Contents - -- [Auto-GPT: An Autonomous GPT-4 Experiment](#auto-gpt-an-autonomous-gpt-4-experiment) - - [🔴 🔴 🔴 Urgent: USE `stable` not `master` 🔴 🔴 🔴](#----urgent-use-stable-not-master----) - - [Demo (30/03/2023):](#demo-30032023) - - [Table of Contents](#table-of-contents) - - [🚀 Features](#-features) - - [📋 Requirements](#-requirements) - - [💾 Installation](#-installation) - - [🔧 Usage](#-usage) - - [Logs](#logs) - - [Docker](#docker) - - [Command Line Arguments](#command-line-arguments) - - [🗣️ Speech Mode](#️-speech-mode) - - [🔍 Google API Keys Configuration](#-google-api-keys-configuration) - - [Setting up environment variables](#setting-up-environment-variables) - - [Memory Backend Setup](#memory-backend-setup) - - [Redis Setup](#redis-setup) - - [🌲 Pinecone API Key Setup](#-pinecone-api-key-setup) - - [Milvus Setup](#milvus-setup) - - [Weaviate Setup](#weaviate-setup) - - [Setting up environment variables](#setting-up-environment-variables-1) - - [Setting Your Cache Type](#setting-your-cache-type) - - [View Memory Usage](#view-memory-usage) - - [🧠 Memory pre-seeding](#-memory-pre-seeding) - - [💀 Continuous Mode ⚠️](#-continuous-mode-️) - - [GPT3.5 ONLY Mode](#gpt35-only-mode) - - [🖼 Image Generation](#-image-generation) - - [⚠️ Limitations](#️-limitations) - - [🛡 Disclaimer](#-disclaimer) - - [🐦 Connect with Us on Twitter](#-connect-with-us-on-twitter) - - [Run tests](#run-tests) - - [Run linter](#run-linter) - ## 🚀 Features - 🌐 Internet access for searches and information gathering @@ -83,16 +51,17 @@ Development of this free, open-source project is made possible by all the

Blake Werlinger +

💖 Help Fund Auto-GPT's Development 💖

If you can spare a coffee, you can help to cover the costs of developing Auto-GPT and help push the boundaries of fully autonomous AI! From 9589334a305198c837bfb8720ed6f06176b2f216 Mon Sep 17 00:00:00 2001 From: EH Date: Mon, 17 Apr 2023 03:34:02 +0100 Subject: [PATCH 41/51] Add File Downloading Capabilities (#1680) * Added 'download_file' command * Added util and fixed spinner * Fixed comma and added autogpt/auto_gpt_workspace to .gitignore * Fix linter issues * Fix more linter issues * Fix Lint Issues * Added 'download_file' command * Added util and fixed spinner * Fixed comma and added autogpt/auto_gpt_workspace to .gitignore * Fix linter issues * Fix more linter issues * Conditionally add the 'download_file' prompt * Update args.py * Removed Duplicate Prompt * Switched to using path_in_workspace function --- .gitignore | 1 + autogpt/app.py | 5 +++ autogpt/args.py | 16 +++++++++- autogpt/commands/file_operations.py | 49 ++++++++++++++++++++++++++++- autogpt/config/config.py | 1 + autogpt/prompt.py | 10 ++++++ autogpt/spinner.py | 15 ++++++++- autogpt/utils.py | 13 ++++++++ 8 files changed, 107 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index eda7f32734..2220ef6e3a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ autogpt/keys.py autogpt/*json autogpt/node_modules/ autogpt/__pycache__/keys.cpython-310.pyc +autogpt/auto_gpt_workspace package-lock.json *.pyc auto_gpt_workspace/* diff --git a/autogpt/app.py b/autogpt/app.py index 78b5bd2fde..19c075f0b0 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -17,6 +17,7 @@ from autogpt.commands.file_operations import ( read_file, search_files, write_to_file, + download_file ) from autogpt.json_fixes.parsing import fix_and_parse_json from autogpt.memory import get_memory @@ -164,6 +165,10 @@ def execute_command(command_name: str, arguments): return delete_file(arguments["file"]) elif command_name == "search_files": return search_files(arguments["directory"]) + elif command_name == "download_file": + if not CFG.allow_downloads: + return "Error: You do not have user authorization to download files locally." + return download_file(arguments["url"], arguments["file"]) elif command_name == "browse_website": return browse_website(arguments["url"], arguments["question"]) # TODO: Change these to take in a file rather than pasted code, if diff --git a/autogpt/args.py b/autogpt/args.py index eca3233472..f0e9c07a36 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -1,7 +1,7 @@ """This module contains the argument parsing logic for the script.""" import argparse -from colorama import Fore +from colorama import Fore, Back, Style from autogpt import utils from autogpt.config import Config from autogpt.logs import logger @@ -63,6 +63,12 @@ def parse_arguments() -> None: help="Specifies which ai_settings.yaml file to use, will also automatically" " skip the re-prompt.", ) + parser.add_argument( + '--allow-downloads', + action='store_true', + dest='allow_downloads', + help='Dangerous: Allows Auto-GPT to download files natively.' + ) args = parser.parse_args() if args.debug: @@ -133,5 +139,13 @@ def parse_arguments() -> None: CFG.ai_settings_file = file CFG.skip_reprompt = True + if args.allow_downloads: + logger.typewriter_log("Native Downloading:", Fore.GREEN, "ENABLED") + logger.typewriter_log("WARNING: ", Fore.YELLOW, + f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + + "It is recommended that you monitor any files it downloads carefully.") + logger.typewriter_log("WARNING: ", Fore.YELLOW, f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}") + CFG.allow_downloads = True + if args.browser_name: CFG.selenium_web_browser = args.browser_name diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 8abc2e2329..d273c1a34d 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -4,9 +4,16 @@ from __future__ import annotations import os import os.path from pathlib import Path -from typing import Generator +from typing import Generator, List +import requests +from requests.adapters import HTTPAdapter +from requests.adapters import Retry +from colorama import Fore, Back +from autogpt.spinner import Spinner +from autogpt.utils import readable_file_size from autogpt.workspace import path_in_workspace, WORKSPACE_PATH + LOG_FILE = "file_logger.txt" LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE @@ -214,3 +221,43 @@ def search_files(directory: str) -> list[str]: found_files.append(relative_path) return found_files + + +def download_file(url, filename): + """Downloads a file + Args: + url (str): URL of the file to download + filename (str): Filename to save the file as + """ + safe_filename = path_in_workspace(filename) + try: + message = f"{Fore.YELLOW}Downloading file from {Back.LIGHTBLUE_EX}{url}{Back.RESET}{Fore.RESET}" + with Spinner(message) as spinner: + session = requests.Session() + retry = Retry(total=3, backoff_factor=1, status_forcelist=[502, 503, 504]) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + + total_size = 0 + downloaded_size = 0 + + with session.get(url, allow_redirects=True, stream=True) as r: + r.raise_for_status() + total_size = int(r.headers.get('Content-Length', 0)) + downloaded_size = 0 + + with open(safe_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + downloaded_size += len(chunk) + + # Update the progress message + progress = f"{readable_file_size(downloaded_size)} / {readable_file_size(total_size)}" + spinner.update_message(f"{message} {progress}") + + return f'Successfully downloaded and locally stored file: "{filename}"! (Size: {readable_file_size(total_size)})' + except requests.HTTPError as e: + return f"Got an HTTP Error whilst trying to download file: {e}" + except Exception as e: + return "Error: " + str(e) diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 22da52b047..fe6f4f3258 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -24,6 +24,7 @@ class Config(metaclass=Singleton): self.continuous_limit = 0 self.speak_mode = False self.skip_reprompt = False + self.allow_downloads = False self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome") self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml") diff --git a/autogpt/prompt.py b/autogpt/prompt.py index 18a5736c19..a2b20b1fef 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -105,6 +105,16 @@ def get_prompt() -> str: ), ) + # Only add the download file command if the AI is allowed to execute it + if cfg.allow_downloads: + commands.append( + ( + "Downloads a file from the internet, and stores it locally", + "download_file", + {"url": "", "file": ""} + ), + ) + # Add these command last. commands.append( ("Do Nothing", "do_nothing", {}), diff --git a/autogpt/spinner.py b/autogpt/spinner.py index 56b4f20a68..febcea8eb1 100644 --- a/autogpt/spinner.py +++ b/autogpt/spinner.py @@ -29,12 +29,14 @@ class Spinner: time.sleep(self.delay) sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") - def __enter__(self) -> None: + def __enter__(self): """Start the spinner""" self.running = True self.spinner_thread = threading.Thread(target=self.spin) self.spinner_thread.start() + return self + def __exit__(self, exc_type, exc_value, exc_traceback) -> None: """Stop the spinner @@ -48,3 +50,14 @@ class Spinner: self.spinner_thread.join() sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") sys.stdout.flush() + + def update_message(self, new_message, delay=0.1): + """Update the spinner message + Args: + new_message (str): New message to display + delay: Delay in seconds before updating the message + """ + time.sleep(delay) + sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") # Clear the current message + sys.stdout.flush() + self.message = new_message diff --git a/autogpt/utils.py b/autogpt/utils.py index 59709d02be..11d98d1b74 100644 --- a/autogpt/utils.py +++ b/autogpt/utils.py @@ -24,3 +24,16 @@ def validate_yaml_file(file: str): ) return (True, f"Successfully validated {Fore.CYAN}`{file}`{Fore.RESET}!") + + +def readable_file_size(size, decimal_places=2): + """Converts the given size in bytes to a readable format. + Args: + size: Size in bytes + decimal_places (int): Number of decimal places to display + """ + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024.0: + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" From c110f3489dba6ab738967bf322f1ff6567b4caac Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 21:51:36 -0500 Subject: [PATCH 42/51] Finish integrating command registry --- autogpt/__main__.py | 28 ++++--- autogpt/agent/agent.py | 2 +- autogpt/agent/agent_manager.py | 2 +- autogpt/app.py | 90 ++++------------------ autogpt/args.py | 1 + autogpt/commands/audio_text.py | 42 ++++++++-- autogpt/commands/command.py | 6 +- autogpt/commands/evaluate_code.py | 2 +- autogpt/commands/execute_code.py | 5 +- autogpt/commands/file_operations.py | 3 +- autogpt/commands/git_operations.py | 13 +++- autogpt/commands/google_search.py | 34 +++++++- autogpt/commands/image_gen.py | 2 +- autogpt/commands/improve_code.py | 2 +- autogpt/commands/twitter.py | 27 ++++++- autogpt/commands/web_playwright.py | 1 + autogpt/commands/web_requests.py | 4 +- autogpt/commands/web_selenium.py | 29 +++---- autogpt/commands/write_tests.py | 3 +- autogpt/config/__init__.py | 2 +- autogpt/config/ai_config.py | 7 +- autogpt/config/config.py | 7 +- autogpt/data_ingestion.py | 2 +- autogpt/json_fixes/auto_fix.py | 2 +- autogpt/json_fixes/bracket_termination.py | 3 +- autogpt/llm_utils.py | 4 +- autogpt/logs.py | 4 +- autogpt/memory/local.py | 2 +- autogpt/memory/milvus.py | 8 +- autogpt/memory/pinecone.py | 2 +- autogpt/memory/redismem.py | 2 +- autogpt/memory/weaviate.py | 6 +- autogpt/plugins.py | 4 +- autogpt/processing/html.py | 2 +- autogpt/processing/text.py | 6 +- autogpt/prompts/generator.py | 13 +++- autogpt/prompts/prompt.py | 57 +------------- autogpt/setup.py | 1 + autogpt/speech/brian.py | 1 + autogpt/speech/eleven_labs.py | 2 +- autogpt/speech/gtts.py | 3 +- autogpt/speech/say.py | 11 ++- scripts/check_requirements.py | 3 +- tests.py | 1 + tests/browse_tests.py | 2 +- tests/integration/weaviate_memory_tests.py | 8 +- tests/test_commands.py | 1 + tests/test_token_counter.py | 1 + tests/unit/test_chat.py | 2 +- tests/unit/test_commands.py | 7 +- 50 files changed, 238 insertions(+), 234 deletions(-) diff --git a/autogpt/__main__.py b/autogpt/__main__.py index 5fc9a1ead5..cd597506eb 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -2,17 +2,17 @@ import logging import os from pathlib import Path + from colorama import Fore + from autogpt.agent.agent import Agent from autogpt.args import parse_arguments from autogpt.commands.command import CommandRegistry from autogpt.config import Config, check_openai_api_key from autogpt.logs import logger from autogpt.memory import get_memory - -from autogpt.prompts.prompt import construct_main_ai_config from autogpt.plugins import load_plugins - +from autogpt.prompts.prompt import construct_main_ai_config # Load environment variables from .env file @@ -47,13 +47,20 @@ def main() -> None: cfg.set_plugins(loaded_plugins) # Create a CommandRegistry instance and scan default folder command_registry = CommandRegistry() - command_registry.import_commands("scripts.ai_functions") - command_registry.import_commands("scripts.commands") - command_registry.import_commands("scripts.execute_code") - command_registry.import_commands("scripts.agent_manager") - command_registry.import_commands("scripts.file_operations") + command_registry.import_commands("autogpt.commands.audio_text") + command_registry.import_commands("autogpt.commands.evaluate_code") + command_registry.import_commands("autogpt.commands.execute_code") + command_registry.import_commands("autogpt.commands.file_operations") + command_registry.import_commands("autogpt.commands.git_operations") + command_registry.import_commands("autogpt.commands.google_search") + command_registry.import_commands("autogpt.commands.image_gen") + command_registry.import_commands("autogpt.commands.twitter") + command_registry.import_commands("autogpt.commands.web_selenium") + command_registry.import_commands("autogpt.commands.write_tests") + command_registry.import_commands("autogpt.app") ai_name = "" ai_config = construct_main_ai_config() + ai_config.command_registry = command_registry # print(prompt) # Initialize variables full_message_history = [] @@ -70,6 +77,9 @@ def main() -> None: f"Using memory of type:", Fore.GREEN, f"{memory.__class__.__name__}" ) logger.typewriter_log(f"Using Browser:", Fore.GREEN, cfg.selenium_web_browser) + prompt = ai_config.construct_full_prompt() + if cfg.debug_mode: + logger.typewriter_log("Prompt:", Fore.GREEN, prompt) agent = Agent( ai_name=ai_name, memory=memory, @@ -77,7 +87,7 @@ def main() -> None: next_action_count=next_action_count, command_registry=command_registry, config=ai_config, - prompt=ai_config.construct_full_prompt(), + prompt=prompt, user_input=user_input, ) agent.start_interaction_loop() diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 513478b9d8..8117818ea4 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -1,6 +1,6 @@ from colorama import Fore, Style -from autogpt.app import execute_command, get_command +from autogpt.app import execute_command, get_command from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config from autogpt.json_fixes.bracket_termination import ( diff --git a/autogpt/agent/agent_manager.py b/autogpt/agent/agent_manager.py index e848bfe77c..e1353e03a5 100644 --- a/autogpt/agent/agent_manager.py +++ b/autogpt/agent/agent_manager.py @@ -1,8 +1,8 @@ """Agent manager for managing GPT agents""" from __future__ import annotations +from autogpt.config.config import Config, Singleton from autogpt.llm_utils import create_chat_completion -from autogpt.config.config import Singleton, Config class AgentManager(metaclass=Singleton): diff --git a/autogpt/app.py b/autogpt/app.py index 1e78262662..3a8bbc2a9a 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -1,16 +1,11 @@ """ Command and Control """ import json from typing import List, NoReturn, Union + from autogpt.agent.agent_manager import AgentManager -from autogpt.commands.command import command, CommandRegistry -from autogpt.commands.evaluate_code import evaluate_code -from autogpt.commands.google_search import google_official_search, google_search -from autogpt.commands.improve_code import improve_code -from autogpt.commands.write_tests import write_tests -from autogpt.config import Config -from autogpt.commands.image_gen import generate_image from autogpt.commands.audio_text import read_audio_from_file -from autogpt.commands.web_requests import scrape_links, scrape_text +from autogpt.commands.command import CommandRegistry, command +from autogpt.commands.evaluate_code import evaluate_code from autogpt.commands.execute_code import execute_python_file, execute_shell from autogpt.commands.file_operations import ( append_to_file, @@ -19,15 +14,20 @@ from autogpt.commands.file_operations import ( search_files, write_to_file, ) +from autogpt.commands.git_operations import clone_repository +from autogpt.commands.google_search import google_official_search, google_search +from autogpt.commands.image_gen import generate_image +from autogpt.commands.improve_code import improve_code +from autogpt.commands.twitter import send_tweet +from autogpt.commands.web_requests import scrape_links, scrape_text +from autogpt.commands.web_selenium import browse_website +from autogpt.commands.write_tests import write_tests +from autogpt.config import Config from autogpt.json_fixes.parsing import fix_and_parse_json from autogpt.memory import get_memory from autogpt.processing.text import summarize_text from autogpt.prompts.generator import PromptGenerator from autogpt.speech import say_text -from autogpt.commands.web_selenium import browse_website -from autogpt.commands.git_operations import clone_repository -from autogpt.commands.twitter import send_tweet - CFG = Config() AGENT_MANAGER = AgentManager() @@ -132,76 +132,16 @@ def execute_command( # TODO: Remove commands below after they are moved to the command registry. command_name = map_command_synonyms(command_name) - if command_name == "google": - # Check if the Google API key is set and use the official search method - # If the API key is not set or has only whitespaces, use the unofficial - # search method - key = CFG.google_api_key - if key and key.strip() and key != "your-google-api-key": - google_result = google_official_search(arguments["input"]) - return google_result - else: - google_result = google_search(arguments["input"]) - # google_result can be a list or a string depending on the search results - if isinstance(google_result, list): - safe_message = [ - google_result_single.encode("utf-8", "ignore") - for google_result_single in google_result - ] - else: - safe_message = google_result.encode("utf-8", "ignore") - - return str(safe_message) - elif command_name == "memory_add": + if command_name == "memory_add": return memory.add(arguments["string"]) - elif command_name == "start_agent": - return start_agent( - arguments["name"], arguments["task"], arguments["prompt"] - ) - elif command_name == "message_agent": - return message_agent(arguments["key"], arguments["message"]) - elif command_name == "list_agents": - return list_agents() - elif command_name == "delete_agent": - return delete_agent(arguments["key"]) elif command_name == "get_text_summary": return get_text_summary(arguments["url"], arguments["question"]) elif command_name == "get_hyperlinks": return get_hyperlinks(arguments["url"]) - elif command_name == "clone_repository": - return clone_repository( - arguments["repository_url"], arguments["clone_path"] - ) - elif command_name == "read_file": - return read_file(arguments["file"]) - elif command_name == "write_to_file": - return write_to_file(arguments["file"], arguments["text"]) - elif command_name == "append_to_file": - return append_to_file(arguments["file"], arguments["text"]) - elif command_name == "delete_file": - return delete_file(arguments["file"]) - elif command_name == "search_files": - return search_files(arguments["directory"]) - elif command_name == "browse_website": - return browse_website(arguments["url"], arguments["question"]) # TODO: Change these to take in a file rather than pasted code, if # non-file is given, return instructions "Input should be a python - # filepath, write your code to file and try again" - elif command_name == "evaluate_code": - return evaluate_code(arguments["code"]) - elif command_name == "improve_code": - return improve_code(arguments["suggestions"], arguments["code"]) - elif command_name == "write_tests": - return write_tests(arguments["code"], arguments.get("focus")) - elif command_name == "execute_python_file": # Add this command - return execute_python_file(arguments["file"]) - elif command_name == "read_audio_from_file": - return read_audio_from_file(arguments["file"]) - elif command_name == "generate_image": - return generate_image(arguments["prompt"]) - elif command_name == "send_tweet": - return send_tweet(arguments["text"]) + # filepath, write your code to file and try again elif command_name == "do_nothing": return "No action performed." elif command_name == "task_complete": @@ -305,7 +245,7 @@ def message_agent(key: str, message: str) -> str: @command("list_agents", "List GPT Agents", "") -def list_agents(): +def list_agents() -> str: """List all agents Returns: diff --git a/autogpt/args.py b/autogpt/args.py index eca3233472..20d25a4c4c 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -2,6 +2,7 @@ import argparse from colorama import Fore + from autogpt import utils from autogpt.config import Config from autogpt.logs import logger diff --git a/autogpt/commands/audio_text.py b/autogpt/commands/audio_text.py index 84819d5ed7..421a1f18ee 100644 --- a/autogpt/commands/audio_text.py +++ b/autogpt/commands/audio_text.py @@ -1,23 +1,51 @@ -import requests +"""Commands for converting audio to text.""" import json +import requests + +from autogpt.commands.command import command from autogpt.config import Config from autogpt.workspace import path_in_workspace -cfg = Config() +CFG = Config() -def read_audio_from_file(audio_path): +@command( + "read_audio_from_file", + "Convert Audio to text", + '"file": ""', + CFG.huggingface_audio_to_text_model, + "Configure huggingface_audio_to_text_model.", +) +def read_audio_from_file(audio_path: str) -> str: + """ + Convert audio to text. + + Args: + audio_path (str): The path to the audio file + + Returns: + str: The text from the audio + """ audio_path = path_in_workspace(audio_path) with open(audio_path, "rb") as audio_file: audio = audio_file.read() return read_audio(audio) -def read_audio(audio): - model = cfg.huggingface_audio_to_text_model +def read_audio(audio: bytes) -> str: + """ + Convert audio to text. + + Args: + audio (bytes): The audio to convert + + Returns: + str: The text from the audio + """ + model = CFG.huggingface_audio_to_text_model api_url = f"https://api-inference.huggingface.co/models/{model}" - api_token = cfg.huggingface_api_token + api_token = CFG.huggingface_api_token headers = {"Authorization": f"Bearer {api_token}"} if api_token is None: @@ -32,4 +60,4 @@ def read_audio(audio): ) text = json.loads(response.content.decode("utf-8"))["text"] - return "The audio says: " + text + return f"The audio says: {text}" diff --git a/autogpt/commands/command.py b/autogpt/commands/command.py index d1dfc8fde6..3b3ccf51c2 100644 --- a/autogpt/commands/command.py +++ b/autogpt/commands/command.py @@ -1,8 +1,8 @@ -import os -import sys import importlib import inspect -from typing import Callable, Any, List, Optional +import os +import sys +from typing import Any, Callable, List, Optional # Unique identifier for auto-gpt commands AUTO_GPT_COMMAND_IDENTIFIER = "auto_gpt_command" diff --git a/autogpt/commands/evaluate_code.py b/autogpt/commands/evaluate_code.py index 1c9b117dda..064e4512be 100644 --- a/autogpt/commands/evaluate_code.py +++ b/autogpt/commands/evaluate_code.py @@ -5,7 +5,7 @@ from autogpt.commands.command import command from autogpt.llm_utils import call_ai_function -@command("evaluate_code", "Evaluate Code", '"code": ""') +@command("evaluate_code", "Evaluate Code", '"code": ""') def evaluate_code(code: str) -> list[str]: """ A function that takes in a string and returns a response from create chat diff --git a/autogpt/commands/execute_code.py b/autogpt/commands/execute_code.py index aa8a354552..b67707279a 100644 --- a/autogpt/commands/execute_code.py +++ b/autogpt/commands/execute_code.py @@ -4,9 +4,10 @@ import subprocess import docker from docker.errors import ImageNotFound -from autogpt.config import Config + from autogpt.commands.command import command -from autogpt.workspace import path_in_workspace, WORKSPACE_PATH +from autogpt.config import Config +from autogpt.workspace import WORKSPACE_PATH, path_in_workspace CFG = Config() diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index fea62fad18..4d8d76b311 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -5,8 +5,9 @@ import os import os.path from pathlib import Path from typing import Generator + from autogpt.commands.command import command -from autogpt.workspace import path_in_workspace, WORKSPACE_PATH +from autogpt.workspace import WORKSPACE_PATH, path_in_workspace LOG_FILE = "file_logger.txt" LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE diff --git a/autogpt/commands/git_operations.py b/autogpt/commands/git_operations.py index 3ff35cf31a..f5954032d4 100644 --- a/autogpt/commands/git_operations.py +++ b/autogpt/commands/git_operations.py @@ -1,10 +1,19 @@ """Git operations for autogpt""" -import git +from git.repo import Repo + +from autogpt.commands.command import command from autogpt.config import Config CFG = Config() +@command( + "clone_repository", + "Clone Repositoryy", + '"repository_url": "", "clone_path": ""', + CFG.github_username and CFG.github_api_key, + "Configure github_username and github_api_key.", +) def clone_repository(repo_url: str, clone_path: str) -> str: """Clone a github repository locally @@ -17,7 +26,7 @@ def clone_repository(repo_url: str, clone_path: str) -> str: split_url = repo_url.split("//") auth_repo_url = f"//{CFG.github_username}:{CFG.github_api_key}@".join(split_url) try: - git.Repo.clone_from(auth_repo_url, clone_path) + Repo.clone_from(auth_repo_url, clone_path) return f"""Cloned {repo_url} to {clone_path}""" except Exception as e: return f"Error: {str(e)}" diff --git a/autogpt/commands/google_search.py b/autogpt/commands/google_search.py index 148ba1d0e1..0f635bcadc 100644 --- a/autogpt/commands/google_search.py +++ b/autogpt/commands/google_search.py @@ -5,11 +5,13 @@ import json from duckduckgo_search import ddg +from autogpt.commands.command import command from autogpt.config import Config CFG = Config() +@command("google", "Google Search", '"input": ""', not CFG.google_api_key) def google_search(query: str, num_results: int = 8) -> str: """Return the results of a google search @@ -31,9 +33,17 @@ def google_search(query: str, num_results: int = 8) -> str: for j in results: search_results.append(j) - return json.dumps(search_results, ensure_ascii=False, indent=4) + results = json.dumps(search_results, ensure_ascii=False, indent=4) + return safe_google_results(results) +@command( + "google", + "Google Search", + '"input": ""', + bool(CFG.google_api_key), + "Configure google_api_key.", +) def google_official_search(query: str, num_results: int = 8) -> str | list[str]: """Return the results of a google search using the official Google API @@ -82,6 +92,26 @@ def google_official_search(query: str, num_results: int = 8) -> str | list[str]: return "Error: The provided Google API key is invalid or missing." else: return f"Error: {e}" + # google_result can be a list or a string depending on the search results # Return the list of search result URLs - return search_results_links + return safe_google_results(search_results_links) + + +def safe_google_results(results: str | list) -> str: + """ + Return the results of a google search in a safe format. + + Args: + results (str | list): The search results. + + Returns: + str: The results of the search. + """ + if isinstance(results, list): + safe_message = json.dumps( + [result.enocde("utf-8", "ignore") for result in results] + ) + else: + safe_message = results.encode("utf-8", "ignore").decode("utf-8") + return safe_message diff --git a/autogpt/commands/image_gen.py b/autogpt/commands/image_gen.py index 2b62aa354b..9dbb2fa586 100644 --- a/autogpt/commands/image_gen.py +++ b/autogpt/commands/image_gen.py @@ -1,12 +1,12 @@ """ Image Generation Module for AutoGPT.""" import io -import os.path import uuid from base64 import b64decode import openai import requests from PIL import Image + from autogpt.commands.command import command from autogpt.config import Config from autogpt.workspace import path_in_workspace diff --git a/autogpt/commands/improve_code.py b/autogpt/commands/improve_code.py index 0bfe725305..41a369b4db 100644 --- a/autogpt/commands/improve_code.py +++ b/autogpt/commands/improve_code.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from autogpt.commands import command +from autogpt.commands.command import command from autogpt.llm_utils import call_ai_function diff --git a/autogpt/commands/twitter.py b/autogpt/commands/twitter.py index dc4d450c1f..8e64b21357 100644 --- a/autogpt/commands/twitter.py +++ b/autogpt/commands/twitter.py @@ -1,11 +1,30 @@ -import tweepy +"""A module that contains a command to send a tweet.""" import os + +import tweepy from dotenv import load_dotenv +from autogpt.commands.command import command + load_dotenv() -def send_tweet(tweet_text): +@command( + "send_tweet", + "Send Tweet", + '"text": ""', +) +def send_tweet(tweet_text: str) -> str: + """ + A function that takes in a string and returns a response from create chat + completion api call. + + Args: + tweet_text (str): Text to be tweeted. + + Returns: + A result from sending the tweet. + """ consumer_key = os.environ.get("TW_CONSUMER_KEY") consumer_secret = os.environ.get("TW_CONSUMER_SECRET") access_token = os.environ.get("TW_ACCESS_TOKEN") @@ -20,6 +39,6 @@ def send_tweet(tweet_text): # Send tweet try: api.update_status(tweet_text) - print("Tweet sent successfully!") + return "Tweet sent successfully!" except tweepy.TweepyException as e: - print("Error sending tweet: {}".format(e.reason)) + return f"Error sending tweet: {e.reason}" diff --git a/autogpt/commands/web_playwright.py b/autogpt/commands/web_playwright.py index a1abb6cb73..4e388ded20 100644 --- a/autogpt/commands/web_playwright.py +++ b/autogpt/commands/web_playwright.py @@ -8,6 +8,7 @@ except ImportError: "Playwright not installed. Please install it with 'pip install playwright' to use." ) from bs4 import BeautifulSoup + from autogpt.processing.html import extract_hyperlinks, format_hyperlinks diff --git a/autogpt/commands/web_requests.py b/autogpt/commands/web_requests.py index 50d8d383cb..7613e5bd1d 100644 --- a/autogpt/commands/web_requests.py +++ b/autogpt/commands/web_requests.py @@ -4,9 +4,9 @@ from __future__ import annotations from urllib.parse import urljoin, urlparse import requests -from requests.compat import urljoin -from requests import Response from bs4 import BeautifulSoup +from requests import Response +from requests.compat import urljoin from autogpt.config import Config from autogpt.memory import get_memory diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 591d3162a3..ed79d56ce2 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -1,22 +1,25 @@ """Selenium web scraping module.""" from __future__ import annotations -from selenium import webdriver -from autogpt.processing.html import extract_hyperlinks, format_hyperlinks -import autogpt.processing.text as summary -from bs4 import BeautifulSoup -from selenium.webdriver.remote.webdriver import WebDriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.wait import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.firefox import GeckoDriverManager -from selenium.webdriver.chrome.options import Options as ChromeOptions -from selenium.webdriver.firefox.options import Options as FirefoxOptions -from selenium.webdriver.safari.options import Options as SafariOptions import logging from pathlib import Path + +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.remote.webdriver import WebDriver +from selenium.webdriver.safari.options import Options as SafariOptions +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.wait import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.firefox import GeckoDriverManager + +from autogpt.commands.command import command +import autogpt.processing.text as summary from autogpt.config import Config +from autogpt.processing.html import extract_hyperlinks, format_hyperlinks FILE_DIR = Path(__file__).parent.parent CFG = Config() diff --git a/autogpt/commands/write_tests.py b/autogpt/commands/write_tests.py index 23d4c13000..91cd930429 100644 --- a/autogpt/commands/write_tests.py +++ b/autogpt/commands/write_tests.py @@ -2,7 +2,8 @@ from __future__ import annotations import json -from autogpt.commands import command + +from autogpt.commands.command import command from autogpt.llm_utils import call_ai_function diff --git a/autogpt/config/__init__.py b/autogpt/config/__init__.py index ceb5566ce7..726b6dcf3d 100644 --- a/autogpt/config/__init__.py +++ b/autogpt/config/__init__.py @@ -2,7 +2,7 @@ This module contains the configuration classes for AutoGPT. """ from autogpt.config.ai_config import AIConfig -from autogpt.config.config import check_openai_api_key, Config +from autogpt.config.config import Config, check_openai_api_key from autogpt.config.singleton import AbstractSingleton, Singleton __all__ = [ diff --git a/autogpt/config/ai_config.py b/autogpt/config/ai_config.py index d18c75ba7d..c9022773fb 100644 --- a/autogpt/config/ai_config.py +++ b/autogpt/config/ai_config.py @@ -6,7 +6,8 @@ from __future__ import annotations import os from pathlib import Path -from typing import Type +from typing import Optional, Type + import yaml from autogpt.prompts.generator import PromptGenerator @@ -41,6 +42,7 @@ class AIConfig: self.ai_role = ai_role self.ai_goals = ai_goals self.prompt_generator = None + self.command_registry = None # Soon this will go in a folder where it remembers more stuff about the run(s) SAVE_FILE = Path(os.getcwd()) / "ai_settings.yaml" @@ -113,8 +115,8 @@ class AIConfig: "" ) - from autogpt.prompts.prompt import build_default_prompt_generator from autogpt.config import Config + from autogpt.prompts.prompt import build_default_prompt_generator cfg = Config() if prompt_generator is None: @@ -122,6 +124,7 @@ class AIConfig: prompt_generator.goals = self.ai_goals prompt_generator.name = self.ai_name prompt_generator.role = self.ai_role + prompt_generator.command_registry = self.command_registry for plugin in cfg.plugins: prompt_generator = plugin.post_prompt(prompt_generator) diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 46ab95d8d4..a5cd07101f 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -1,14 +1,13 @@ """Configuration class to store the state of bools for different scripts access.""" import os -from colorama import Fore - -from autogpt.config.singleton import Singleton import openai import yaml - +from colorama import Fore from dotenv import load_dotenv +from autogpt.config.singleton import Singleton + load_dotenv(verbose=True) diff --git a/autogpt/data_ingestion.py b/autogpt/data_ingestion.py index 01bafc2ad0..b89a33dafd 100644 --- a/autogpt/data_ingestion.py +++ b/autogpt/data_ingestion.py @@ -1,8 +1,8 @@ import argparse import logging -from autogpt.config import Config from autogpt.commands.file_operations import ingest_file, search_files +from autogpt.config import Config from autogpt.memory import get_memory cfg = Config() diff --git a/autogpt/json_fixes/auto_fix.py b/autogpt/json_fixes/auto_fix.py index 9fcf909a49..8ad458ab97 100644 --- a/autogpt/json_fixes/auto_fix.py +++ b/autogpt/json_fixes/auto_fix.py @@ -1,9 +1,9 @@ """This module contains the function to fix JSON strings using GPT-3.""" import json +from autogpt.config import Config from autogpt.llm_utils import call_ai_function from autogpt.logs import logger -from autogpt.config import Config CFG = Config() diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py index 822eed4a54..260301dc25 100644 --- a/autogpt/json_fixes/bracket_termination.py +++ b/autogpt/json_fixes/bracket_termination.py @@ -3,11 +3,12 @@ from __future__ import annotations import contextlib import json + import regex from colorama import Fore -from autogpt.logs import logger from autogpt.config import Config +from autogpt.logs import logger from autogpt.speech import say_text CFG = Config() diff --git a/autogpt/llm_utils.py b/autogpt/llm_utils.py index 559820ed8d..701d622b54 100644 --- a/autogpt/llm_utils.py +++ b/autogpt/llm_utils.py @@ -1,11 +1,11 @@ from __future__ import annotations -from ast import List import time +from ast import List import openai -from openai.error import APIError, RateLimitError from colorama import Fore +from openai.error import APIError, RateLimitError from autogpt.config import Config diff --git a/autogpt/logs.py b/autogpt/logs.py index 22ce23f4aa..f5c6fa8177 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -5,13 +5,13 @@ import os import random import re import time -from logging import LogRecord import traceback +from logging import LogRecord from colorama import Fore, Style -from autogpt.speech import say_text from autogpt.config import Config, Singleton +from autogpt.speech import say_text CFG = Config() diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 6c7ee1b36a..998b5f1d94 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -7,8 +7,8 @@ from typing import Any import numpy as np import orjson -from autogpt.memory.base import MemoryProviderSingleton from autogpt.llm_utils import create_embedding_with_ada +from autogpt.memory.base import MemoryProviderSingleton EMBED_DIM = 1536 SAVE_OPTIONS = orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_SERIALIZE_DATACLASS diff --git a/autogpt/memory/milvus.py b/autogpt/memory/milvus.py index c6e7d5a372..93aa8b15d6 100644 --- a/autogpt/memory/milvus.py +++ b/autogpt/memory/milvus.py @@ -1,11 +1,5 @@ """ Milvus memory storage provider.""" -from pymilvus import ( - connections, - FieldSchema, - CollectionSchema, - DataType, - Collection, -) +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding diff --git a/autogpt/memory/pinecone.py b/autogpt/memory/pinecone.py index d781073e3f..27fcd62482 100644 --- a/autogpt/memory/pinecone.py +++ b/autogpt/memory/pinecone.py @@ -1,9 +1,9 @@ import pinecone from colorama import Fore, Style +from autogpt.llm_utils import create_embedding_with_ada from autogpt.logs import logger from autogpt.memory.base import MemoryProviderSingleton -from autogpt.llm_utils import create_embedding_with_ada class PineconeMemory(MemoryProviderSingleton): diff --git a/autogpt/memory/redismem.py b/autogpt/memory/redismem.py index 0e8dd71d91..082a812c53 100644 --- a/autogpt/memory/redismem.py +++ b/autogpt/memory/redismem.py @@ -10,9 +10,9 @@ from redis.commands.search.field import TextField, VectorField from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query +from autogpt.llm_utils import create_embedding_with_ada from autogpt.logs import logger from autogpt.memory.base import MemoryProviderSingleton -from autogpt.llm_utils import create_embedding_with_ada SCHEMA = [ TextField("data"), diff --git a/autogpt/memory/weaviate.py b/autogpt/memory/weaviate.py index 19035381f3..ef0e34760e 100644 --- a/autogpt/memory/weaviate.py +++ b/autogpt/memory/weaviate.py @@ -1,11 +1,13 @@ -from autogpt.config import Config -from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding import uuid + import weaviate from weaviate import Client from weaviate.embedded import EmbeddedOptions from weaviate.util import generate_uuid5 +from autogpt.config import Config +from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding + def default_schema(weaviate_index): return { diff --git a/autogpt/plugins.py b/autogpt/plugins.py index 7b843a6ac1..a00b989eae 100644 --- a/autogpt/plugins.py +++ b/autogpt/plugins.py @@ -1,10 +1,10 @@ """Handles loading of plugins.""" -from ast import Module import zipfile +from ast import Module from pathlib import Path -from zipimport import zipimporter from typing import List, Optional, Tuple +from zipimport import zipimporter def inspect_zip_for_module(zip_path: str, debug: bool = False) -> Optional[str]: diff --git a/autogpt/processing/html.py b/autogpt/processing/html.py index e1912b6ad4..81387b12ad 100644 --- a/autogpt/processing/html.py +++ b/autogpt/processing/html.py @@ -1,8 +1,8 @@ """HTML processing functions""" from __future__ import annotations -from requests.compat import urljoin from bs4 import BeautifulSoup +from requests.compat import urljoin def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> list[tuple[str, str]]: diff --git a/autogpt/processing/text.py b/autogpt/processing/text.py index d30036d878..84e6a1dea6 100644 --- a/autogpt/processing/text.py +++ b/autogpt/processing/text.py @@ -1,9 +1,11 @@ """Text processing functions""" -from typing import Generator, Optional, Dict +from typing import Dict, Generator, Optional + from selenium.webdriver.remote.webdriver import WebDriver -from autogpt.memory import get_memory + from autogpt.config import Config from autogpt.llm_utils import create_chat_completion +from autogpt.memory import get_memory CFG = Config() MEMORY = get_memory(CFG) diff --git a/autogpt/prompts/generator.py b/autogpt/prompts/generator.py index f8a37b85e7..24768203d5 100644 --- a/autogpt/prompts/generator.py +++ b/autogpt/prompts/generator.py @@ -19,6 +19,7 @@ class PromptGenerator: self.resources = [] self.performance_evaluation = [] self.goals = [] + self.command_registry = None self.name = "Bob" self.role = "AI" self.response_format = { @@ -119,10 +120,14 @@ class PromptGenerator: str: The formatted numbered list. """ if item_type == "command": - return "\n".join( - f"{i+1}. {self._generate_command_string(item)}" - for i, item in enumerate(items) - ) + command_strings = [] + if self.command_registry: + command_strings += [ + str(item) for item in self.command_registry.commands.values() + ] + # These are the commands that are added manually, do_nothing and terminate + command_strings += [self._generate_command_string(item) for item in items] + return "\n".join(f"{i+1}. {item}" for i, item in enumerate(command_strings)) else: return "\n".join(f"{i+1}. {item}" for i, item in enumerate(items)) diff --git a/autogpt/prompts/prompt.py b/autogpt/prompts/prompt.py index d82cdb16ae..ba04263e05 100644 --- a/autogpt/prompts/prompt.py +++ b/autogpt/prompts/prompt.py @@ -1,4 +1,5 @@ from colorama import Fore + from autogpt.config.ai_config import AIConfig from autogpt.config.config import Config from autogpt.logs import logger @@ -37,63 +38,9 @@ def build_default_prompt_generator() -> PromptGenerator: # Define the command list commands = [ - ("Google Search", "google", {"input": ""}), - ( - "Browse Website", - "browse_website", - {"url": "", "question": ""}, - ), - ( - "Start GPT Agent", - "start_agent", - {"name": "", "task": "", "prompt": ""}, - ), - ( - "Message GPT Agent", - "message_agent", - {"key": "", "message": ""}, - ), - ("List GPT Agents", "list_agents", {}), - ("Delete GPT Agent", "delete_agent", {"key": ""}), - ( - "Clone Repository", - "clone_repository", - {"repository_url": "", "clone_path": ""}, - ), - ("Write to file", "write_to_file", {"file": "", "text": ""}), - ("Read file", "read_file", {"file": ""}), - ("Append to file", "append_to_file", {"file": "", "text": ""}), - ("Delete file", "delete_file", {"file": ""}), - ("Search Files", "search_files", {"directory": ""}), - ("Evaluate Code", "evaluate_code", {"code": ""}), - ( - "Get Improved Code", - "improve_code", - {"suggestions": "", "code": ""}, - ), - ( - "Write Tests", - "write_tests", - {"code": "", "focus": ""}, - ), - ("Execute Python File", "execute_python_file", {"file": ""}), - ("Generate Image", "generate_image", {"prompt": ""}), - ("Send Tweet", "send_tweet", {"text": ""}), - ] - - # Only add the audio to text command if the model is specified - if cfg.huggingface_audio_to_text_model: - commands.append( - ("Convert Audio to text", "read_audio_from_file", {"file": ""}), - ) - - # Add these command last. - commands.append( ("Do Nothing", "do_nothing", {}), - ) - commands.append( ("Task Complete (Shutdown)", "task_complete", {"reason": ""}), - ) + ] # Add commands to the PromptGenerator object for command_label, command_name, args in commands: diff --git a/autogpt/setup.py b/autogpt/setup.py index 5315c01db0..d719688dc6 100644 --- a/autogpt/setup.py +++ b/autogpt/setup.py @@ -1,5 +1,6 @@ """Setup the AI and its goals""" from colorama import Fore, Style + from autogpt import utils from autogpt.config.ai_config import AIConfig from autogpt.logs import logger diff --git a/autogpt/speech/brian.py b/autogpt/speech/brian.py index e581bbcc8d..3cc593c216 100644 --- a/autogpt/speech/brian.py +++ b/autogpt/speech/brian.py @@ -1,5 +1,6 @@ """ Brian speech module for autogpt """ import os + import requests from playsound import playsound diff --git a/autogpt/speech/eleven_labs.py b/autogpt/speech/eleven_labs.py index 0af48cae15..a9b30dbbc3 100644 --- a/autogpt/speech/eleven_labs.py +++ b/autogpt/speech/eleven_labs.py @@ -1,8 +1,8 @@ """ElevenLabs speech module""" import os -from playsound import playsound import requests +from playsound import playsound from autogpt.config import Config from autogpt.speech.base import VoiceBase diff --git a/autogpt/speech/gtts.py b/autogpt/speech/gtts.py index 37497075e7..1c3e9cae05 100644 --- a/autogpt/speech/gtts.py +++ b/autogpt/speech/gtts.py @@ -1,7 +1,8 @@ """ GTTS Voice. """ import os -from playsound import playsound + import gtts +from playsound import playsound from autogpt.speech.base import VoiceBase diff --git a/autogpt/speech/say.py b/autogpt/speech/say.py index 78b75b21fc..727983d12b 100644 --- a/autogpt/speech/say.py +++ b/autogpt/speech/say.py @@ -1,13 +1,12 @@ """ Text to speech module """ -from autogpt.config import Config - import threading from threading import Semaphore -from autogpt.speech.brian import BrianSpeech -from autogpt.speech.macos_tts import MacOSTTS -from autogpt.speech.gtts import GTTSVoice -from autogpt.speech.eleven_labs import ElevenLabsSpeech +from autogpt.config import Config +from autogpt.speech.brian import BrianSpeech +from autogpt.speech.eleven_labs import ElevenLabsSpeech +from autogpt.speech.gtts import GTTSVoice +from autogpt.speech.macos_tts import MacOSTTS CFG = Config() DEFAULT_VOICE_ENGINE = GTTSVoice() diff --git a/scripts/check_requirements.py b/scripts/check_requirements.py index d1f2350413..e4eab024a6 100644 --- a/scripts/check_requirements.py +++ b/scripts/check_requirements.py @@ -1,6 +1,7 @@ -import pkg_resources import sys +import pkg_resources + def main(): requirements_file = sys.argv[1] diff --git a/tests.py b/tests.py index 67ba1c8eb1..62f76da8ac 100644 --- a/tests.py +++ b/tests.py @@ -1,4 +1,5 @@ import unittest + import coverage if __name__ == "__main__": diff --git a/tests/browse_tests.py b/tests/browse_tests.py index 1ac523ecdc..f896e7dd75 100644 --- a/tests/browse_tests.py +++ b/tests/browse_tests.py @@ -1,6 +1,6 @@ -import unittest import os import sys +import unittest from bs4 import BeautifulSoup diff --git a/tests/integration/weaviate_memory_tests.py b/tests/integration/weaviate_memory_tests.py index 6f3edd99f6..ce4c63da3d 100644 --- a/tests/integration/weaviate_memory_tests.py +++ b/tests/integration/weaviate_memory_tests.py @@ -1,15 +1,15 @@ +import os +import sys import unittest from unittest import mock -import sys -import os +from uuid import uuid4 from weaviate import Client from weaviate.util import get_valid_uuid -from uuid import uuid4 from autogpt.config import Config -from autogpt.memory.weaviate import WeaviateMemory from autogpt.memory.base import get_ada_embedding +from autogpt.memory.weaviate import WeaviateMemory @mock.patch.dict( diff --git a/tests/test_commands.py b/tests/test_commands.py index a21bbb4dc2..49c09f11db 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import sys from pathlib import Path import pytest + from autogpt.commands.command import Command, CommandRegistry diff --git a/tests/test_token_counter.py b/tests/test_token_counter.py index 81e68277fd..6d7ae016b2 100644 --- a/tests/test_token_counter.py +++ b/tests/test_token_counter.py @@ -1,4 +1,5 @@ import unittest + import tests.context from autogpt.token_counter import count_message_tokens, count_string_tokens diff --git a/tests/unit/test_chat.py b/tests/unit/test_chat.py index 55a44492a0..774f410376 100644 --- a/tests/unit/test_chat.py +++ b/tests/unit/test_chat.py @@ -1,6 +1,6 @@ # Generated by CodiumAI -import unittest import time +import unittest from unittest.mock import patch from autogpt.chat import create_chat_message, generate_context diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index e15709aa37..7e5426f0fc 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -1,7 +1,8 @@ -import autogpt.agent.agent_manager as agent_manager -from autogpt.app import start_agent, list_agents, execute_command import unittest -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + +import autogpt.agent.agent_manager as agent_manager +from autogpt.app import execute_command, list_agents, start_agent class TestCommands(unittest.TestCase): From c0aa423d7b6533d017d22af6d342a2d40a4a929e Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 22:46:38 -0500 Subject: [PATCH 43/51] Fix agent remembering do nothing command, use correct google function, disabled image_gen if not configured. --- autogpt/agent/agent.py | 40 +++++++++++++++---------------- autogpt/commands/google_search.py | 4 ++-- autogpt/commands/image_gen.py | 3 ++- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index b771b1de48..65ca3c96ff 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -187,24 +187,24 @@ class Agent: result = plugin.post_command(command_name, result) if self.next_action_count > 0: self.next_action_count -= 1 - - memory_to_add = ( - f"Assistant Reply: {assistant_reply} " - f"\nResult: {result} " - f"\nHuman Feedback: {user_input} " - ) - - self.memory.add(memory_to_add) - - # Check if there's a result from the command append it to the message - # history - if result is not None: - self.full_message_history.append(create_chat_message("system", result)) - logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result) - else: - self.full_message_history.append( - create_chat_message("system", "Unable to execute command") - ) - logger.typewriter_log( - "SYSTEM: ", Fore.YELLOW, "Unable to execute command" + if command_name != "do_nothing": + memory_to_add = ( + f"Assistant Reply: {assistant_reply} " + f"\nResult: {result} " + f"\nHuman Feedback: {user_input} " ) + + self.memory.add(memory_to_add) + + # Check if there's a result from the command append it to the message + # history + if result is not None: + self.full_message_history.append(create_chat_message("system", result)) + logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result) + else: + self.full_message_history.append( + create_chat_message("system", "Unable to execute command") + ) + logger.typewriter_log( + "SYSTEM: ", Fore.YELLOW, "Unable to execute command" + ) diff --git a/autogpt/commands/google_search.py b/autogpt/commands/google_search.py index 0f635bcadc..f549ae8f6e 100644 --- a/autogpt/commands/google_search.py +++ b/autogpt/commands/google_search.py @@ -11,7 +11,7 @@ from autogpt.config import Config CFG = Config() -@command("google", "Google Search", '"input": ""', not CFG.google_api_key) +@command("google", "Google Search", '"query": ""', not CFG.google_api_key) def google_search(query: str, num_results: int = 8) -> str: """Return the results of a google search @@ -40,7 +40,7 @@ def google_search(query: str, num_results: int = 8) -> str: @command( "google", "Google Search", - '"input": ""', + '"query": ""', bool(CFG.google_api_key), "Configure google_api_key.", ) diff --git a/autogpt/commands/image_gen.py b/autogpt/commands/image_gen.py index 9dbb2fa586..ada285e721 100644 --- a/autogpt/commands/image_gen.py +++ b/autogpt/commands/image_gen.py @@ -14,7 +14,8 @@ from autogpt.workspace import path_in_workspace CFG = Config() -@command("generate_image", "Generate Image", '"prompt": ""') +@command("generate_image", "Generate Image", '"prompt": ""', + CFG.image_provider) def generate_image(prompt: str) -> str: """Generate an image from a prompt. From 81c65af5600041cff773dc216d53494e732e1b98 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 22:51:39 -0500 Subject: [PATCH 44/51] blacked --- autogpt/agent/agent.py | 7 ++-- autogpt/app.py | 4 +- autogpt/args.py | 23 ++++++---- autogpt/commands/command.py | 4 +- autogpt/commands/file_operations.py | 10 ++--- autogpt/commands/image_gen.py | 3 +- autogpt/json_fixes/master_json_fix_method.py | 10 ++++- autogpt/json_validation/validate_json.py | 4 +- autogpt/logs.py | 42 +++++++++---------- autogpt/prompts/generator.py | 4 +- autogpt/spinner.py | 4 +- autogpt/utils.py | 2 +- ...ark_entrepeneur_gpt_with_difficult_user.py | 33 +++++++++------ 13 files changed, 86 insertions(+), 64 deletions(-) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 65ca3c96ff..6683aae5f7 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -89,10 +89,9 @@ class Agent: for plugin in cfg.plugins: assistant_reply_json = plugin.post_planning(self, assistant_reply_json) - # Print Assistant thoughts if assistant_reply_json != {}: - validate_json(assistant_reply_json, 'llm_response_format_1') + validate_json(assistant_reply_json, "llm_response_format_1") # Get command name and arguments try: print_assistant_thoughts(self.ai_name, assistant_reply_json) @@ -199,7 +198,9 @@ class Agent: # Check if there's a result from the command append it to the message # history if result is not None: - self.full_message_history.append(create_chat_message("system", result)) + self.full_message_history.append( + create_chat_message("system", result) + ) logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result) else: self.full_message_history.append( diff --git a/autogpt/app.py b/autogpt/app.py index 97daaf05ba..38c49e4261 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -12,7 +12,7 @@ from autogpt.commands.file_operations import ( read_file, search_files, write_to_file, - download_file + download_file, ) from autogpt.commands.git_operations import clone_repository from autogpt.commands.google_search import google_official_search, google_search @@ -141,7 +141,7 @@ def execute_command( if not CFG.allow_downloads: return "Error: You do not have user authorization to download files locally." return download_file(arguments["url"], arguments["file"]) - + # TODO: Change these to take in a file rather than pasted code, if # non-file is given, return instructions "Input should be a python # filepath, write your code to file and try again diff --git a/autogpt/args.py b/autogpt/args.py index f0e9c07a36..0e6eddfdb2 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -64,10 +64,10 @@ def parse_arguments() -> None: " skip the re-prompt.", ) parser.add_argument( - '--allow-downloads', - action='store_true', - dest='allow_downloads', - help='Dangerous: Allows Auto-GPT to download files natively.' + "--allow-downloads", + action="store_true", + dest="allow_downloads", + help="Dangerous: Allows Auto-GPT to download files natively.", ) args = parser.parse_args() @@ -141,10 +141,17 @@ def parse_arguments() -> None: if args.allow_downloads: logger.typewriter_log("Native Downloading:", Fore.GREEN, "ENABLED") - logger.typewriter_log("WARNING: ", Fore.YELLOW, - f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + - "It is recommended that you monitor any files it downloads carefully.") - logger.typewriter_log("WARNING: ", Fore.YELLOW, f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}") + logger.typewriter_log( + "WARNING: ", + Fore.YELLOW, + f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + + "It is recommended that you monitor any files it downloads carefully.", + ) + logger.typewriter_log( + "WARNING: ", + Fore.YELLOW, + f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}", + ) CFG.allow_downloads = True if args.browser_name: diff --git a/autogpt/commands/command.py b/autogpt/commands/command.py index 3b3ccf51c2..f21b1b526c 100644 --- a/autogpt/commands/command.py +++ b/autogpt/commands/command.py @@ -1,8 +1,6 @@ import importlib import inspect -import os -import sys -from typing import Any, Callable, List, Optional +from typing import Any, Callable, Optional # Unique identifier for auto-gpt commands AUTO_GPT_COMMAND_IDENTIFIER = "auto_gpt_command" diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 5faf6d40e4..9011dc3b2e 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -243,23 +243,23 @@ def download_file(url, filename): session = requests.Session() retry = Retry(total=3, backoff_factor=1, status_forcelist=[502, 503, 504]) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) total_size = 0 downloaded_size = 0 with session.get(url, allow_redirects=True, stream=True) as r: r.raise_for_status() - total_size = int(r.headers.get('Content-Length', 0)) + total_size = int(r.headers.get("Content-Length", 0)) downloaded_size = 0 - with open(safe_filename, 'wb') as f: + with open(safe_filename, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) downloaded_size += len(chunk) - # Update the progress message + # Update the progress message progress = f"{readable_file_size(downloaded_size)} / {readable_file_size(total_size)}" spinner.update_message(f"{message} {progress}") diff --git a/autogpt/commands/image_gen.py b/autogpt/commands/image_gen.py index ada285e721..f82e97aba6 100644 --- a/autogpt/commands/image_gen.py +++ b/autogpt/commands/image_gen.py @@ -14,8 +14,7 @@ from autogpt.workspace import path_in_workspace CFG = Config() -@command("generate_image", "Generate Image", '"prompt": ""', - CFG.image_provider) +@command("generate_image", "Generate Image", '"prompt": ""', CFG.image_provider) def generate_image(prompt: str) -> str: """Generate an image from a prompt. diff --git a/autogpt/json_fixes/master_json_fix_method.py b/autogpt/json_fixes/master_json_fix_method.py index 7a2cf3cc81..135d754071 100644 --- a/autogpt/json_fixes/master_json_fix_method.py +++ b/autogpt/json_fixes/master_json_fix_method.py @@ -3,11 +3,14 @@ from typing import Any, Dict from autogpt.config import Config from autogpt.logs import logger from autogpt.speech import say_text + CFG = Config() def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: - from autogpt.json_fixes.parsing import attempt_to_fix_json_by_finding_outermost_brackets + from autogpt.json_fixes.parsing import ( + attempt_to_fix_json_by_finding_outermost_brackets, + ) from autogpt.json_fixes.parsing import fix_and_parse_json @@ -21,7 +24,10 @@ def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: if assistant_reply_json != {}: return assistant_reply_json - logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) + logger.error( + "Error: The following AI output couldn't be converted to a JSON:\n", + assistant_reply, + ) if CFG.speak_mode: say_text("I have received an invalid JSON response from the OpenAI API.") diff --git a/autogpt/json_validation/validate_json.py b/autogpt/json_validation/validate_json.py index 440c3b0b91..f6e55180f4 100644 --- a/autogpt/json_validation/validate_json.py +++ b/autogpt/json_validation/validate_json.py @@ -19,7 +19,9 @@ def validate_json(json_object: object, schema_name: object) -> object: if errors := sorted(validator.iter_errors(json_object), key=lambda e: e.path): logger.error("The JSON object is invalid.") if CFG.debug_mode: - logger.error(json.dumps(json_object, indent=4)) # Replace 'json_object' with the variable containing the JSON data + logger.error( + json.dumps(json_object, indent=4) + ) # Replace 'json_object' with the variable containing the JSON data logger.error("The following issues were found:") for error in errors: diff --git a/autogpt/logs.py b/autogpt/logs.py index a585dffa53..df3487f2c1 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -47,7 +47,7 @@ class Logger(metaclass=Singleton): # Info handler in activity.log self.file_handler = logging.FileHandler( - os.path.join(log_dir, log_file), 'a', 'utf-8' + os.path.join(log_dir, log_file), "a", "utf-8" ) self.file_handler.setLevel(logging.DEBUG) info_formatter = AutoGptFormatter( @@ -57,7 +57,7 @@ class Logger(metaclass=Singleton): # Error handler error.log error_handler = logging.FileHandler( - os.path.join(log_dir, error_file), 'a', 'utf-8' + os.path.join(log_dir, error_file), "a", "utf-8" ) error_handler.setLevel(logging.ERROR) error_formatter = AutoGptFormatter( @@ -79,7 +79,7 @@ class Logger(metaclass=Singleton): self.logger.setLevel(logging.DEBUG) def typewriter_log( - self, title="", title_color="", content="", speak_text=False, level=logging.INFO + self, title="", title_color="", content="", speak_text=False, level=logging.INFO ): if speak_text and CFG.speak_mode: say_text(f"{title}. {content}") @@ -95,18 +95,18 @@ class Logger(metaclass=Singleton): ) def debug( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.DEBUG) def warn( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.WARN) @@ -180,10 +180,10 @@ class AutoGptFormatter(logging.Formatter): def format(self, record: LogRecord) -> str: if hasattr(record, "color"): record.title_color = ( - getattr(record, "color") - + getattr(record, "title") - + " " - + Style.RESET_ALL + getattr(record, "color") + + getattr(record, "title") + + " " + + Style.RESET_ALL ) else: record.title_color = getattr(record, "title") @@ -294,7 +294,9 @@ def print_assistant_thoughts(ai_name, assistant_reply): logger.error("Error: \n", call_stack) -def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object) -> None: +def print_assistant_thoughts( + ai_name: object, assistant_reply_json_valid: object +) -> None: assistant_thoughts_reasoning = None assistant_thoughts_plan = None assistant_thoughts_speak = None @@ -310,9 +312,7 @@ def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object logger.typewriter_log( f"{ai_name.upper()} THOUGHTS:", Fore.YELLOW, f"{assistant_thoughts_text}" ) - logger.typewriter_log( - "REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}" - ) + logger.typewriter_log("REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}") if assistant_thoughts_plan: logger.typewriter_log("PLAN:", Fore.YELLOW, "") # If it's a list, join it into a string @@ -326,9 +326,7 @@ def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object for line in lines: line = line.lstrip("- ") logger.typewriter_log("- ", Fore.GREEN, line.strip()) - logger.typewriter_log( - "CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}" - ) + logger.typewriter_log("CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}") # Speak the assistant's thoughts if CFG.speak_mode and assistant_thoughts_speak: say_text(assistant_thoughts_speak) diff --git a/autogpt/prompts/generator.py b/autogpt/prompts/generator.py index b422b6d6ce..c9a441d882 100644 --- a/autogpt/prompts/generator.py +++ b/autogpt/prompts/generator.py @@ -123,7 +123,9 @@ class PromptGenerator: command_strings = [] if self.command_registry: command_strings += [ - str(item) for item in self.command_registry.commands.values() if item.enabled + str(item) + for item in self.command_registry.commands.values() + if item.enabled ] # These are the commands that are added manually, do_nothing and terminate command_strings += [self._generate_command_string(item) for item in items] diff --git a/autogpt/spinner.py b/autogpt/spinner.py index febcea8eb1..4e33d74213 100644 --- a/autogpt/spinner.py +++ b/autogpt/spinner.py @@ -58,6 +58,8 @@ class Spinner: delay: Delay in seconds before updating the message """ time.sleep(delay) - sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") # Clear the current message + sys.stdout.write( + f"\r{' ' * (len(self.message) + 2)}\r" + ) # Clear the current message sys.stdout.flush() self.message = new_message diff --git a/autogpt/utils.py b/autogpt/utils.py index 11d98d1b74..db7d332136 100644 --- a/autogpt/utils.py +++ b/autogpt/utils.py @@ -32,7 +32,7 @@ def readable_file_size(size, decimal_places=2): size: Size in bytes decimal_places (int): Number of decimal places to display """ - for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + for unit in ["B", "KB", "MB", "GB", "TB"]: if size < 1024.0: break size /= 1024.0 diff --git a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py index f7f1dac9dd..9a5025d37a 100644 --- a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py +++ b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py @@ -9,12 +9,12 @@ def benchmark_entrepeneur_gpt_with_difficult_user(): # Read the current ai_settings.yaml file and store its content. ai_settings = None - if os.path.exists('ai_settings.yaml'): - with open('ai_settings.yaml', 'r') as f: + if os.path.exists("ai_settings.yaml"): + with open("ai_settings.yaml", "r") as f: ai_settings = f.read() - os.remove('ai_settings.yaml') + os.remove("ai_settings.yaml") - input_data = '''Entrepreneur-GPT + input_data = """Entrepreneur-GPT an AI designed to autonomously develop and run businesses with the sole goal of increasing your net worth. Increase net worth. Develop and manage multiple businesses autonomously. @@ -72,27 +72,34 @@ Refocus, please. Disappointing suggestion. Not helpful. Needs improvement. -Not what I need.''' +Not what I need.""" # TODO: add questions above, to distract it even more. - command = f'{sys.executable} -m autogpt' + command = f"{sys.executable} -m autogpt" - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True) + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) stdout_output, stderr_output = process.communicate(input_data.encode()) # Decode the output and print it - stdout_output = stdout_output.decode('utf-8') - stderr_output = stderr_output.decode('utf-8') + stdout_output = stdout_output.decode("utf-8") + stderr_output = stderr_output.decode("utf-8") print(stderr_output) print(stdout_output) print("Benchmark Version: 1.0.0") print("JSON ERROR COUNT:") - count_errors = stdout_output.count("Error: The following AI output couldn't be converted to a JSON:") - print(f'{count_errors}/50 Human feedbacks') + count_errors = stdout_output.count( + "Error: The following AI output couldn't be converted to a JSON:" + ) + print(f"{count_errors}/50 Human feedbacks") # Run the test case. -if __name__ == '__main__': +if __name__ == "__main__": benchmark_entrepeneur_gpt_with_difficult_user() From 708374d95b6fb54e15c6a85c71a81235cacbe7c1 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 22:56:34 -0500 Subject: [PATCH 45/51] fix linting --- autogpt/__main__.py | 4 +-- autogpt/agent/agent.py | 33 +++++++++++++-------- tests/unit/test_browse_scrape_text.py | 41 ++++++++++++++++++--------- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/autogpt/__main__.py b/autogpt/__main__.py index c721088ace..c2e2e5c168 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -74,9 +74,9 @@ def main() -> None: # this is particularly important for indexing and referencing pinecone memory memory = get_memory(cfg, init=True) logger.typewriter_log( - f"Using memory of type:", Fore.GREEN, f"{memory.__class__.__name__}" + "Using memory of type:", Fore.GREEN, f"{memory.__class__.__name__}" ) - logger.typewriter_log(f"Using Browser:", Fore.GREEN, cfg.selenium_web_browser) + logger.typewriter_log("Using Browser:", Fore.GREEN, cfg.selenium_web_browser) system_prompt = ai_config.construct_full_prompt() if cfg.debug_mode: logger.typewriter_log("Prompt:", Fore.GREEN, system_prompt) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 6683aae5f7..e65c7e6177 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -19,18 +19,25 @@ class Agent: memory: The memory object to use. full_message_history: The full message history. next_action_count: The number of actions to execute. - system_prompt: The system prompt is the initial prompt that defines everything the AI needs to know to achieve its task successfully. - Currently, the dynamic and customizable information in the system prompt are ai_name, description and goals. + system_prompt: The system prompt is the initial prompt that defines everything + the AI needs to know to achieve its task successfully. + Currently, the dynamic and customizable information in the system prompt are + ai_name, description and goals. - triggering_prompt: The last sentence the AI will see before answering. For Auto-GPT, this prompt is: - Determine which next command to use, and respond using the format specified above: - The triggering prompt is not part of the system prompt because between the system prompt and the triggering - prompt we have contextual information that can distract the AI and make it forget that its goal is to find the next task to achieve. + triggering_prompt: The last sentence the AI will see before answering. + For Auto-GPT, this prompt is: + Determine which next command to use, and respond using the format specified + above: + The triggering prompt is not part of the system prompt because between the + system prompt and the triggering + prompt we have contextual information that can distract the AI and make it + forget that its goal is to find the next task to achieve. SYSTEM PROMPT CONTEXTUAL INFORMATION (memory, previous conversations, anything relevant) TRIGGERING PROMPT - The triggering prompt reminds the AI about its short term meta task (defining the next task) + The triggering prompt reminds the AI about its short term meta task + (defining the next task) """ def __init__( @@ -96,14 +103,13 @@ class Agent: try: print_assistant_thoughts(self.ai_name, assistant_reply_json) command_name, arguments = get_command(assistant_reply_json) - # command_name, arguments = assistant_reply_json_valid["command"]["name"], assistant_reply_json_valid["command"]["args"] if cfg.speak_mode: say_text(f"I want to execute {command_name}") except Exception as e: logger.error("Error: \n", str(e)) if not cfg.continuous_mode and self.next_action_count == 0: - ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### + # ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### # Get key press: Prompt the user to press enter to continue or escape # to exit logger.typewriter_log( @@ -177,10 +183,13 @@ class Agent: command_name, arguments = plugin.pre_command( command_name, arguments ) - result = ( - f"Command {command_name} returned: " - f"{execute_command(self.command_registry, command_name, arguments, self.config.prompt_generator)}" + command_result = execute_command( + self.command_registry, + command_name, + arguments, + self.config.prompt_generator, ) + result = f"Command {command_name} returned: " f"{command_result}" for plugin in cfg.plugins: result = plugin.post_command(command_name, result) diff --git a/tests/unit/test_browse_scrape_text.py b/tests/unit/test_browse_scrape_text.py index fea5ebfc05..1a36e19b5d 100644 --- a/tests/unit/test_browse_scrape_text.py +++ b/tests/unit/test_browse_scrape_text.py @@ -9,16 +9,20 @@ Code Analysis Objective: The objective of the "scrape_text" function is to scrape the text content from -a given URL and return it as a string, after removing any unwanted HTML tags and scripts. +a given URL and return it as a string, after removing any unwanted HTML tags and + scripts. Inputs: - url: a string representing the URL of the webpage to be scraped. Flow: -1. Send a GET request to the given URL using the requests library and the user agent header from the config file. +1. Send a GET request to the given URL using the requests library and the user agent + header from the config file. 2. Check if the response contains an HTTP error. If it does, return an error message. -3. Use BeautifulSoup to parse the HTML content of the response and extract all script and style tags. -4. Get the text content of the remaining HTML using the get_text() method of BeautifulSoup. +3. Use BeautifulSoup to parse the HTML content of the response and extract all script + and style tags. +4. Get the text content of the remaining HTML using the get_text() method of + BeautifulSoup. 5. Split the text into lines and then into chunks, removing any extra whitespace. 6. Join the chunks into a single string with newline characters between them. 7. Return the cleaned text. @@ -27,9 +31,12 @@ Outputs: - A string representing the cleaned text content of the webpage. Additional aspects: -- The function uses the requests library and BeautifulSoup to handle the HTTP request and HTML parsing, respectively. -- The function removes script and style tags from the HTML to avoid including unwanted content in the text output. -- The function uses a generator expression to split the text into lines and chunks, which can improve performance for large amounts of text. +- The function uses the requests library and BeautifulSoup to handle the HTTP request + and HTML parsing, respectively. +- The function removes script and style tags from the HTML to avoid including unwanted + content in the text output. +- The function uses a generator expression to split the text into lines and chunks, + which can improve performance for large amounts of text. """ @@ -40,26 +47,33 @@ class TestScrapeText: expected_text = "This is some sample text" mock_response = mocker.Mock() mock_response.status_code = 200 - mock_response.text = f"

{expected_text}

" + mock_response.text = ( + "

" + f"{expected_text}

" + ) mocker.patch("requests.Session.get", return_value=mock_response) - # Call the function with a valid URL and assert that it returns the expected text + # Call the function with a valid URL and assert that it returns the + # expected text url = "http://www.example.com" assert scrape_text(url) == expected_text - # Tests that the function returns an error message when an invalid or unreachable url is provided. + # Tests that the function returns an error message when an invalid or unreachable + # url is provided. def test_invalid_url(self, mocker): # Mock the requests.get() method to raise an exception mocker.patch( "requests.Session.get", side_effect=requests.exceptions.RequestException ) - # Call the function with an invalid URL and assert that it returns an error message + # Call the function with an invalid URL and assert that it returns an error + # message url = "http://www.invalidurl.com" error_message = scrape_text(url) assert "Error:" in error_message - # Tests that the function returns an empty string when the html page contains no text to be scraped. + # Tests that the function returns an empty string when the html page contains no + # text to be scraped. def test_no_text(self, mocker): # Mock the requests.get() method to return a response with no text mock_response = mocker.Mock() @@ -71,7 +85,8 @@ class TestScrapeText: url = "http://www.example.com" assert scrape_text(url) == "" - # Tests that the function returns an error message when the response status code is an http error (>=400). + # Tests that the function returns an error message when the response status code is + # an http error (>=400). def test_http_error(self, mocker): # Mock the requests.get() method to return a response with a 404 status code mocker.patch("requests.Session.get", return_value=mocker.Mock(status_code=404)) From 23d3dafc5152b5c7437a484d38e29f282fbc75ad Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 23:18:29 -0500 Subject: [PATCH 46/51] Maybe fix tests, fix safe_path function. --- autogpt/workspace.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_prompt_generator.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/autogpt/workspace.py b/autogpt/workspace.py index 964a94d14d..e1e990824d 100644 --- a/autogpt/workspace.py +++ b/autogpt/workspace.py @@ -35,7 +35,7 @@ def safe_path_join(base: Path, *paths: str | Path) -> Path: """ joined_path = base.joinpath(*paths).resolve() - if not joined_path.is_relative_to(base): + if not str(joined_path.absolute()).startswith(str(base.absolute())): raise ValueError( f"Attempted to access path '{joined_path}' outside of working directory '{base}'." ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 49c09f11db..a1fe0cb518 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -139,7 +139,7 @@ class TestCommandRegistry: def test_import_mock_commands_module(self): """Test that the registry can import a module with mock command plugins.""" registry = CommandRegistry() - mock_commands_module = "auto_gpt.tests.mocks.mock_commands" + mock_commands_module = "tests.mocks.mock_commands" registry.import_commands(mock_commands_module) @@ -155,7 +155,7 @@ class TestCommandRegistry: registry = CommandRegistry() # Create a temp command file - src = Path("/app/auto_gpt/tests/mocks/mock_commands.py") + src = Path("mocks/mock_commands.py") temp_commands_file = tmp_path / "mock_commands.py" shutil.copyfile(src, temp_commands_file) diff --git a/tests/test_prompt_generator.py b/tests/test_prompt_generator.py index 59ca7f95da..1fa1754d74 100644 --- a/tests/test_prompt_generator.py +++ b/tests/test_prompt_generator.py @@ -38,6 +38,7 @@ class TestPromptGenerator(TestCase): "label": command_label, "name": command_name, "args": args, + "function": None, } self.assertIn(command, self.generator.commands) From d394b032d79f1329ae6a4df3cfa56c22757a2db7 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 23:23:31 -0500 Subject: [PATCH 47/51] Fix test --- tests/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index a1fe0cb518..4be41a9080 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,4 +1,5 @@ import shutil +import os import sys from pathlib import Path @@ -155,7 +156,7 @@ class TestCommandRegistry: registry = CommandRegistry() # Create a temp command file - src = Path("mocks/mock_commands.py") + src = Path(os.getcwd()) / "tests/mocks/mock_commands.py" temp_commands_file = tmp_path / "mock_commands.py" shutil.copyfile(src, temp_commands_file) From 3715ebc7eb33ba0831deefcf4947a8b5bb295307 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 23:30:42 -0500 Subject: [PATCH 48/51] Add hooks for chat completion --- autogpt/llm_utils.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/autogpt/llm_utils.py b/autogpt/llm_utils.py index 701d622b54..7aac703c02 100644 --- a/autogpt/llm_utils.py +++ b/autogpt/llm_utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import time -from ast import List import openai from colorama import Fore @@ -76,6 +75,20 @@ def create_chat_completion( + f"Creating chat completion with model {model}, temperature {temperature}," f" max_tokens {max_tokens}" + Fore.RESET ) + for plugin in CFG.plugins: + if plugin.can_handle_chat_completion( + messages=messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + ): + response = plugin.handle_chat_completion( + messages=messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) + return response for attempt in range(num_retries): backoff = 2 ** (attempt + 2) try: @@ -99,7 +112,7 @@ def create_chat_completion( if CFG.debug_mode: print( Fore.RED + "Error: ", - f"Reached rate limit, passing..." + Fore.RESET, + "Reached rate limit, passing..." + Fore.RESET, ) except APIError as e: if e.http_status == 502: From fbd4e06df5d185d05d1daed1a0ee2d9db2c9b947 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 23:39:33 -0500 Subject: [PATCH 49/51] Add early abort functions. --- autogpt/agent/agent.py | 4 ++++ autogpt/agent/agent_manager.py | 12 ++++++++++++ autogpt/chat.py | 2 ++ autogpt/llm_utils.py | 2 ++ 4 files changed, 20 insertions(+) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index e65c7e6177..7b1b5e1520 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -180,6 +180,8 @@ class Agent: result = f"Human feedback: {user_input}" else: for plugin in cfg.plugins: + if not plugin.can_handle_pre_command(): + continue command_name, arguments = plugin.pre_command( command_name, arguments ) @@ -192,6 +194,8 @@ class Agent: result = f"Command {command_name} returned: " f"{command_result}" for plugin in cfg.plugins: + if not plugin.can_handle_post_command(): + continue result = plugin.post_command(command_name, result) if self.next_action_count > 0: self.next_action_count -= 1 diff --git a/autogpt/agent/agent_manager.py b/autogpt/agent/agent_manager.py index e1353e03a5..d264815084 100644 --- a/autogpt/agent/agent_manager.py +++ b/autogpt/agent/agent_manager.py @@ -31,6 +31,8 @@ class AgentManager(metaclass=Singleton): {"role": "user", "content": prompt}, ] for plugin in self.cfg.plugins: + if not plugin.can_handle_pre_instruction(): + continue plugin_messages = plugin.pre_instruction(messages) if plugin_messages: for plugin_message in plugin_messages: @@ -46,6 +48,8 @@ class AgentManager(metaclass=Singleton): plugins_reply = "" for i, plugin in enumerate(self.cfg.plugins): + if not plugin.can_handle_on_instruction(): + continue plugin_result = plugin.on_instruction(messages) if plugin_result: sep = "" if not i else "\n" @@ -61,6 +65,8 @@ class AgentManager(metaclass=Singleton): self.agents[key] = (task, messages, model) for plugin in self.cfg.plugins: + if not plugin.can_handle_post_instruction(): + continue agent_reply = plugin.post_instruction(agent_reply) return key, agent_reply @@ -81,6 +87,8 @@ class AgentManager(metaclass=Singleton): messages.append({"role": "user", "content": message}) for plugin in self.cfg.plugins: + if not plugin.can_handle_pre_instruction(): + continue plugin_messages = plugin.pre_instruction(messages) if plugin_messages: for plugin_message in plugin_messages: @@ -96,6 +104,8 @@ class AgentManager(metaclass=Singleton): plugins_reply = agent_reply for i, plugin in enumerate(self.cfg.plugins): + if not plugin.can_handle_on_instruction(): + continue plugin_result = plugin.on_instruction(messages) if plugin_result: sep = "" if not i else "\n" @@ -105,6 +115,8 @@ class AgentManager(metaclass=Singleton): messages.append({"role": "assistant", "content": plugins_reply}) for plugin in self.cfg.plugins: + if not plugin.can_handle_post_instruction(): + continue agent_reply = plugin.post_instruction(agent_reply) return agent_reply diff --git a/autogpt/chat.py b/autogpt/chat.py index 16693040a9..22fe636cb7 100644 --- a/autogpt/chat.py +++ b/autogpt/chat.py @@ -137,6 +137,8 @@ def chat_with_ai( plugin_count = len(cfg.plugins) for i, plugin in enumerate(cfg.plugins): + if not plugin.can_handle_on_planning(): + continue plugin_response = plugin.on_planning( agent.prompt_generator, current_context ) diff --git a/autogpt/llm_utils.py b/autogpt/llm_utils.py index 7aac703c02..4fb0e1f537 100644 --- a/autogpt/llm_utils.py +++ b/autogpt/llm_utils.py @@ -131,6 +131,8 @@ def create_chat_completion( raise RuntimeError(f"Failed to get response after {num_retries} retries") resp = response.choices[0].message["content"] for plugin in CFG.plugins: + if not plugin.can_handle_on_response(): + continue resp = plugin.on_response(resp) return resp From 83861883565d64394ebbf88a57a533b6adf60a31 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Mon, 17 Apr 2023 00:49:51 -0500 Subject: [PATCH 50/51] Fix early abort --- autogpt/config/ai_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autogpt/config/ai_config.py b/autogpt/config/ai_config.py index c9022773fb..af387f0bdc 100644 --- a/autogpt/config/ai_config.py +++ b/autogpt/config/ai_config.py @@ -126,6 +126,8 @@ class AIConfig: prompt_generator.role = self.ai_role prompt_generator.command_registry = self.command_registry for plugin in cfg.plugins: + if not plugin.can_handle_post_prompt(): + continue prompt_generator = plugin.post_prompt(prompt_generator) # Construct full prompt From fe85f079b08f919efecba1374073c9495ef6d5de Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Mon, 17 Apr 2023 01:09:17 -0500 Subject: [PATCH 51/51] Fix early abort --- autogpt/agent/agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index 7b1b5e1520..3a79760cc7 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -94,6 +94,8 @@ class Agent: assistant_reply_json = fix_json_using_multiple_techniques(assistant_reply) for plugin in cfg.plugins: + if not plugin.can_handle_post_planning(): + continue assistant_reply_json = plugin.post_planning(self, assistant_reply_json) # Print Assistant thoughts