From 0cf790b633b417c092f84b52715f5ec057c045aa Mon Sep 17 00:00:00 2001 From: meta-fx Date: Mon, 10 Apr 2023 20:00:43 -0500 Subject: [PATCH 01/40] Added new env variable and speech function for alternative TTS voice --- .env.template | 1 + scripts/config.py | 11 ++++++++--- scripts/speak.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.env.template b/.env.template index 01735615ca..98d2ca91e5 100644 --- a/.env.template +++ b/.env.template @@ -13,3 +13,4 @@ OPENAI_AZURE_DEPLOYMENT_ID=deployment-id-for-azure IMAGE_PROVIDER=dalle HUGGINGFACE_API_TOKEN= USE_MAC_OS_TTS=False +USE_BRIAN_TTS=False \ No newline at end of file diff --git a/scripts/config.py b/scripts/config.py index 27cc946cf0..3bf5cd9a81 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -57,7 +57,10 @@ class Config(metaclass=Singleton): self.use_mac_os_tts = False self.use_mac_os_tts = os.getenv("USE_MAC_OS_TTS") - + + self.use_brian_tts = False + self.use_brian_tts = os.getenv("USE_BRIAN_TTS") + self.google_api_key = os.getenv("GOOGLE_API_KEY") self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID") @@ -69,11 +72,13 @@ class Config(metaclass=Singleton): # User agent headers to use when browsing web # Some websites might just completely deny request with an error code if no user agent was found. - self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} + self.user_agent_header = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} self.redis_host = os.getenv("REDIS_HOST", "localhost") self.redis_port = os.getenv("REDIS_PORT", "6379") self.redis_password = os.getenv("REDIS_PASSWORD", "") - self.wipe_redis_on_start = os.getenv("WIPE_REDIS_ON_START", "True") == 'True' + self.wipe_redis_on_start = os.getenv( + "WIPE_REDIS_ON_START", "True") == 'True' self.memory_index = os.getenv("MEMORY_INDEX", 'auto-gpt') # Note that indexes must be created on db 0 in redis, this is not configureable. diff --git a/scripts/speak.py b/scripts/speak.py index 5d1e153c26..ebaae8d182 100644 --- a/scripts/speak.py +++ b/scripts/speak.py @@ -1,9 +1,9 @@ +import gtts import os from playsound import playsound import requests from config import Config cfg = Config() -import gtts # TODO: Nicer names for these ids @@ -14,6 +14,7 @@ tts_headers = { "xi-api-key": cfg.elevenlabs_api_key } + def eleven_labs_speech(text, voice_index=0): """Speak text using elevenlabs.io's API""" tts_url = "https://api.elevenlabs.io/v1/text-to-speech/{voice_id}".format( @@ -33,23 +34,46 @@ def eleven_labs_speech(text, voice_index=0): print("Response content:", response.content) return False + +def brian_speech(text): + """Speak text using Brian with the streamelements API""" + tts_url = f"https://api.streamelements.com/kappa/v2/speech?voice=Brian&text={text}" + response = requests.get(tts_url) + + if response.status_code == 200: + with open("speech.mp3", "wb") as f: + f.write(response.content) + playsound("speech.mp3") + os.remove("speech.mp3") + return True + else: + print("Request failed with status code:", response.status_code) + print("Response content:", response.content) + return False + + def gtts_speech(text): tts = gtts.gTTS(text) tts.save("speech.mp3") playsound("speech.mp3") os.remove("speech.mp3") + def macos_tts_speech(text): os.system(f'say "{text}"') + def say_text(text, voice_index=0): if not cfg.elevenlabs_api_key: if cfg.use_mac_os_tts == 'True': macos_tts_speech(text) + elif cfg.use_brian_tts == 'True': + success = brian_speech(text) + if not success: + gtts_speech(text) else: gtts_speech(text) else: success = eleven_labs_speech(text, voice_index) if not success: gtts_speech(text) - From 3ee62211db3003312e09ff02517b0f250d7717a6 Mon Sep 17 00:00:00 2001 From: meta-fx Date: Mon, 10 Apr 2023 20:56:27 -0500 Subject: [PATCH 02/40] Fixed formatting issues --- scripts/config.py | 8 +++----- scripts/speak.py | 9 ++------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index 3bf5cd9a81..f636da7dfd 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -72,13 +72,11 @@ class Config(metaclass=Singleton): # User agent headers to use when browsing web # Some websites might just completely deny request with an error code if no user agent was found. - self.user_agent_header = { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} + self.user_agent_header = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} self.redis_host = os.getenv("REDIS_HOST", "localhost") self.redis_port = os.getenv("REDIS_PORT", "6379") self.redis_password = os.getenv("REDIS_PASSWORD", "") - self.wipe_redis_on_start = os.getenv( - "WIPE_REDIS_ON_START", "True") == 'True' + self.wipe_redis_on_start = os.getenv("WIPE_REDIS_ON_START", "True") == 'True' self.memory_index = os.getenv("MEMORY_INDEX", 'auto-gpt') # Note that indexes must be created on db 0 in redis, this is not configureable. @@ -139,4 +137,4 @@ class Config(metaclass=Singleton): def set_debug_mode(self, value: bool): """Set the debug mode value.""" - self.debug = value + self.debug = value \ No newline at end of file diff --git a/scripts/speak.py b/scripts/speak.py index ebaae8d182..2464c62596 100644 --- a/scripts/speak.py +++ b/scripts/speak.py @@ -1,9 +1,9 @@ -import gtts import os from playsound import playsound import requests from config import Config cfg = Config() +import gtts # TODO: Nicer names for these ids @@ -14,7 +14,6 @@ tts_headers = { "xi-api-key": cfg.elevenlabs_api_key } - def eleven_labs_speech(text, voice_index=0): """Speak text using elevenlabs.io's API""" tts_url = "https://api.elevenlabs.io/v1/text-to-speech/{voice_id}".format( @@ -34,7 +33,6 @@ def eleven_labs_speech(text, voice_index=0): print("Response content:", response.content) return False - def brian_speech(text): """Speak text using Brian with the streamelements API""" tts_url = f"https://api.streamelements.com/kappa/v2/speech?voice=Brian&text={text}" @@ -51,18 +49,15 @@ def brian_speech(text): print("Response content:", response.content) return False - def gtts_speech(text): tts = gtts.gTTS(text) tts.save("speech.mp3") playsound("speech.mp3") os.remove("speech.mp3") - def macos_tts_speech(text): os.system(f'say "{text}"') - def say_text(text, voice_index=0): if not cfg.elevenlabs_api_key: if cfg.use_mac_os_tts == 'True': @@ -76,4 +71,4 @@ def say_text(text, voice_index=0): else: success = eleven_labs_speech(text, voice_index) if not success: - gtts_speech(text) + gtts_speech(text) \ No newline at end of file From 3cdde2d49cba4b43045ebef7f16236fed3a4acc9 Mon Sep 17 00:00:00 2001 From: meta-fx Date: Tue, 11 Apr 2023 08:15:58 -0500 Subject: [PATCH 03/40] Resolved conflicts in config.py and speak.py --- scripts/config.py | 7 ++---- scripts/speak.py | 60 ++++++++++++++++++++++++++++++----------------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index f636da7dfd..cdf0287c4b 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -33,7 +33,7 @@ class Config(metaclass=Singleton): def __init__(self): """Initialize the Config class""" - self.debug = False + self.debug_mode = False self.continuous_mode = False self.speak_mode = False @@ -92,9 +92,6 @@ class Config(metaclass=Singleton): """Set the speak mode value.""" self.speak_mode = value - def set_debug_mode(self, value: bool): - self.debug_mode = value - def set_fast_llm_model(self, value: str): """Set the fast LLM model value.""" self.fast_llm_model = value @@ -137,4 +134,4 @@ class Config(metaclass=Singleton): def set_debug_mode(self, value: bool): """Set the debug mode value.""" - self.debug = value \ No newline at end of file + self.debug_mode = value \ No newline at end of file diff --git a/scripts/speak.py b/scripts/speak.py index 2464c62596..bf5c6034fa 100644 --- a/scripts/speak.py +++ b/scripts/speak.py @@ -4,6 +4,8 @@ import requests from config import Config cfg = Config() import gtts +import threading +from threading import Lock, Semaphore # TODO: Nicer names for these ids @@ -14,6 +16,9 @@ tts_headers = { "xi-api-key": cfg.elevenlabs_api_key } +mutex_lock = Lock() # Ensure only one sound is played at a time +queue_semaphore = Semaphore(1) # The amount of sounds to queue before blocking the main thread + def eleven_labs_speech(text, voice_index=0): """Speak text using elevenlabs.io's API""" tts_url = "https://api.elevenlabs.io/v1/text-to-speech/{voice_id}".format( @@ -23,10 +28,11 @@ def eleven_labs_speech(text, voice_index=0): tts_url, headers=tts_headers, json=formatted_message) if response.status_code == 200: - with open("speech.mpeg", "wb") as f: - f.write(response.content) - playsound("speech.mpeg") - os.remove("speech.mpeg") + with mutex_lock: + with open("speech.mpeg", "wb") as f: + f.write(response.content) + playsound("speech.mpeg", True) + os.remove("speech.mpeg") return True else: print("Request failed with status code:", response.status_code) @@ -39,10 +45,11 @@ def brian_speech(text): response = requests.get(tts_url) if response.status_code == 200: - with open("speech.mp3", "wb") as f: - f.write(response.content) - playsound("speech.mp3") - os.remove("speech.mp3") + with mutex_lock: + with open("speech.mp3", "wb") as f: + f.write(response.content) + playsound("speech.mp3") + os.remove("speech.mp3") return True else: print("Request failed with status code:", response.status_code) @@ -51,24 +58,33 @@ def brian_speech(text): def gtts_speech(text): tts = gtts.gTTS(text) - tts.save("speech.mp3") - playsound("speech.mp3") - os.remove("speech.mp3") + with mutex_lock: + tts.save("speech.mp3") + playsound("speech.mp3", True) + os.remove("speech.mp3") def macos_tts_speech(text): os.system(f'say "{text}"') def say_text(text, voice_index=0): - if not cfg.elevenlabs_api_key: - if cfg.use_mac_os_tts == 'True': - macos_tts_speech(text) - elif cfg.use_brian_tts == 'True': - success = brian_speech(text) - if not success: + + def speak(): + if not cfg.elevenlabs_api_key: + if cfg.use_mac_os_tts == 'True': + macos_tts_speech(text) + elif cfg.use_brian_tts == 'True': + success = brian_speech(text) + if not success: + gtts_speech(text) + else: gtts_speech(text) else: - gtts_speech(text) - else: - success = eleven_labs_speech(text, voice_index) - if not success: - gtts_speech(text) \ No newline at end of file + success = eleven_labs_speech(text, voice_index) + if not success: + gtts_speech(text) + + queue_semaphore.release() + + queue_semaphore.acquire(True) + thread = threading.Thread(target=speak) + thread.start() \ No newline at end of file From 570f76bd51a2f01c78788938eadc17348bb92fe3 Mon Sep 17 00:00:00 2001 From: meta-fx Date: Tue, 11 Apr 2023 14:40:05 -0500 Subject: [PATCH 04/40] Removed trailing spaces and fixed CRLF being removed --- scripts/config.py | 4 ++-- scripts/speak.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index cdf0287c4b..c4ad3bf4eb 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -72,7 +72,7 @@ class Config(metaclass=Singleton): # User agent headers to use when browsing web # Some websites might just completely deny request with an error code if no user agent was found. - self.user_agent_header = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} + self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} self.redis_host = os.getenv("REDIS_HOST", "localhost") self.redis_port = os.getenv("REDIS_PORT", "6379") self.redis_password = os.getenv("REDIS_PASSWORD", "") @@ -134,4 +134,4 @@ class Config(metaclass=Singleton): def set_debug_mode(self, value: bool): """Set the debug mode value.""" - self.debug_mode = value \ No newline at end of file + self.debug_mode = value diff --git a/scripts/speak.py b/scripts/speak.py index bf5c6034fa..4934ecef1d 100644 --- a/scripts/speak.py +++ b/scripts/speak.py @@ -67,7 +67,7 @@ def macos_tts_speech(text): os.system(f'say "{text}"') def say_text(text, voice_index=0): - + def speak(): if not cfg.elevenlabs_api_key: if cfg.use_mac_os_tts == 'True': @@ -87,4 +87,4 @@ def say_text(text, voice_index=0): queue_semaphore.acquire(True) thread = threading.Thread(target=speak) - thread.start() \ No newline at end of file + thread.start() From 0dddc94bdac94707062b1863f3c5a72d113432ca Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:13:04 +0200 Subject: [PATCH 05/40] Add file ingestion methode in file_operations.py Add the following functions to ingest data into memory before Auto-GPT run. - split_file: given a content, split it in chunks of max_length with (or without) a specified overlap - ingest_file: read a file, use split_file to split it in chunks and load each chunk in memory - ingest_directory: ingest all files in a directory in memory --- scripts/file_operations.py | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index c6066ef930..3493c2bfa3 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -1,5 +1,10 @@ import os import os.path +from config import Config +from memory import get_memory + +cfg = Config() +memory = get_memory(cfg) # Set a dedicated folder for file I/O working_directory = "auto_gpt_workspace" @@ -20,6 +25,30 @@ def safe_join(base, *paths): return norm_new_path +def split_file(content, max_length=4000, overlap=0): + """ + Split text into chunks of a specified maximum length with a specified overlap + between chunks. + + :param text: The input text to be split into chunks + :param max_length: The maximum length of each chunk, default is 4000 (about 1k token) + :param overlap: The number of overlapping characters between chunks, default is no overlap + :return: A generator yielding chunks of text + """ + start = 0 + content_length = len(content) + + while start < content_length: + end = start + max_length + chunk = content[start:end] + yield chunk + start += max_length - overlap + if start + max_length - overlap >= content_length: + break + if end + overlap > content_length: + start = content_length - max_length + + def read_file(filename): """Read a file and return the contents""" try: @@ -31,6 +60,52 @@ def read_file(filename): return "Error: " + str(e) +def ingest_file(filename, memory, max_length=4000, overlap=200): + """ + Ingest a file by reading its content, splitting it into chunks with a specified + maximum length and overlap, and adding the chunks to the memory storage. + + :param filename: The name of the file to ingest + :param memory: An object with an add() method to store the chunks in memory + :param max_length: The maximum length of each chunk, default is 4000 + :param overlap: The number of overlapping characters between chunks, default is 200 + """ + try: + print(f"Working with file {filename}") + content = read_file(filename) + content_length = len(content) + print(f"File length: {content_length} characters") + + chunks = list(split_file(content, max_length=max_length, overlap=overlap)) + + num_chunks = len(chunks) + for i, chunk in enumerate(chunks): + print(f"Ingesting chunk {i + 1} / {num_chunks} into memory") + memory_to_add = f"Filename: {filename}\n" \ + f"Content part#{i + 1}/{num_chunks}: {chunk}" + + memory.add(memory_to_add) + + print(f"Done ingesting {num_chunks} chunks from {filename}.") + except Exception as e: + print(f"Error while ingesting file '{filename}': {str(e)}") + + +def ingest_directory(directory, memory): + """ + Ingest all files in a directory by calling the ingest_file function for each file. + + :param directory: The directory containing the files to ingest + :param memory: An object with an add() method to store the chunks in memory + """ + try: + files = search_files(directory) + for file in files: + ingest_file(file, memory) + except Exception as e: + print(f"Error while ingesting directory '{directory}': {str(e)}") + + def write_to_file(filename, text): """Write text to a file""" try: From 7975c184a55a477e884e1920ed87dc67ca4b4261 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:46:39 +0200 Subject: [PATCH 06/40] Update .gitignore add new log file to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa0dceaa9e..fc496609b4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ log.txt # Coverage reports .coverage coverage.xml -htmlcov/ \ No newline at end of file +htmlcov/ +log-ingestion.txt From c91117616f7b5e16743208b8649ce4335077915b Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:46:58 +0200 Subject: [PATCH 07/40] Update file_operations.py revert change in import as we don't need them --- scripts/file_operations.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index 3493c2bfa3..8e807bba32 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -1,10 +1,5 @@ import os import os.path -from config import Config -from memory import get_memory - -cfg = Config() -memory = get_memory(cfg) # Set a dedicated folder for file I/O working_directory = "auto_gpt_workspace" From 8faa6ef949bf7fbbb8bd875a66bfd4fd231ecebc Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:47:51 +0200 Subject: [PATCH 08/40] Create data_ingestion.py This script is use when we want to seed Auto-GPT memory with one or multiple documents. The document are read, split into chunks and store in the memory. --- scripts/data_ingestion.py | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/data_ingestion.py diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py new file mode 100644 index 0000000000..3f6d13221c --- /dev/null +++ b/scripts/data_ingestion.py @@ -0,0 +1,52 @@ +import argparse +import logging +from config import Config +from memory import get_memory +from file_operations import ingest_file, ingest_directory + +cfg = Config() + +def configure_logging(): + logging.basicConfig(filename='log-ingestion.txt', + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + datefmt='%H:%M:%S', + level=logging.DEBUG) + return logging.getLogger('AutoGPT-Ingestion') + + +def main(): + logger = configure_logging() + + parser = argparse.ArgumentParser(description="Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--file", type=str, help="The file to ingest.") + group.add_argument("--dir", type=str, help="The directory containing the files to ingest.") + parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content", default=False) + args = parser.parse_args() + + + # Initialize memory + memory = get_memory(cfg, init=args.init) + print('Using memory of type: ' + memory.__class__.__name__) + + if args.file: + try: + ingest_file(args.file, memory) + print(f"File '{args.file}' ingested successfully.") + except Exception as e: + logger.error(f"Error while ingesting file '{args.file}': {str(e)}") + print(f"Error while ingesting file '{args.file}': {str(e)}") + elif args.dir: + try: + ingest_directory(args.dir, memory) + print(f"Directory '{args.dir}' ingested successfully.") + except Exception as e: + logger.error(f"Error while ingesting directory '{args.dir}': {str(e)}") + print(f"Error while ingesting directory '{args.dir}': {str(e)}") + else: + print("Please provide either a file path (--file) or a directory name (--dir) inside the auto_gpt_workspace directory as input.") + + +if __name__ == "__main__": + main() From 4465486ea39b0bc65715e48a1c7861a565b5126f Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:18:59 +0200 Subject: [PATCH 09/40] Update file_operations.py move the search_file function inside the data_ingestion script --- scripts/file_operations.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index 8e807bba32..e664fcc91d 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -86,21 +86,6 @@ def ingest_file(filename, memory, max_length=4000, overlap=200): print(f"Error while ingesting file '{filename}': {str(e)}") -def ingest_directory(directory, memory): - """ - Ingest all files in a directory by calling the ingest_file function for each file. - - :param directory: The directory containing the files to ingest - :param memory: An object with an add() method to store the chunks in memory - """ - try: - files = search_files(directory) - for file in files: - ingest_file(file, memory) - except Exception as e: - print(f"Error while ingesting directory '{directory}': {str(e)}") - - def write_to_file(filename, text): """Write text to a file""" try: From 280647ff387bc29127b8403c7dd46f2c94d6a0b8 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:19:36 +0200 Subject: [PATCH 10/40] Update data_ingestion.py move the search_file function inside the data_ingestion script add memory initialisation argument add overlap argument add chunk max_length argument --- scripts/data_ingestion.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index 3f6d13221c..3281116629 100644 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -2,7 +2,7 @@ import argparse import logging from config import Config from memory import get_memory -from file_operations import ingest_file, ingest_directory +from file_operations import ingest_file, search_files cfg = Config() @@ -15,6 +15,21 @@ def configure_logging(): return logging.getLogger('AutoGPT-Ingestion') +def ingest_directory(directory, memory, args): + """ + Ingest all files in a directory by calling the ingest_file function for each file. + + :param directory: The directory containing the files to ingest + :param memory: An object with an add() method to store the chunks in memory + """ + try: + files = search_files(directory) + for file in files: + ingest_file(file, memory, args.max_length, args.overlap) + except Exception as e: + print(f"Error while ingesting directory '{directory}': {str(e)}") + + def main(): logger = configure_logging() @@ -22,7 +37,10 @@ def main(): group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--file", type=str, help="The file to ingest.") group.add_argument("--dir", type=str, help="The directory containing the files to ingest.") - parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content", default=False) + parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content (default: False)", default=False) + parser.add_argument("--overlap", type=int, help="The overlap size between chunks when ingesting files (default: 200)", default=200) + parser.add_argument("--max_length", type=int, help="The max_length of each chunk when ingesting files (default: 4000)", default=4000) + args = parser.parse_args() @@ -32,14 +50,14 @@ def main(): if args.file: try: - ingest_file(args.file, memory) + ingest_file(args.file, memory, args.max_length, args.overlap) print(f"File '{args.file}' ingested successfully.") except Exception as e: logger.error(f"Error while ingesting file '{args.file}': {str(e)}") print(f"Error while ingesting file '{args.file}': {str(e)}") elif args.dir: try: - ingest_directory(args.dir, memory) + ingest_directory(args.dir, memory, args) print(f"Directory '{args.dir}' ingested successfully.") except Exception as e: logger.error(f"Error while ingesting directory '{args.dir}': {str(e)}") From 65cc4f833f56000ae3928cccc3c9821fece53958 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:47:46 +0200 Subject: [PATCH 11/40] Add Memory Pre-Seeding information to readme.md Add the documentation for memory pre-seeding --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index 2900daa968..6262467d36 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Your support is greatly appreciated - [๐Ÿ—ฃ๏ธ Speech Mode](#๏ธ-speech-mode) - [๐Ÿ” Google API Keys Configuration](#-google-api-keys-configuration) - [Setting up environment variables](#setting-up-environment-variables) + - [๐Ÿง  Memory pre-seeding](#memory-pre-seeding) - [๐Ÿ’€ Continuous Mode โš ๏ธ](#-continuous-mode-๏ธ) - [GPT3.5 ONLY Mode](#gpt35-only-mode) - [๐Ÿ–ผ Image Generation](#image-generation) @@ -245,6 +246,52 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that 1. View memory usage by using the `--debug` flag :) + +## ๐Ÿง  Memory pre-seeding + +``` +# python scripts/data_ingestion.py -h +usage: data_ingestion.py [-h] (--file FILE | --dir DIR) [--init] [--overlap OVERLAP] [--max_length MAX_LENGTH] + +Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script. + +options: + -h, --help show this help message and exit + --file FILE The file to ingest. + --dir DIR The directory containing the files to ingest. + --init Init the memory and wipe its content (default: False) + --overlap OVERLAP The overlap size between chunks when ingesting files (default: 200) + --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000 + +# python scripts/data_ingestion.py --dir seed_data --init --overlap 200 --max_length 1000 +``` + +This script located at scripts/data_ingestion.py, allows you to ingest files into memory and pre-seed it before running Auto-GPT. + +Memory pre-seeding is a technique that involves ingesting relevant documents or data into the AI's memory so that it can use this information to generate more informed and accurate responses. + +To pre-seed the memory, the content of each document is split into chunks of a specified maximum length with a specified overlap between chunks, and then each chunk is added to the memory backend set in the .env file. When the AI is prompted to recall information, it can then access those pre-seeded memories to generate more informed and accurate responses. + +This technique is particularly useful when working with large amounts of data or when there is specific information that the AI needs to be able to access quickly. +By pre-seeding the memory, the AI can retrieve and use this information more efficiently, saving time, API call and improving the accuracy of its responses. + +You could for example download the documentation of an API, a Github repository, etc. and ingest it into memory before running Auto-GPT. + +โš ๏ธ If you use Redis as your memory, make sure to run Auto-GPT with the WIPE_REDIS_ON_START set to False in your .env file. + +โš ๏ธFor other memory backend, we currently forcefully wipe the memory when starting Auto-GPT. To ingest data with those memory backend, you can call the data_ingestion.py script anytime during an Auto-GPT run. + +Memories will be available to the AI immediately as they are ingested, even if ingested while Auto-GPT is running. + +In the example above, the script initializes the memory, ingests all files within the seed_data directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. +Note that you can also use the --file argument to ingest a single file into memory and that the script will only ingest files within the auto_gpt_workspace directory. + +You can adjust the max_length and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: + +- Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. +- Reducing the max_length value will create more chunks, which can save prompt tokens by allowing for more message history in the context, but will also increase the number of chunks. +- Increasing the max_length value will provide the AI with more contextual information from each chunk, reducing the number of chunks created and saving on OpenAI API requests. However, this may also use more prompt tokens and decrease the overall context available to the AI. + ## ๐Ÿ’€ Continuous Mode โš ๏ธ Run the AI **without** user authorisation, 100% automated. From 2e0b44ae05fce7795f662a81c765eeeeae32a768 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:46:49 +0200 Subject: [PATCH 12/40] fix chunk creation the last chunk wasn't correctly created, this commit fix that issue. --- scripts/file_operations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index e664fcc91d..f2a2b072d4 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -38,11 +38,12 @@ def split_file(content, max_length=4000, overlap=0): chunk = content[start:end] yield chunk start += max_length - overlap - if start + max_length - overlap >= content_length: - break - if end + overlap > content_length: + if start + max_length > content_length: start = content_length - max_length - + end = content_length + chunk = content[start:end] + yield chunk + break def read_file(filename): """Read a file and return the contents""" From 4e914e5ec1a4f7d39cba04cc2ebc0ba7f0055423 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:51:52 +0200 Subject: [PATCH 13/40] Revert "Update .gitignore" This reverts commit 7975c184a55a477e884e1920ed87dc67ca4b4261. --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index fc496609b4..aa0dceaa9e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,4 @@ log.txt # Coverage reports .coverage coverage.xml -htmlcov/ -log-ingestion.txt +htmlcov/ \ No newline at end of file From 2f1181f9a12bbbbf55b8f2224ecc645d22c5d90d Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:52:37 +0200 Subject: [PATCH 14/40] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cfa3b08b5d..403417ebdb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ ai_settings.yaml .idea/* auto-gpt.json log.txt +log-ingestion.txt # Coverage reports .coverage From 36d455c20e52aa1e09766a01c880f7914c5c24b7 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 23:31:26 +0200 Subject: [PATCH 15/40] split_file() rework rework the split_file function to make it simple and only have one yield while providing an overlap at the start and end of each chunk --- scripts/file_operations.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index db4702c526..c12774b9d9 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -35,15 +35,12 @@ def split_file(content, max_length=4000, overlap=0): while start < content_length: end = start + max_length - chunk = content[start:end] + if end + overlap < content_length: + chunk = content[start:end+overlap] + else: + chunk = content[start:content_length] yield chunk start += max_length - overlap - if start + max_length > content_length: - start = content_length - max_length - end = content_length - chunk = content[start:end] - yield chunk - break def read_file(filename): """Read a file and return the contents""" From 428caa9bef83e93a6f97a1341a03a0f41b71dec0 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Thu, 13 Apr 2023 12:57:57 +0300 Subject: [PATCH 16/40] Added flags, and implemented skip-reprompt --- scripts/config.py | 1 + scripts/main.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/config.py b/scripts/config.py index ebf1b08b22..fd370a72be 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -37,6 +37,7 @@ class Config(metaclass=Singleton): self.debug_mode = False self.continuous_mode = False self.speak_mode = False + self.skip_reprompt = False self.fast_llm_model = os.getenv("FAST_LLM_MODEL", "gpt-3.5-turbo") self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4") diff --git a/scripts/main.py b/scripts/main.py index 81f560b216..f81b09a76a 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -183,7 +183,11 @@ def load_variables(config_file="config.yaml"): def construct_prompt(): """Construct the prompt for the AI to respond to""" config = AIConfig.load() - if config.ai_name: + if cfg.skip_reprompt and config.ai_name: + logger.typewriter_log("Name :", Fore.GREEN, config.ai_name) + logger.typewriter_log("Role :", Fore.GREEN, config.ai_role) + logger.typewriter_log("Goals:", Fore.GREEN, config.ai_goals) + elif config.ai_name: logger.typewriter_log( f"Welcome back! ", Fore.GREEN, @@ -270,12 +274,14 @@ def parse_arguments(): cfg.set_speak_mode(False) parser = argparse.ArgumentParser(description='Process arguments.') - parser.add_argument('--continuous', action='store_true', help='Enable Continuous Mode') + parser.add_argument('--continuous', '-c', action='store_true', help='Enable Continuous Mode') parser.add_argument('--speak', action='store_true', help='Enable Speak Mode') parser.add_argument('--debug', action='store_true', help='Enable Debug Mode') parser.add_argument('--gpt3only', action='store_true', help='Enable GPT3.5 Only Mode') parser.add_argument('--gpt4only', action='store_true', help='Enable GPT4 Only Mode') parser.add_argument('--use-memory', '-m', dest="memory_type", help='Defines which Memory backend to use') + parser.add_argument('--skip-reprompt', '-y', dest='skip_reprompt', action='store_true', help='Skips the re-prompting messages at the beginning of the script') + parser.add_argument('--ai-settings', '-C', dest='ai_settings_file', help="Specifies which ai_settings.yaml file to use, will also automatically skip the re-prompt.") args = parser.parse_args() if args.debug: @@ -315,6 +321,10 @@ def parse_arguments(): else: cfg.memory_backend = chosen + if args.skip_reprompt: + logger.typewriter_log("Skip Re-prompt: ", Fore.GREEN, "ENABLED") + cfg.skip_reprompt = True + # TODO: fill in llm values here check_openai_api_key() From 0f6fba7d65302591f2c77a41483953df43d12d2b Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Thu, 13 Apr 2023 14:02:42 +0300 Subject: [PATCH 17/40] Implemented the '--ai-settings' flag --- scripts/config.py | 1 + scripts/main.py | 16 +++++++++++++++- scripts/utils.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/config.py b/scripts/config.py index fd370a72be..ad968fb247 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -39,6 +39,7 @@ class Config(metaclass=Singleton): self.speak_mode = False self.skip_reprompt = False + self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml") self.fast_llm_model = os.getenv("FAST_LLM_MODEL", "gpt-3.5-turbo") self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4") self.fast_token_limit = int(os.getenv("FAST_TOKEN_LIMIT", 4000)) diff --git a/scripts/main.py b/scripts/main.py index f81b09a76a..07d2bbd292 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -182,7 +182,7 @@ def load_variables(config_file="config.yaml"): def construct_prompt(): """Construct the prompt for the AI to respond to""" - config = AIConfig.load() + config = AIConfig.load(cfg.ai_settings_file) if cfg.skip_reprompt and config.ai_name: logger.typewriter_log("Name :", Fore.GREEN, config.ai_name) logger.typewriter_log("Role :", Fore.GREEN, config.ai_role) @@ -324,7 +324,21 @@ def parse_arguments(): if args.skip_reprompt: logger.typewriter_log("Skip Re-prompt: ", Fore.GREEN, "ENABLED") cfg.skip_reprompt = True + + if args.ai_settings_file: + file = args.ai_settings_file + # Validate file + (validated, message) = utils.validate_yaml_file(file) + if not validated: + logger.typewriter_log("FAILED FILE VALIDATION", Fore.RED, message) + exit(1) + + logger.typewriter_log("Using AI Settings File:", Fore.GREEN, file) + cfg.ai_settings_file = file + cfg.skip_reprompt = True + + # TODO: fill in llm values here check_openai_api_key() diff --git a/scripts/utils.py b/scripts/utils.py index 5039796fbd..bca8d4a898 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,3 +1,6 @@ +import yaml +from colorama import Fore + def clean_input(prompt: str=''): try: return input(prompt) @@ -6,3 +9,14 @@ def clean_input(prompt: str=''): print("Quitting...") exit(0) + +def validate_yaml_file(file: str): + try: + with open(file) as file: + yaml.load(file, Loader=yaml.FullLoader) + except FileNotFoundError: + return (False, f"The file {Fore.CYAN}`{file}`{Fore.RESET} wasn't found") + except yaml.YAMLError as e: + return (False, f"There was an issue while trying to read with your AI Settings file: {e}") + + return (True, f"Successfully validated {Fore.CYAN}`{file}`{Fore.RESET}!") \ No newline at end of file From a10ffc1dbed88ce74f7ebb1dae0c90fb18bae9f6 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Thu, 13 Apr 2023 14:26:16 +0300 Subject: [PATCH 18/40] Fixed error logging when choosing non-supported memory backend with '--use-memory' --- scripts/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 07d2bbd292..59cb565e95 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -316,8 +316,8 @@ def parse_arguments(): supported_memory = get_supported_memory_backends() chosen = args.memory_type if not chosen in supported_memory: - print_to_console("ONLY THE FOLLOWING MEMORY BACKENDS ARE SUPPORTED: ", Fore.RED, f'{supported_memory}') - print_to_console(f"Defaulting to: ", Fore.YELLOW, cfg.memory_backend) + logger.typewriter_log("ONLY THE FOLLOWING MEMORY BACKENDS ARE SUPPORTED: ", Fore.RED, f'{supported_memory}') + logger.typewriter_log(f"Defaulting to: ", Fore.YELLOW, cfg.memory_backend) else: cfg.memory_backend = chosen From ff094c7ecc58fad572dccbc8a376a75045d91733 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Thu, 13 Apr 2023 15:09:24 +0300 Subject: [PATCH 19/40] Resolve Linter Issues --- scripts/main.py | 5 ++--- scripts/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 59cb565e95..0674db47b6 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -327,19 +327,18 @@ def parse_arguments(): if args.ai_settings_file: file = args.ai_settings_file - + # Validate file (validated, message) = utils.validate_yaml_file(file) if not validated: logger.typewriter_log("FAILED FILE VALIDATION", Fore.RED, message) exit(1) - + logger.typewriter_log("Using AI Settings File:", Fore.GREEN, file) cfg.ai_settings_file = file cfg.skip_reprompt = True - # TODO: fill in llm values here check_openai_api_key() parse_arguments() diff --git a/scripts/utils.py b/scripts/utils.py index bca8d4a898..2b51c1fce3 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -18,5 +18,5 @@ def validate_yaml_file(file: str): return (False, f"The file {Fore.CYAN}`{file}`{Fore.RESET} wasn't found") except yaml.YAMLError as e: return (False, f"There was an issue while trying to read with your AI Settings file: {e}") - - return (True, f"Successfully validated {Fore.CYAN}`{file}`{Fore.RESET}!") \ No newline at end of file + + return (True, f"Successfully validated {Fore.CYAN}`{file}`{Fore.RESET}!") From 41f17f89043b19e7a5990894996c6dc407d734c0 Mon Sep 17 00:00:00 2001 From: Peter Edwards Date: Thu, 13 Apr 2023 16:02:15 +0200 Subject: [PATCH 20/40] Small README.md clarity update and usage fixup --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0d12ae23a2..90706cbff1 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ Your support is greatly appreciated - [Python 3.8 or later](https://www.tutorialspoint.com/how-to-install-python-in-windows) - [OpenAI API key](https://platform.openai.com/account/api-keys) -- [PINECONE API key](https://www.pinecone.io/) + Optional: +- [PINECONE API key](https://www.pinecone.io/) (If you want Pinecone backed memory) - ElevenLabs Key (If you want the AI to speak) ## ๐Ÿ’พ Installation @@ -114,8 +115,8 @@ pip install -r requirements.txt python scripts/main.py ``` -2. After each of AUTO-GPT's actions, type "NEXT COMMAND" to authorise them to continue. -3. To exit the program, type "exit" and press Enter. +2. After each of action, enter 'y' to authorise command, 'y -N' to run N continuous commands, 'n' to exit program, or enter additional feedback for the AI. + ### Logs @@ -217,7 +218,10 @@ Pinecone enables the storage of vast amounts of vector-based memory, allowing fo ### Setting up environment variables -Simply set them in the `.env` file. +In the `.env` file set: +- `PINECONE_API_KEY` +- `PINECONE_ENV` (something like: us-east4-gcp) +- `MEMORY_BACKEND=pinecone` Alternatively, you can set them from the command line (advanced): @@ -226,7 +230,7 @@ For Windows Users: ``` setx PINECONE_API_KEY "YOUR_PINECONE_API_KEY" setx PINECONE_ENV "Your pinecone region" # something like: us-east4-gcp - +setx MEMORY_BACKEND "pinecone" ``` For macOS and Linux users: @@ -234,7 +238,7 @@ For macOS and Linux users: ``` export PINECONE_API_KEY="YOUR_PINECONE_API_KEY" export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp - +export MEMORY_BACKEND="pinecone" ``` From f9cbddc9f06baed21bc5bc0f3bd6848310240737 Mon Sep 17 00:00:00 2001 From: Ron Balter <44070810+Ronbalt@users.noreply.github.com> Date: Fri, 14 Apr 2023 00:58:51 +0300 Subject: [PATCH 21/40] Enable Custom Search API in gcp project While following this guide to enable google search, this step was missing for me and the API calls to https://customsearch.googleapis.com/customsearch/v1?q= failed with: """ Custom Search API has not been used in project before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/customsearch.googleapis.com/overview?project= then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry. """ Also, checked that merely https://console.developers.google.com/apis/api/customsearch.googleapis.com redirects to the active project used in the last session in GCP, so no need to provide the projectId parameter. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d11219769c..3d8fd003b6 100644 --- a/README.md +++ b/README.md @@ -154,9 +154,10 @@ To use the `google_official_search` command, you need to set up your Google API 4. Go to the [APIs & Services Dashboard](https://console.cloud.google.com/apis/dashboard) and click "Enable APIs and Services". Search for "Custom Search API" and click on it, then click "Enable". 5. Go to the [Credentials](https://console.cloud.google.com/apis/credentials) page and click "Create Credentials". Choose "API Key". 6. Copy the API key and set it as an environment variable named `GOOGLE_API_KEY` on your machine. See setting up environment variables below. -7. Go to the [Custom Search Engine](https://cse.google.com/cse/all) page and click "Add". -8. Set up your search engine by following the prompts. You can choose to search the entire web or specific sites. -9. Once you've created your search engine, click on "Control Panel" and then "Basics". Copy the "Search engine ID" and set it as an environment variable named `CUSTOM_SEARCH_ENGINE_ID` on your machine. See setting up environment variables below. +7. [Enable](https://console.developers.google.com/apis/api/customsearch.googleapis.com) the Custom Search API on your project. (Might need to wait few minutes to propagate) +8. Go to the [Custom Search Engine](https://cse.google.com/cse/all) page and click "Add". +9. Set up your search engine by following the prompts. You can choose to search the entire web or specific sites. +10. Once you've created your search engine, click on "Control Panel" and then "Basics". Copy the "Search engine ID" and set it as an environment variable named `CUSTOM_SEARCH_ENGINE_ID` on your machine. See setting up environment variables below. _Remember that your free daily custom search quota allows only up to 100 searches. To increase this limit, you need to assign a billing account to the project to profit from up to 10K daily searches._ @@ -357,4 +358,4 @@ flake8 scripts/ tests/ # Or, if you want to run flake8 with the same configuration as the CI: flake8 scripts/ tests/ --select E303,W293,W291,W292,E305,E231,E302 -``` \ No newline at end of file +``` From c59b6b5543793a01d1231552f6c81f4f72ddc740 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Thu, 13 Apr 2023 15:19:41 -0700 Subject: [PATCH 22/40] wrap infinite loop in class agent --- scripts/main.py | 226 ++++++++++++++++++++++++++++-------------------- 1 file changed, 132 insertions(+), 94 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 466f50dd1c..9c823ab110 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -351,110 +351,148 @@ def main(): # this is particularly important for indexing and referencing pinecone memory memory = get_memory(cfg, init=True) print('Using memory of type: ' + memory.__class__.__name__) - # Interaction Loop - loop_count = 0 - while True: - # Discontinue if continuous limit is reached - loop_count += 1 - if cfg.continuous_mode and cfg.continuous_limit > 0 and loop_count > cfg.continuous_limit: - logger.typewriter_log("Continuous Limit Reached: ", Fore.YELLOW, f"{cfg.continuous_limit}") - break + agent = Agent( + ai_name=ai_name, + memory=memory, + full_message_history=full_message_history, + next_action_count=next_action_count, + prompt=prompt, + user_input=user_input + ) + agent.start_interaction_loop() - # Send message to AI, get response - with Spinner("Thinking... "): - assistant_reply = chat.chat_with_ai( - prompt, - user_input, - full_message_history, - memory, - cfg.fast_token_limit) # TODO: This hardcodes the model to use GPT3.5. Make this an argument - # Print Assistant thoughts - print_assistant_thoughts(assistant_reply) +class Agent: + """Agent class for interacting with Auto-GPT. - # Get command name and arguments - try: - command_name, arguments = cmd.get_command( - attempt_to_fix_json_by_finding_outermost_brackets(assistant_reply)) - if cfg.speak_mode: - speak.say_text(f"I want to execute {command_name}") - except Exception as e: - logger.error("Error: \n", str(e)) + Attributes: + ai_name: The name of the agent. + memory: The memory object to use. + full_message_history: The full message history. + next_action_count: The number of actions to execute. + prompt: The prompt to use. + user_input: The user input. - if not cfg.continuous_mode and next_action_count == 0: - ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### - # Get key press: Prompt the user to press enter to continue or escape - # to exit - user_input = "" - logger.typewriter_log( - "NEXT ACTION: ", - Fore.CYAN, - f"COMMAND = {Fore.CYAN}{command_name}{Style.RESET_ALL} ARGUMENTS = {Fore.CYAN}{arguments}{Style.RESET_ALL}") - print( - f"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 'n' to exit program, or enter feedback for {ai_name}...", - flush=True) - while True: - console_input = utils.clean_input(Fore.MAGENTA + "Input:" + Style.RESET_ALL) - if console_input.lower().rstrip() == "y": - user_input = "GENERATE NEXT COMMAND JSON" - break - elif console_input.lower().startswith("y -"): - try: - next_action_count = abs(int(console_input.split(" ")[1])) - user_input = "GENERATE NEXT COMMAND JSON" - except ValueError: - print("Invalid input format. Please enter 'y -n' where n is the number of continuous tasks.") - continue - break - elif console_input.lower() == "n": - user_input = "EXIT" - break - else: - user_input = console_input - command_name = "human_feedback" - break + """ + def __init__(self, + ai_name, + memory, + full_message_history, + next_action_count, + prompt, + user_input): + self.ai_name = ai_name + self.memory = memory + self.full_message_history = full_message_history + self.next_action_count = next_action_count + self.prompt = prompt + self.user_input = user_input - if user_input == "GENERATE NEXT COMMAND JSON": - logger.typewriter_log( - "-=-=-=-=-=-=-= COMMAND AUTHORISED BY USER -=-=-=-=-=-=-=", - Fore.MAGENTA, - "") - elif user_input == "EXIT": - print("Exiting...", flush=True) + def start_interaction_loop(self): + # Interaction Loop + loop_count = 0 + while True: + # Discontinue if continuous limit is reached + loop_count += 1 + if cfg.continuous_mode and cfg.continuous_limit > 0 and loop_count > cfg.continuous_limit: + logger.typewriter_log("Continuous Limit Reached: ", Fore.YELLOW, f"{cfg.continuous_limit}") break - else: - # Print command - logger.typewriter_log( - "NEXT ACTION: ", - Fore.CYAN, - f"COMMAND = {Fore.CYAN}{command_name}{Style.RESET_ALL} ARGUMENTS = {Fore.CYAN}{arguments}{Style.RESET_ALL}") - # Execute command - if command_name is not None and command_name.lower().startswith("error"): - result = f"Command {command_name} threw the following error: " + arguments - elif command_name == "human_feedback": - result = f"Human feedback: {user_input}" - else: - result = f"Command {command_name} returned: {cmd.execute_command(command_name, arguments)}" - if next_action_count > 0: - next_action_count -= 1 + # Send message to AI, get response + with Spinner("Thinking... "): + assistant_reply = chat.chat_with_ai( + self.prompt, + self.user_input, + self.full_message_history, + self.memory, + cfg.fast_token_limit) # TODO: This hardcodes the model to use GPT3.5. Make this an argument - memory_to_add = f"Assistant Reply: {assistant_reply} " \ - f"\nResult: {result} " \ - f"\nHuman Feedback: {user_input} " + # Print Assistant thoughts + print_assistant_thoughts(assistant_reply) - memory.add(memory_to_add) + # Get command name and arguments + try: + command_name, arguments = cmd.get_command( + attempt_to_fix_json_by_finding_outermost_brackets(assistant_reply)) + if cfg.speak_mode: + speak.say_text(f"I want to execute {command_name}") + except Exception as e: + logger.error("Error: \n", str(e)) - # Check if there's a result from the command append it to the message - # history - if result is not None: - full_message_history.append(chat.create_chat_message("system", result)) - logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result) - else: - full_message_history.append( - chat.create_chat_message( - "system", "Unable to execute command")) - logger.typewriter_log("SYSTEM: ", Fore.YELLOW, "Unable to execute command") + if not cfg.continuous_mode and self.next_action_count == 0: + ### GET USER AUTHORIZATION TO EXECUTE COMMAND ### + # Get key press: Prompt the user to press enter to continue or escape + # to exit + self.user_input = "" + logger.typewriter_log( + "NEXT ACTION: ", + Fore.CYAN, + f"COMMAND = {Fore.CYAN}{command_name}{Style.RESET_ALL} ARGUMENTS = {Fore.CYAN}{arguments}{Style.RESET_ALL}") + print( + f"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 'n' to exit program, or enter feedback for {self.ai_name}...", + flush=True) + while True: + console_input = utils.clean_input(Fore.MAGENTA + "Input:" + Style.RESET_ALL) + if console_input.lower().rstrip() == "y": + self.user_input = "GENERATE NEXT COMMAND JSON" + break + elif console_input.lower().startswith("y -"): + try: + self.next_action_count = abs(int(console_input.split(" ")[1])) + self.user_input = "GENERATE NEXT COMMAND JSON" + except ValueError: + print("Invalid input format. Please enter 'y -n' where n is the number of continuous tasks.") + continue + break + elif console_input.lower() == "n": + self.user_input = "EXIT" + break + else: + self.user_input = console_input + command_name = "human_feedback" + break + + if self.user_input == "GENERATE NEXT COMMAND JSON": + logger.typewriter_log( + "-=-=-=-=-=-=-= COMMAND AUTHORISED BY USER -=-=-=-=-=-=-=", + Fore.MAGENTA, + "") + elif self.user_input == "EXIT": + print("Exiting...", flush=True) + break + else: + # Print command + logger.typewriter_log( + "NEXT ACTION: ", + Fore.CYAN, + f"COMMAND = {Fore.CYAN}{command_name}{Style.RESET_ALL} ARGUMENTS = {Fore.CYAN}{arguments}{Style.RESET_ALL}") + + # Execute command + if command_name is not None and command_name.lower().startswith("error"): + result = f"Command {command_name} threw the following error: " + arguments + elif command_name == "human_feedback": + result = f"Human feedback: {self.user_input}" + else: + result = f"Command {command_name} returned: {cmd.execute_command(command_name, arguments)}" + if self.next_action_count > 0: + self.next_action_count -= 1 + + memory_to_add = f"Assistant Reply: {assistant_reply} " \ + f"\nResult: {result} " \ + f"\nHuman Feedback: {self.user_input} " + + self.memory.add(memory_to_add) + + # Check if there's a result from the command append it to the message + # history + if result is not None: + self.full_message_history.append(chat.create_chat_message("system", result)) + logger.typewriter_log("SYSTEM: ", Fore.YELLOW, result) + else: + self.full_message_history.append( + chat.create_chat_message( + "system", "Unable to execute command")) + logger.typewriter_log("SYSTEM: ", Fore.YELLOW, "Unable to execute command") if __name__ == "__main__": From 47b72df262b894752b7fd0324f42f71ffc70e38c Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Fri, 14 Apr 2023 01:20:43 +0300 Subject: [PATCH 23/40] Added 'AI_SETTINGS_FILE' to .env --- .env.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.template b/.env.template index 474b272761..c5cb54fdd2 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,8 @@ ################################################################################ # EXECUTE_LOCAL_COMMANDS - Allow local command execution (Example: False) EXECUTE_LOCAL_COMMANDS=False +# AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml) +AI_SETTINGS_FILE=ai_settings.yaml ################################################################################ ### LLM PROVIDER From 05f6e9673f285ac40cf982a544dfa14750cf6af1 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Fri, 14 Apr 2023 01:23:23 +0300 Subject: [PATCH 24/40] Resolve Linter Issues --- scripts/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/utils.py b/scripts/utils.py index 2b51c1fce3..7521df298e 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,6 +1,7 @@ import yaml from colorama import Fore + def clean_input(prompt: str=''): try: return input(prompt) From 43efbff4b80edbad2fc35010773372e76c3eb34b Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Thu, 13 Apr 2023 15:22:43 -0700 Subject: [PATCH 25/40] remove useless load_variables_method --- scripts/main.py | 54 ------------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 466f50dd1c..1425e4200a 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -129,60 +129,6 @@ def print_assistant_thoughts(assistant_reply): logger.error("Error: \n", call_stack) -def load_variables(config_file="config.yaml"): - """Load variables from yaml file if it exists, otherwise prompt the user for input""" - try: - with open(config_file) as file: - config = yaml.load(file, Loader=yaml.FullLoader) - ai_name = config.get("ai_name") - ai_role = config.get("ai_role") - ai_goals = config.get("ai_goals") - except FileNotFoundError: - ai_name = "" - ai_role = "" - ai_goals = [] - - # Prompt the user for input if config file is missing or empty values - if not ai_name: - ai_name = utils.clean_input("Name your AI: ") - if ai_name == "": - ai_name = "Entrepreneur-GPT" - - if not ai_role: - ai_role = utils.clean_input(f"{ai_name} is: ") - if ai_role == "": - ai_role = "an AI designed to autonomously develop and run businesses with the sole goal of increasing your net worth." - - if not ai_goals: - print("Enter up to 5 goals for your AI: ") - print("For example: \nIncrease net worth, Grow Twitter Account, Develop and manage multiple businesses autonomously'") - print("Enter nothing to load defaults, enter nothing when finished.") - ai_goals = [] - for i in range(5): - ai_goal = utils.clean_input(f"Goal {i+1}: ") - if ai_goal == "": - break - ai_goals.append(ai_goal) - if len(ai_goals) == 0: - ai_goals = ["Increase net worth", "Grow Twitter Account", "Develop and manage multiple businesses autonomously"] - - # Save variables to yaml file - config = {"ai_name": ai_name, "ai_role": ai_role, "ai_goals": ai_goals} - with open(config_file, "w") as file: - documents = yaml.dump(config, file) - - prompt = get_prompt() - prompt_start = """Your decisions must always be made independently without seeking user assistance. Play to your strengths as an LLM and pursue simple strategies with no legal complications.""" - - # Construct full prompt - full_prompt = f"You are {ai_name}, {ai_role}\n{prompt_start}\n\nGOALS:\n\n" - for i, goal in enumerate(ai_goals): - full_prompt += f"{i+1}. {goal}\n" - - full_prompt += f"\n\n{prompt}" - return full_prompt - - def construct_prompt(): """Construct the prompt for the AI to respond to""" config = AIConfig.load() From 8472bbd4556999cdd62e4930ae3723f18b746ef4 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Fri, 14 Apr 2023 01:34:30 +0300 Subject: [PATCH 26/40] Added 'Command Line Arguments' section to README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index d11219769c..8d402e7d69 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,14 @@ To output debug logs: ``` python scripts/main.py --debug ``` +### Command Line Arguments +Here are some common arguments you can use when running Auto-GPT: +> Replace anything in angled brackets (<>) to a value you want to specify +* `python scripts/main.py --help` to see a list of all available command line arguments +* `python scripts/main.py --ai-settings ` to run Auto-GPT with a different AI Settings file. +* `python scripts/main.py --use-memory ` to specify one of 3 memory backends: `local`, `redis` or `pinecone` + +> **NOTE**: There are shorthands for some of these flags, for example `-m` for `--use-memory`. Use `python scripts/main.py --help` for more information ## ๐Ÿ—ฃ๏ธ Speech Mode From 6702a04f767702d1e57ddcec81f2481def19f8a7 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Fri, 14 Apr 2023 01:50:13 +0300 Subject: [PATCH 27/40] Add 'no_memory' support for memory flag --- scripts/memory/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/memory/__init__.py b/scripts/memory/__init__.py index a0afc874e2..9b53d8d29a 100644 --- a/scripts/memory/__init__.py +++ b/scripts/memory/__init__.py @@ -3,7 +3,7 @@ from memory.no_memory import NoMemory # List of supported memory backends # Add a backend to this list if the import attempt is successful -supported_memory = ['local'] +supported_memory = ['local', 'no_memory'] try: from memory.redismem import RedisMemory From 4f923ece60baee2c086c29610a05c4f130e43aa9 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Fri, 14 Apr 2023 01:56:45 +0300 Subject: [PATCH 28/40] Added double_check logging to AI Settings validator, and updated README for 'no_memory' --- README.md | 4 ++-- scripts/main.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d402e7d69..b55a80c562 100644 --- a/README.md +++ b/README.md @@ -137,9 +137,9 @@ python scripts/main.py --debug ### Command Line Arguments Here are some common arguments you can use when running Auto-GPT: > Replace anything in angled brackets (<>) to a value you want to specify -* `python scripts/main.py --help` to see a list of all available command line arguments +* `python scripts/main.py --help` to see a list of all available command line arguments. * `python scripts/main.py --ai-settings ` to run Auto-GPT with a different AI Settings file. -* `python scripts/main.py --use-memory ` to specify one of 3 memory backends: `local`, `redis` or `pinecone` +* `python scripts/main.py --use-memory ` to specify one of 3 memory backends: `local`, `redis`, `pinecone` or 'no_memory'. > **NOTE**: There are shorthands for some of these flags, for example `-m` for `--use-memory`. Use `python scripts/main.py --help` for more information diff --git a/scripts/main.py b/scripts/main.py index 400eb1f638..78ffe2430a 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -348,6 +348,7 @@ def parse_arguments(): (validated, message) = utils.validate_yaml_file(file) if not validated: logger.typewriter_log("FAILED FILE VALIDATION", Fore.RED, message) + logger.double_check() exit(1) logger.typewriter_log("Using AI Settings File:", Fore.GREEN, file) From 4666ea015074506196762cc8008489d06a3d02bb Mon Sep 17 00:00:00 2001 From: Jesse R Weigel Date: Thu, 13 Apr 2023 21:57:31 -0400 Subject: [PATCH 29/40] fix misspelling --- scripts/call_ai_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/call_ai_function.py b/scripts/call_ai_function.py index 6f1d6ceee8..940eacfe0f 100644 --- a/scripts/call_ai_function.py +++ b/scripts/call_ai_function.py @@ -13,7 +13,7 @@ def call_ai_function(function, args, description, model=None): model = cfg.smart_llm_model # For each arg, if any are None, convert to "None": args = [str(arg) if arg is not None else "None" for arg in args] - # parse args to comma seperated string + # parse args to comma separated string args = ", ".join(args) messages = [ { From 1f21998f0c2115a76b1c53ae2b89ea581d2fd106 Mon Sep 17 00:00:00 2001 From: sunnypranay Date: Thu, 13 Apr 2023 21:47:28 -0500 Subject: [PATCH 30/40] Improve Dockerfile with best practices and optimizations --- Dockerfile | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d264c88c8..e776664e84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,23 @@ +# Use an official Python base image from the Docker Hub FROM python:3.11-slim -ENV PIP_NO_CACHE_DIR=yes -WORKDIR /app -COPY requirements.txt . -RUN pip install -r requirements.txt -COPY scripts/ . -ENTRYPOINT ["python", "main.py"] + +# Set environment variables +ENV PIP_NO_CACHE_DIR=yes \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# Create a non-root user and set permissions +RUN useradd --create-home appuser +WORKDIR /home/appuser +RUN chown appuser:appuser /home/appuser +USER appuser + +# Copy the requirements.txt file and install the requirements +COPY --chown=appuser:appuser requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +# Copy the application files +COPY --chown=appuser:appuser scripts/ . + +# Set the entrypoint +ENTRYPOINT ["python", "main.py"] \ No newline at end of file From aeb81aa597a1eedcddcc42b060b98580a1870324 Mon Sep 17 00:00:00 2001 From: eng-cc <66scc66@gmail.com> Date: Fri, 14 Apr 2023 10:54:59 +0800 Subject: [PATCH 31/40] [environments] add devcontainer environment --- .devcontainer/Dockerfile | 23 +++++++++++++++++++ .devcontainer/devcontainer.json | 39 +++++++++++++++++++++++++++++++++ README.md | 4 +++- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..f3b2e2dbb5 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,23 @@ +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT=3-bullseye +FROM python:3.8 + +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 + && apt-get purge -y imagemagick imagemagick-6-common + +# Temporary: Upgrade python packages due to https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-40897 +# They are installed by the base image (python) which does not have the patch. +RUN python3 -m pip install --upgrade setuptools + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..5fefd9c13d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,39 @@ +{ + "build": { + "dockerfile": "./Dockerfile", + "context": "." + }, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": "true", + "username": "vscode", + "userUid": "1000", + "userGid": "1000", + "upgradePackages": "true" + }, + "ghcr.io/devcontainers/features/python:1": "none", + "ghcr.io/devcontainers/features/node:1": "none", + "ghcr.io/devcontainers/features/git:1": { + "version": "latest", + "ppa": "false" + } + }, + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python" + } + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "pip3 install --user -r requirements.txt", + + // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/README.md b/README.md index d11219769c..849679977c 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,9 @@ Your support is greatly appreciated ## ๐Ÿ“‹ Requirements -- [Python 3.8 or later](https://www.tutorialspoint.com/how-to-install-python-in-windows) +- environments(just choose one) + - [vscode + devcontainer](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers): It has been configured in the .devcontainer folder and can be used directly + - [Python 3.8 or later](https://www.tutorialspoint.com/how-to-install-python-in-windows) - [OpenAI API key](https://platform.openai.com/account/api-keys) - [PINECONE API key](https://www.pinecone.io/) From 3128397988de6dc5280eb45cca814e98cd8c0a2d Mon Sep 17 00:00:00 2001 From: GyDi Date: Fri, 14 Apr 2023 11:17:46 +0800 Subject: [PATCH 32/40] fix: remove duplicate debug mode logger --- scripts/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 466f50dd1c..0221af4c72 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -318,10 +318,6 @@ def parse_arguments(): logger.typewriter_log("GPT4 Only Mode: ", Fore.GREEN, "ENABLED") cfg.set_fast_llm_model(cfg.smart_llm_model) - if args.debug: - logger.typewriter_log("Debug Mode: ", Fore.GREEN, "ENABLED") - cfg.set_debug_mode(True) - if args.memory_type: supported_memory = get_supported_memory_backends() chosen = args.memory_type From 5e6d0b620a582f2f143d91df1921d44c22e648df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AB=E5=B0=94=E7=B4=A2?= Date: Fri, 14 Apr 2023 11:38:29 +0800 Subject: [PATCH 33/40] Resolving Unicode encoding issues Solve the problem that Chinese, Japanese, Korean and other non-English languages are all encoded in Unicode when writing ai_settings.yaml configuration. --- scripts/ai_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ai_config.py b/scripts/ai_config.py index 4eb076ef2d..89a4e07eaa 100644 --- a/scripts/ai_config.py +++ b/scripts/ai_config.py @@ -70,8 +70,8 @@ class AIConfig: """ config = {"ai_name": self.ai_name, "ai_role": self.ai_role, "ai_goals": self.ai_goals} - with open(config_file, "w") as file: - yaml.dump(config, file) + with open(config_file, "w", encoding='utf-8') as file: + yaml.dump(config, file, allow_unicode=True) def construct_full_prompt(self) -> str: """ From 2fd96b68bdc9e7ab48b34eeff8d96e32b9eafdcf Mon Sep 17 00:00:00 2001 From: meta-fx Date: Fri, 14 Apr 2023 01:28:47 -0500 Subject: [PATCH 34/40] Added new line and elevenlabs elements back to the env --- .env.template | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.env.template b/.env.template index aa7b04e977..e22ce4f522 100644 --- a/.env.template +++ b/.env.template @@ -104,4 +104,12 @@ USE_MAC_OS_TTS=False ### STREAMELEMENTS # USE_BRIAN_TTS - Use Brian TTS or not (Default: False) -USE_BRIAN_TTS=False \ No newline at end of file +USE_BRIAN_TTS=False + +### ELEVENLABS +# ELEVENLABS_API_KEY - Eleven Labs API key (Example: my-elevenlabs-api-key) +# ELEVENLABS_VOICE_1_ID - Eleven Labs voice 1 ID (Example: my-voice-id-1) +# ELEVENLABS_VOICE_2_ID - Eleven Labs voice 2 ID (Example: my-voice-id-2) +ELEVENLABS_API_KEY=your-elevenlabs-api-key +ELEVENLABS_VOICE_1_ID=your-voice-id-1 +ELEVENLABS_VOICE_2_ID=your-voice-id-2 From 1612069594402540da4116f9f599b47091b8f041 Mon Sep 17 00:00:00 2001 From: meta-fx Date: Fri, 14 Apr 2023 02:18:17 -0500 Subject: [PATCH 35/40] Fixed E302 expected 2 blank lines, found 1 --- scripts/speak.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speak.py b/scripts/speak.py index d71b5bcac6..3afa591dd5 100644 --- a/scripts/speak.py +++ b/scripts/speak.py @@ -52,6 +52,7 @@ def eleven_labs_speech(text, voice_index=0): print("Response content:", response.content) return False + def brian_speech(text): """Speak text using Brian with the streamelements API""" tts_url = f"https://api.streamelements.com/kappa/v2/speech?voice=Brian&text={text}" @@ -69,6 +70,7 @@ def brian_speech(text): print("Response content:", response.content) return False + def gtts_speech(text): tts = gtts.gTTS(text) with mutex_lock: From 6403bf112790b34fa122bdd519703e4b110f6875 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Fri, 14 Apr 2023 10:35:30 +0200 Subject: [PATCH 36/40] Update data_ingestion.py fixed linting --- scripts/data_ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index 3281116629..9addc34bb2 100644 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -6,6 +6,7 @@ from file_operations import ingest_file, search_files cfg = Config() + def configure_logging(): logging.basicConfig(filename='log-ingestion.txt', filemode='a', @@ -43,7 +44,6 @@ def main(): args = parser.parse_args() - # Initialize memory memory = get_memory(cfg, init=args.init) print('Using memory of type: ' + memory.__class__.__name__) From c0462dbe7768d41ac205644987ad0fa9f14a5afc Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Fri, 14 Apr 2023 10:35:52 +0200 Subject: [PATCH 37/40] Update file_operations.py fixed linting --- scripts/file_operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index ed5aa4ec53..1a072561b4 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -42,6 +42,7 @@ def split_file(content, max_length=4000, overlap=0): yield chunk start += max_length - overlap + def read_file(filename): """Read a file and return the contents""" try: From 475edd3b40c8769b22519083af2106bcbd08e559 Mon Sep 17 00:00:00 2001 From: Mike Kelly Date: Fri, 14 Apr 2023 12:32:33 +0100 Subject: [PATCH 38/40] make the path reference in logger more robust --- scripts/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/logger.py b/scripts/logger.py index 91bdb6f605..4c7e588f20 100644 --- a/scripts/logger.py +++ b/scripts/logger.py @@ -24,7 +24,8 @@ For console handler: simulates typing class Logger(metaclass=Singleton): def __init__(self): # create log directory if it doesn't exist - log_dir = os.path.join('..', 'logs') + this_files_dir_path = os.path.dirname(__file__) + log_dir = os.path.join(this_files_dir_path, '../logs') if not os.path.exists(log_dir): os.makedirs(log_dir) From 9e27e0165d15d441e58e5771e22957ae4fbd0063 Mon Sep 17 00:00:00 2001 From: Mike Kelly Date: Fri, 14 Apr 2023 12:36:18 +0100 Subject: [PATCH 39/40] gitignore the logs file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cf6e75df15..0a98328ca6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ last_run_ai_settings.yaml .idea/* auto-gpt.json log.txt +logs # Coverage reports .coverage From b18530a9854f7b3e0cd5fb8333221bb0000cc4cb Mon Sep 17 00:00:00 2001 From: sagarishere <5121817+sagarishere@users.noreply.github.com> Date: Fri, 14 Apr 2023 16:31:45 +0300 Subject: [PATCH 40/40] update to modern python format syntax update to modern python format syntax no logic change --- scripts/spinner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/spinner.py b/scripts/spinner.py index df1f4ddf93..d232152926 100644 --- a/scripts/spinner.py +++ b/scripts/spinner.py @@ -17,10 +17,10 @@ class Spinner: def spin(self): """Spin the spinner""" while self.running: - sys.stdout.write(next(self.spinner) + " " + self.message + "\r") + sys.stdout.write(f"{next(self.spinner)} {self.message}\r") sys.stdout.flush() time.sleep(self.delay) - sys.stdout.write('\r' + ' ' * (len(self.message) + 2) + '\r') + sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") def __enter__(self): """Start the spinner""" @@ -32,5 +32,5 @@ class Spinner: """Stop the spinner""" self.running = False self.spinner_thread.join() - sys.stdout.write('\r' + ' ' * (len(self.message) + 2) + '\r') + sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") sys.stdout.flush()