github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-04-08 03:00:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/BillSchumacher/Auto-GPT into plugin-support

Browse Source
This commit is contained in:
BillSchumacher
2023-04-19 17:28:17 -05:00
parent d5523600c7 fa91bc154c
commit 23c650ca10
44 changed files with 698 additions and 3812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
autogpt/agent/agent.py
View File

@@ -3,8 +3,8 @@ from colorama import Fore, Style
from autogpt.app import execute_command, get_command
from autogpt.chat import chat_with_ai, create_chat_message
from autogpt.config import Config
from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques
from autogpt.json_validation.validate_json import validate_json
from autogpt.json_utils.json_fix_llm import fix_json_using_multiple_techniques
from autogpt.json_utils.utilities import validate_json
from autogpt.logs import logger, print_assistant_thoughts
from autogpt.speech import say_text
from autogpt.spinner import Spinner

5
autogpt/app.py
View File

@@ -3,6 +3,7 @@ import json
from typing import Dict, List, NoReturn, Union
from autogpt.agent.agent_manager import AgentManager
from autogpt.commands.analyze_code import analyze_code
from autogpt.commands.audio_text import read_audio_from_file
from autogpt.commands.command import CommandRegistry, command
from autogpt.commands.evaluate_code import evaluate_code
@@ -28,7 +29,7 @@ from autogpt.commands.web_requests import scrape_links, scrape_text
from autogpt.commands.web_selenium import browse_website
from autogpt.commands.write_tests import write_tests
from autogpt.config import Config
from autogpt.json_fixes.parsing import fix_and_parse_json
from autogpt.json_utils.json_fix_llm import fix_and_parse_json
from autogpt.memory import get_memory
from autogpt.processing.text import summarize_text
from autogpt.prompts.generator import PromptGenerator
@@ -141,6 +142,8 @@ def execute_command(
return get_text_summary(arguments["url"], arguments["question"])
elif command_name == "get_hyperlinks":
return get_hyperlinks(arguments["url"])
elif command_name == "analyze_code":
return analyze_code(arguments["code"])
elif command_name == "download_file":
if not CFG.allow_downloads:
return "Error: You do not have user authorization to download files locally."

20
autogpt/cli.py
View File

@@ -70,6 +70,7 @@ def main(
"""
# Put imports inside function to avoid importing everything when starting the CLI
import logging
import sys
from colorama import Fore
@@ -81,7 +82,7 @@ def main(
from autogpt.memory import get_memory
from autogpt.plugins import scan_plugins
from autogpt.prompts.prompt import construct_main_ai_config
from autogpt.utils import get_latest_bulletin
from autogpt.utils import get_current_git_branch, get_latest_bulletin
if ctx.invoked_subcommand is None:
cfg = Config()
@@ -107,6 +108,23 @@ def main(
motd = get_latest_bulletin()
if motd:
logger.typewriter_log("NEWS: ", Fore.GREEN, motd)
git_branch = get_current_git_branch()
if git_branch and git_branch != "stable":
logger.typewriter_log(
"WARNING: ",
Fore.RED,
f"You are running on `{git_branch}` branch "
"- this is not a supported branch.",
)
if sys.version_info < (3, 10):
logger.typewriter_log(
"WARNING: ",
Fore.RED,
"You are running on an older version of Python. "
"Some people have observed problems with certain "
"parts of Auto-GPT with this version. "
"Please consider upgrading to Python 3.10 or higher.",
)
cfg = Config()
cfg.set_plugins(scan_plugins(cfg, cfg.debug_mode))

4
autogpt/commands/evaluate_code.py → autogpt/commands/analyze_code.py
View File

@@ -1,12 +1,10 @@
"""Code evaluation module."""
from __future__ import annotations
from autogpt.commands.command import command
from autogpt.llm_utils import call_ai_function
@command("evaluate_code", "Evaluate Code", '"code": "<full_code_string>"')
def evaluate_code(code: str) -> list[str]:
def analyze_code(code: str) -> list[str]:
"""
A function that takes in a string and returns a response from create chat
completion api call.

6
autogpt/commands/execute_code.py
View File

@@ -89,6 +89,12 @@ def execute_python_file(file: str) -> str:
return logs
except docker.errors.DockerException as e:
print(
"Could not run the script in a container. If you haven't already, please install Docker https://docs.docker.com/get-docker/"
)
return f"Error: {str(e)}"
except Exception as e:
return f"Error: {str(e)}"

3
autogpt/commands/file_operations.py
View File

@@ -3,8 +3,7 @@ from __future__ import annotations
import os
import os.path
from pathlib import Path
from typing import Generator, List
from typing import Generator
import requests
from colorama import Back, Fore

81
autogpt/commands/image_gen.py
View File

@@ -15,11 +15,12 @@ CFG = Config()
@command("generate_image", "Generate Image", '"prompt": "<prompt>"', CFG.image_provider)
def generate_image(prompt: str) -> str:
def generate_image(prompt: str, size: int = 256) -> str:
"""Generate an image from a prompt.
Args:
prompt (str): The prompt to use
size (int, optional): The size of the image. Defaults to 256. (Not supported by HuggingFace)
Returns:
str: The filename of the image
@@ -28,11 +29,14 @@ def generate_image(prompt: str) -> str:
# DALL-E
if CFG.image_provider == "dalle":
return generate_image_with_dalle(prompt, filename)
elif CFG.image_provider == "sd":
return generate_image_with_dalle(prompt, filename, size)
# HuggingFace
elif CFG.image_provider == "huggingface":
return generate_image_with_hf(prompt, filename)
else:
return "No Image Provider Set"
# SD WebUI
elif CFG.image_provider == "sdwebui":
return generate_image_with_sd_webui(prompt, filename, size)
return "No Image Provider Set"
def generate_image_with_hf(prompt: str, filename: str) -> str:
@@ -46,13 +50,16 @@ def generate_image_with_hf(prompt: str, filename: str) -> str:
str: The filename of the image
"""
API_URL = (
"https://api-inference.huggingface.co/models/CompVis/stable-diffusion-v1-4"
f"https://api-inference.huggingface.co/models/{CFG.huggingface_image_model}"
)
if CFG.huggingface_api_token is None:
raise ValueError(
"You need to set your Hugging Face API token in the config file."
)
headers = {"Authorization": f"Bearer {CFG.huggingface_api_token}"}
headers = {
"Authorization": f"Bearer {CFG.huggingface_api_token}",
"X-Use-Cache": "false",
}
response = requests.post(
API_URL,
@@ -82,10 +89,18 @@ def generate_image_with_dalle(prompt: str, filename: str) -> str:
"""
openai.api_key = CFG.openai_api_key
# Check for supported image sizes
if size not in [256, 512, 1024]:
closest = min([256, 512, 1024], key=lambda x: abs(x - size))
print(
f"DALL-E only supports image sizes of 256x256, 512x512, or 1024x1024. Setting to {closest}, was {size}."
)
size = closest
response = openai.Image.create(
prompt=prompt,
n=1,
size="256x256",
size=f"{size}x{size}",
response_format="b64_json",
)
@@ -97,3 +112,53 @@ def generate_image_with_dalle(prompt: str, filename: str) -> str:
png.write(image_data)
return f"Saved to disk:{filename}"
def generate_image_with_sd_webui(
prompt: str,
filename: str,
size: int = 512,
negative_prompt: str = "",
extra: dict = {},
) -> str:
"""Generate an image with Stable Diffusion webui.
Args:
prompt (str): The prompt to use
filename (str): The filename to save the image to
size (int, optional): The size of the image. Defaults to 256.
negative_prompt (str, optional): The negative prompt to use. Defaults to "".
extra (dict, optional): Extra parameters to pass to the API. Defaults to {}.
Returns:
str: The filename of the image
"""
# Create a session and set the basic auth if needed
s = requests.Session()
if CFG.sd_webui_auth:
username, password = CFG.sd_webui_auth.split(":")
s.auth = (username, password or "")
# Generate the images
response = requests.post(
f"{CFG.sd_webui_url}/sdapi/v1/txt2img",
json={
"prompt": prompt,
"negative_prompt": negative_prompt,
"sampler_index": "DDIM",
"steps": 20,
"cfg_scale": 7.0,
"width": size,
"height": size,
"n_iter": 1,
**extra,
},
)
print(f"Image Generated for prompt:{prompt}")
# Save the image to disk
response = response.json()
b64 = b64decode(response["images"][0].split(",", 1)[0])
image = Image.open(io.BytesIO(b64))
image.save(path_in_workspace(filename))
return f"Saved to disk:{filename}"

20
autogpt/config/config.py
View File

@@ -33,14 +33,20 @@ class Config(metaclass=Singleton):
self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4")
self.fast_token_limit = int(os.getenv("FAST_TOKEN_LIMIT", 4000))
self.smart_token_limit = int(os.getenv("SMART_TOKEN_LIMIT", 8000))
self.browse_chunk_max_length = int(os.getenv("BROWSE_CHUNK_MAX_LENGTH", 8192))
self.browse_chunk_max_length = int(os.getenv("BROWSE_CHUNK_MAX_LENGTH", 3000))
self.browse_spacy_language_model = os.getenv(
"BROWSE_SPACY_LANGUAGE_MODEL", "en_core_web_sm"
)
self.openai_api_key = os.getenv("OPENAI_API_KEY")
self.temperature = float(os.getenv("TEMPERATURE", "1"))
self.temperature = float(os.getenv("TEMPERATURE", "0"))
self.use_azure = os.getenv("USE_AZURE") == "True"
self.execute_local_commands = (
os.getenv("EXECUTE_LOCAL_COMMANDS", "False") == "True"
)
self.restrict_to_workspace = (
os.getenv("RESTRICT_TO_WORKSPACE", "True") == "True"
)
if self.use_azure:
self.load_azure_config()
@@ -84,10 +90,16 @@ class Config(metaclass=Singleton):
self.milvus_collection = os.getenv("MILVUS_COLLECTION", "autogpt")
self.image_provider = os.getenv("IMAGE_PROVIDER")
self.image_size = int(os.getenv("IMAGE_SIZE", 256))
self.huggingface_api_token = os.getenv("HUGGINGFACE_API_TOKEN")
self.huggingface_image_model = os.getenv(
"HUGGINGFACE_IMAGE_MODEL", "CompVis/stable-diffusion-v1-4"
)
self.huggingface_audio_to_text_model = os.getenv(
"HUGGINGFACE_AUDIO_TO_TEXT_MODEL"
)
self.sd_webui_url = os.getenv("SD_WEBUI_URL", "http://localhost:7860")
self.sd_webui_auth = os.getenv("SD_WEBUI_AUTH")
# Selenium browser settings
self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome")
@@ -150,7 +162,7 @@ class Config(metaclass=Singleton):
else:
return ""
AZURE_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "..", "azure.yaml")
AZURE_CONFIG_FILE = os.path.join(os.path.dirname(__file__), "../..", "azure.yaml")
def load_azure_config(self, config_file: str = AZURE_CONFIG_FILE) -> None:
"""
@@ -173,7 +185,7 @@ class Config(metaclass=Singleton):
self.openai_api_version = (
config_params.get("azure_api_version") or "2023-03-15-preview"
)
self.azure_model_to_deployment_id_map = config_params.get("azure_model_map", [])
self.azure_model_to_deployment_id_map = config_params.get("azure_model_map", {})
def set_continuous_mode(self, value: bool) -> None:
"""Set the continuous mode value."""

53
autogpt/json_fixes/auto_fix.py
View File

@@ -1,53 +0,0 @@
"""This module contains the function to fix JSON strings using GPT-3."""
import json
from autogpt.config import Config
from autogpt.llm_utils import call_ai_function
from autogpt.logs import logger
CFG = Config()
def fix_json(json_string: str, schema: str) -> str:
"""Fix the given JSON string to make it parseable and fully compliant with
the provided schema.
Args:
json_string (str): The JSON string to fix.
schema (str): The schema to use to fix the JSON.
Returns:
str: The fixed JSON string.
"""
# Try to fix the JSON using GPT:
function_string = "def fix_json(json_string: str, schema:str=None) -> str:"
args = [f"'''{json_string}'''", f"'''{schema}'''"]
description_string = (
"This function takes a JSON string and ensures that it"
" is parseable and fully compliant with the provided schema. If an object"
" or field specified in the schema isn't contained within the correct JSON,"
" it is omitted. The function also escapes any double quotes within JSON"
" string values to ensure that they are valid. If the JSON string contains"
" any None or NaN values, they are replaced with null before being parsed."
)
# If it doesn't already start with a "`", add one:
if not json_string.startswith("`"):
json_string = "```json\n" + json_string + "\n```"
result_string = call_ai_function(
function_string, args, description_string, model=CFG.fast_llm_model
)
logger.debug("------------ JSON FIX ATTEMPT ---------------")
logger.debug(f"Original JSON: {json_string}")
logger.debug("-----------")
logger.debug(f"Fixed JSON: {result_string}")
logger.debug("----------- END OF FIX ATTEMPT ----------------")
try:
json.loads(result_string) # just check the validity
return result_string
except json.JSONDecodeError:
# Get the call stack:
# import traceback
# call_stack = traceback.format_exc()
# print(f"Failed to fix JSON: '{json_string}' "+call_stack)
return "failed"

37
autogpt/json_fixes/bracket_termination.py
View File

@@ -1,37 +0,0 @@
"""Fix JSON brackets."""
from __future__ import annotations
import contextlib
import json
from typing import Optional
from autogpt.config import Config
CFG = Config()
def balance_braces(json_string: str) -> Optional[str]:
"""
Balance the braces in a JSON string.
Args:
json_string (str): The JSON string.
Returns:
str: The JSON string with braces balanced.
"""
open_braces_count = json_string.count("{")
close_braces_count = json_string.count("}")
while open_braces_count > close_braces_count:
json_string += "}"
close_braces_count += 1
while close_braces_count > open_braces_count:
json_string = json_string.rstrip("}")
close_braces_count -= 1
with contextlib.suppress(json.JSONDecodeError):
json.loads(json_string)
return json_string

33
autogpt/json_fixes/escaping.py
View File

@@ -1,33 +0,0 @@
""" Fix invalid escape sequences in JSON strings. """
import json
from autogpt.config import Config
from autogpt.json_fixes.utilities import extract_char_position
CFG = Config()
def fix_invalid_escape(json_to_load: str, error_message: str) -> str:
"""Fix invalid escape sequences in JSON strings.
Args:
json_to_load (str): The JSON string.
error_message (str): The error message from the JSONDecodeError
exception.
Returns:
str: The JSON string with invalid escape sequences fixed.
"""
while error_message.startswith("Invalid \\escape"):
bad_escape_location = extract_char_position(error_message)
json_to_load = (
json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :]
)
try:
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error - fix invalid escape", e)
error_message = str(e)
return json_to_load

33
autogpt/json_fixes/master_json_fix_method.py
View File

@@ -1,33 +0,0 @@
from typing import Any, Dict
from autogpt.config import Config
from autogpt.logs import logger
from autogpt.speech import say_text
CFG = Config()
def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]:
from autogpt.json_fixes.parsing import (
attempt_to_fix_json_by_finding_outermost_brackets,
fix_and_parse_json,
)
# Parse and print Assistant response
assistant_reply_json = fix_and_parse_json(assistant_reply)
if assistant_reply_json == {}:
assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets(
assistant_reply
)
if assistant_reply_json != {}:
return assistant_reply_json
logger.error(
"Error: The following AI output couldn't be converted to a JSON:\n",
assistant_reply,
)
if CFG.speak_mode:
say_text("I have received an invalid JSON response from the OpenAI API.")
return {}

27
autogpt/json_fixes/missing_quotes.py
View File

@@ -1,27 +0,0 @@
"""Fix quotes in a JSON string."""
import json
import re
def add_quotes_to_property_names(json_string: str) -> str:
"""
Add quotes to property names in a JSON string.
Args:
json_string (str): The JSON string.
Returns:
str: The JSON string with quotes added to property names.
"""
def replace_func(match: re.Match) -> str:
return f'"{match[1]}":'
property_name_pattern = re.compile(r"(\w+):")
corrected_json_string = property_name_pattern.sub(replace_func, json_string)
try:
json.loads(corrected_json_string)
return corrected_json_string
except json.JSONDecodeError as e:
raise e

20
autogpt/json_fixes/utilities.py
View File

@@ -1,20 +0,0 @@
"""Utilities for the json_fixes package."""
import re
def extract_char_position(error_message: str) -> int:
"""Extract the character position from the JSONDecodeError message.
Args:
error_message (str): The error message from the JSONDecodeError
exception.
Returns:
int: The character position.
"""
char_pattern = re.compile(r"\(char (\d+)\)")
if match := char_pattern.search(error_message):
return int(match[1])
else:
raise ValueError("Character position not found in the error message.")

0
autogpt/json_fixes/__init__.py → autogpt/json_utils/__init__.py
View File

124
autogpt/json_utils/json_fix_general.py Normal file
View File

@@ -0,0 +1,124 @@
"""This module contains functions to fix JSON strings using general programmatic approaches, suitable for addressing
common JSON formatting issues."""
from __future__ import annotations
import contextlib
import json
import re
from typing import Optional
from autogpt.config import Config
from autogpt.json_utils.utilities import extract_char_position
CFG = Config()
def fix_invalid_escape(json_to_load: str, error_message: str) -> str:
"""Fix invalid escape sequences in JSON strings.
Args:
json_to_load (str): The JSON string.
error_message (str): The error message from the JSONDecodeError
exception.
Returns:
str: The JSON string with invalid escape sequences fixed.
"""
while error_message.startswith("Invalid \\escape"):
bad_escape_location = extract_char_position(error_message)
json_to_load = (
json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :]
)
try:
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error - fix invalid escape", e)
error_message = str(e)
return json_to_load
def balance_braces(json_string: str) -> Optional[str]:
"""
Balance the braces in a JSON string.
Args:
json_string (str): The JSON string.
Returns:
str: The JSON string with braces balanced.
"""
open_braces_count = json_string.count("{")
close_braces_count = json_string.count("}")
while open_braces_count > close_braces_count:
json_string += "}"
close_braces_count += 1
while close_braces_count > open_braces_count:
json_string = json_string.rstrip("}")
close_braces_count -= 1
with contextlib.suppress(json.JSONDecodeError):
json.loads(json_string)
return json_string
def add_quotes_to_property_names(json_string: str) -> str:
"""
Add quotes to property names in a JSON string.
Args:
json_string (str): The JSON string.
Returns:
str: The JSON string with quotes added to property names.
"""
def replace_func(match: re.Match) -> str:
return f'"{match[1]}":'
property_name_pattern = re.compile(r"(\w+):")
corrected_json_string = property_name_pattern.sub(replace_func, json_string)
try:
json.loads(corrected_json_string)
return corrected_json_string
except json.JSONDecodeError as e:
raise e
def correct_json(json_to_load: str) -> str:
"""
Correct common JSON errors.
Args:
json_to_load (str): The JSON string.
"""
try:
if CFG.debug_mode:
print("json", json_to_load)
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error", e)
error_message = str(e)
if error_message.startswith("Invalid \\escape"):
json_to_load = fix_invalid_escape(json_to_load, error_message)
if error_message.startswith(
"Expecting property name enclosed in double quotes"
):
json_to_load = add_quotes_to_property_names(json_to_load)
try:
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error - add quotes", e)
error_message = str(e)
if balanced_str := balance_braces(json_to_load):
return balanced_str
return json_to_load

114
autogpt/json_fixes/parsing.py → autogpt/json_utils/json_fix_llm.py
View File

@@ -1,23 +1,20 @@
"""Fix and parse JSON strings."""
"""This module contains functions to fix JSON strings generated by LLM models, such as ChatGPT, using the assistance
of the ChatGPT API or LLM models."""
from __future__ import annotations
import contextlib
import json
from typing import Any, Dict, Union
from typing import Any, Dict
from colorama import Fore
from regex import regex
from autogpt.config import Config
from autogpt.json_fixes.auto_fix import fix_json
from autogpt.json_fixes.bracket_termination import balance_braces
from autogpt.json_fixes.escaping import fix_invalid_escape
from autogpt.json_fixes.missing_quotes import add_quotes_to_property_names
from autogpt.json_utils.json_fix_general import correct_json
from autogpt.llm_utils import call_ai_function
from autogpt.logs import logger
from autogpt.speech import say_text
CFG = Config()
JSON_SCHEMA = """
{
"command": {
@@ -37,39 +34,82 @@ JSON_SCHEMA = """
}
"""
CFG = Config()
def auto_fix_json(json_string: str, schema: str) -> str:
"""Fix the given JSON string to make it parseable and fully compliant with
the provided schema using GPT-3.
def correct_json(json_to_load: str) -> str:
"""
Correct common JSON errors.
Args:
json_to_load (str): The JSON string.
json_string (str): The JSON string to fix.
schema (str): The schema to use to fix the JSON.
Returns:
str: The fixed JSON string.
"""
# Try to fix the JSON using GPT:
function_string = "def fix_json(json_string: str, schema:str=None) -> str:"
args = [f"'''{json_string}'''", f"'''{schema}'''"]
description_string = (
"This function takes a JSON string and ensures that it"
" is parseable and fully compliant with the provided schema. If an object"
" or field specified in the schema isn't contained within the correct JSON,"
" it is omitted. The function also escapes any double quotes within JSON"
" string values to ensure that they are valid. If the JSON string contains"
" any None or NaN values, they are replaced with null before being parsed."
)
# If it doesn't already start with a "`", add one:
if not json_string.startswith("`"):
json_string = "```json\n" + json_string + "\n```"
result_string = call_ai_function(
function_string, args, description_string, model=CFG.fast_llm_model
)
logger.debug("------------ JSON FIX ATTEMPT ---------------")
logger.debug(f"Original JSON: {json_string}")
logger.debug("-----------")
logger.debug(f"Fixed JSON: {result_string}")
logger.debug("----------- END OF FIX ATTEMPT ----------------")
try:
if CFG.debug_mode:
print("json", json_to_load)
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error", e)
error_message = str(e)
if error_message.startswith("Invalid \\escape"):
json_to_load = fix_invalid_escape(json_to_load, error_message)
if error_message.startswith(
"Expecting property name enclosed in double quotes"
):
json_to_load = add_quotes_to_property_names(json_to_load)
try:
json.loads(json_to_load)
return json_to_load
except json.JSONDecodeError as e:
if CFG.debug_mode:
print("json loads error - add quotes", e)
error_message = str(e)
if balanced_str := balance_braces(json_to_load):
return balanced_str
return json_to_load
json.loads(result_string) # just check the validity
return result_string
except json.JSONDecodeError: # noqa: E722
# Get the call stack:
# import traceback
# call_stack = traceback.format_exc()
# print(f"Failed to fix JSON: '{json_string}' "+call_stack)
return "failed"
def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]:
"""Fix the given JSON string to make it parseable and fully compliant with two techniques.
Args:
json_string (str): The JSON string to fix.
Returns:
str: The fixed JSON string.
"""
# Parse and print Assistant response
assistant_reply_json = fix_and_parse_json(assistant_reply)
if assistant_reply_json == {}:
assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets(
assistant_reply
)
if assistant_reply_json != {}:
return assistant_reply_json
logger.error(
"Error: The following AI output couldn't be converted to a JSON:\n",
assistant_reply,
)
if CFG.speak_mode:
say_text("I have received an invalid JSON response from the OpenAI API.")
return {}
def fix_and_parse_json(
@@ -136,7 +176,7 @@ def try_ai_fix(
" slightly."
)
# Now try to fix this up using the ai_functions
ai_fixed_json = fix_json(json_to_load, JSON_SCHEMA)
ai_fixed_json = auto_fix_json(json_to_load, JSON_SCHEMA)
if ai_fixed_json != "failed":
return json.loads(ai_fixed_json)

0
autogpt/json_schemas/llm_response_format_1.json → autogpt/json_utils/llm_response_format_1.json
View File

22
autogpt/json_validation/validate_json.py → autogpt/json_utils/utilities.py
View File

@@ -1,4 +1,6 @@
"""Utilities for the json_fixes package."""
import json
import re
from jsonschema import Draft7Validator
@@ -8,13 +10,31 @@ from autogpt.logs import logger
CFG = Config()
def extract_char_position(error_message: str) -> int:
"""Extract the character position from the JSONDecodeError message.
Args:
error_message (str): The error message from the JSONDecodeError
exception.
Returns:
int: The character position.
"""
char_pattern = re.compile(r"\(char (\d+)\)")
if match := char_pattern.search(error_message):
return int(match[1])
else:
raise ValueError("Character position not found in the error message.")
def validate_json(json_object: object, schema_name: object) -> object:
"""
:type schema_name: object
:param schema_name:
:type json_object: object
"""
with open(f"autogpt/json_schemas/{schema_name}.json", "r") as f:
with open(f"autogpt/json_utils/{schema_name}.json", "r") as f:
schema = json.load(f)
validator = Draft7Validator(schema)

4
autogpt/logs.py
View File

@@ -204,10 +204,10 @@ logger = Logger()
def print_assistant_thoughts(ai_name, assistant_reply):
"""Prints the assistant's thoughts to the console"""
from autogpt.json_fixes.bracket_termination import (
from autogpt.json_utils.json_fix_llm import (
attempt_to_fix_json_by_finding_outermost_brackets,
fix_and_parse_json,
)
from autogpt.json_fixes.parsing import fix_and_parse_json
try:
try:

74
autogpt/processing/text.py
View File

@@ -1,8 +1,10 @@
"""Text processing functions"""
from typing import Dict, Generator, Optional
import spacy
from selenium.webdriver.remote.webdriver import WebDriver
from autogpt import token_counter
from autogpt.config import Config
from autogpt.llm_utils import create_chat_completion
from autogpt.memory import get_memory
@@ -11,7 +13,12 @@ CFG = Config()
MEMORY = get_memory(CFG)
def split_text(text: str, max_length: int = 8192) -> Generator[str, None, None]:
def split_text(
text: str,
max_length: int = CFG.browse_chunk_max_length,
model: str = CFG.fast_llm_model,
question: str = "",
) -> Generator[str, None, None]:
"""Split text into chunks of a maximum length
Args:
@@ -24,21 +31,46 @@ def split_text(text: str, max_length: int = 8192) -> Generator[str, None, None]:
Raises:
ValueError: If the text is longer than the maximum length
"""
paragraphs = text.split("\n")
current_length = 0
flatened_paragraphs = " ".join(text.split("\n"))
nlp = spacy.load(CFG.browse_spacy_language_model)
nlp.add_pipe("sentencizer")
doc = nlp(flatened_paragraphs)
sentences = [sent.text.strip() for sent in doc.sents]
current_chunk = []
for paragraph in paragraphs:
if current_length + len(paragraph) + 1 <= max_length:
current_chunk.append(paragraph)
current_length += len(paragraph) + 1
for sentence in sentences:
message_with_additional_sentence = [
create_message(" ".join(current_chunk) + " " + sentence, question)
]
expected_token_usage = (
token_usage_of_chunk(messages=message_with_additional_sentence, model=model)
+ 1
)
if expected_token_usage <= max_length:
current_chunk.append(sentence)
else:
yield "\n".join(current_chunk)
current_chunk = [paragraph]
current_length = len(paragraph) + 1
yield " ".join(current_chunk)
current_chunk = [sentence]
message_this_sentence_only = [
create_message(" ".join(current_chunk), question)
]
expected_token_usage = (
token_usage_of_chunk(messages=message_this_sentence_only, model=model)
+ 1
)
if expected_token_usage > max_length:
raise ValueError(
f"Sentence is too long in webpage: {expected_token_usage} tokens."
)
if current_chunk:
yield "\n".join(current_chunk)
yield " ".join(current_chunk)
def token_usage_of_chunk(messages, model):
return token_counter.count_message_tokens(messages, model)
def summarize_text(
@@ -58,11 +90,16 @@ def summarize_text(
if not text:
return "Error: No text to summarize"
model = CFG.fast_llm_model
text_length = len(text)
print(f"Text length: {text_length} characters")
summaries = []
chunks = list(split_text(text))
chunks = list(
split_text(
text, max_length=CFG.browse_chunk_max_length, model=model, question=question
),
)
scroll_ratio = 1 / len(chunks)
for i, chunk in enumerate(chunks):
@@ -74,15 +111,20 @@ def summarize_text(
MEMORY.add(memory_to_add)
print(f"Summarizing chunk {i + 1} / {len(chunks)}")
messages = [create_message(chunk, question)]
tokens_for_chunk = token_counter.count_message_tokens(messages, model)
print(
f"Summarizing chunk {i + 1} / {len(chunks)} of length {len(chunk)} characters, or {tokens_for_chunk} tokens"
)
summary = create_chat_completion(
model=CFG.fast_llm_model,
model=model,
messages=messages,
)
summaries.append(summary)
print(f"Added chunk {i + 1} summary to memory")
print(
f"Added chunk {i + 1} summary to memory, of length {len(summary)} characters"
)
memory_to_add = f"Source: {url}\n" f"Content summary part#{i + 1}: {summary}"
@@ -94,7 +136,7 @@ def summarize_text(
messages = [create_message(combined_summary, question)]
return create_chat_completion(
model=CFG.fast_llm_model,
model=model,
messages=messages,
)

10
autogpt/utils.py
View File

@@ -3,6 +3,7 @@ import os
import requests
import yaml
from colorama import Fore
from git import Repo
def clean_input(prompt: str = ""):
@@ -53,6 +54,15 @@ def get_bulletin_from_web() -> str:
return ""
def get_current_git_branch() -> str:
try:
repo = Repo(search_parent_directories=True)
branch = repo.active_branch
return branch.name
except:
return ""
def get_latest_bulletin() -> str:
exists = os.path.exists("CURRENT_BULLETIN.md")
current_bulletin = ""

9
autogpt/workspace.py
View File

@@ -3,6 +3,10 @@ from __future__ import annotations
import os
from pathlib import Path
from autogpt.config import Config
CFG = Config()
# Set a dedicated folder for file I/O
WORKSPACE_PATH = Path(os.getcwd()) / "auto_gpt_workspace"
@@ -33,11 +37,12 @@ def safe_path_join(base: Path, *paths: str | Path) -> Path:
Returns:
Path: The joined path
"""
base = base.resolve()
joined_path = base.joinpath(*paths).resolve()
if not joined_path.is_relative_to(base):
if CFG.restrict_to_workspace and not joined_path.is_relative_to(base):
raise ValueError(
f"Attempted to access path '{joined_path}' outside of working directory '{base}'."
f"Attempted to access path '{joined_path}' outside of workspace '{base}'."
)
return joined_path