github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-01-08 22:58:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

Rework plugin config to be file-based (#4673)

Browse Source
This commit is contained in:
Erik Peterson
2023-06-13 20:54:55 -07:00
committed by GitHub
parent 0c8f2cfd1c
commit 49d1a5a17b
10 changed files with 263 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.env.template
View File

@@ -19,6 +19,9 @@ OPENAI_API_KEY=your-openai-api-key
## AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml)
# AI_SETTINGS_FILE=ai_settings.yaml
## PLUGINS_CONFIG_FILE - The path to the plugins_config.yaml file (Default plugins_config.yaml)
# PLUGINS_CONFIG_FILE=plugins_config.yaml
## PROMPT_SETTINGS_FILE - Specifies which Prompt Settings file to use (defaults to prompt_settings.yaml)
# PROMPT_SETTINGS_FILE=prompt_settings.yaml
@@ -38,7 +41,6 @@ OPENAI_API_KEY=your-openai-api-key
## DISABLED_COMMAND_CATEGORIES - The list of categories of commands that are disabled (Default: None)
# DISABLED_COMMAND_CATEGORIES=
################################################################################
### LLM PROVIDER
################################################################################
@@ -194,16 +196,6 @@ OPENAI_API_KEY=your-openai-api-key
## ELEVENLABS_VOICE_ID - Eleven Labs voice ID (Example: None)
# ELEVENLABS_VOICE_ID=
################################################################################
### ALLOWLISTED PLUGINS
################################################################################
## ALLOWLISTED_PLUGINS - Sets the listed plugins that are allowed (Default: None)
# ALLOWLISTED_PLUGINS=
## DENYLISTED_PLUGINS - Sets the listed plugins that are not allowed (Default: None)
# DENYLISTED_PLUGINS=
################################################################################
### CHAT MESSAGES
################################################################################

18
autogpt/config/config.py
View File

@@ -7,6 +7,7 @@ import yaml
from auto_gpt_plugin_template import AutoGPTPluginTemplate
from colorama import Fore
import autogpt
from autogpt.singleton import Singleton
@@ -156,20 +157,37 @@ class Config(metaclass=Singleton):
self.plugins: List[AutoGPTPluginTemplate] = []
self.plugins_openai = []
# Deprecated. Kept for backwards-compatibility. Will remove in a future version.
plugins_allowlist = os.getenv("ALLOWLISTED_PLUGINS")
if plugins_allowlist:
self.plugins_allowlist = plugins_allowlist.split(",")
else:
self.plugins_allowlist = []
# Deprecated. Kept for backwards-compatibility. Will remove in a future version.
plugins_denylist = os.getenv("DENYLISTED_PLUGINS")
if plugins_denylist:
self.plugins_denylist = plugins_denylist.split(",")
else:
self.plugins_denylist = []
# Avoid circular imports
from autogpt.plugins import DEFAULT_PLUGINS_CONFIG_FILE
self.plugins_config_file = os.getenv(
"PLUGINS_CONFIG_FILE", DEFAULT_PLUGINS_CONFIG_FILE
)
self.load_plugins_config()
self.chat_messages_enabled = os.getenv("CHAT_MESSAGES_ENABLED") == "True"
def load_plugins_config(self) -> "autogpt.plugins.PluginsConfig":
# Avoid circular import
from autogpt.plugins.plugins_config import PluginsConfig
self.plugins_config = PluginsConfig.load_config(global_config=self)
return self.plugins_config
def get_azure_deployment_id_for_model(self, model: str) -> str:
"""
Returns the relevant deployment id for the model specified.

59
autogpt/plugins.py → autogpt/plugins/__init__.py
View File

@@ -16,10 +16,14 @@ import requests
from auto_gpt_plugin_template import AutoGPTPluginTemplate
from openapi_python_client.config import Config as OpenAPIConfig
from autogpt.config import Config
from autogpt.config.config import Config
from autogpt.logs import logger
from autogpt.models.base_open_ai_plugin import BaseOpenAIPlugin
DEFAULT_PLUGINS_CONFIG_FILE = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "..", "plugins_config.yaml"
)
def inspect_zip_for_modules(zip_path: str, debug: bool = False) -> list[str]:
"""
@@ -215,9 +219,7 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
loaded_plugins = []
# Generic plugins
plugins_path_path = Path(cfg.plugins_dir)
logger.debug(f"Allowlisted Plugins: {cfg.plugins_allowlist}")
logger.debug(f"Denylisted Plugins: {cfg.plugins_denylist}")
plugins_config = cfg.plugins_config
# Directory-based plugins
for plugin_path in [f.path for f in os.scandir(cfg.plugins_dir) if f.is_dir()]:
@@ -232,11 +234,14 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
__import__(qualified_module_name)
plugin = sys.modules[qualified_module_name]
if not plugins_config.is_enabled(plugin_module_name):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue
for _, class_obj in inspect.getmembers(plugin):
if (
hasattr(class_obj, "_abc_impl")
and AutoGPTPluginTemplate in class_obj.__bases__
and denylist_allowlist_check(plugin_module_name, cfg)
):
loaded_plugins.append(class_obj())
@@ -249,6 +254,12 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
logger.debug(f"Plugin: {plugin} Module: {module}")
zipped_package = zipimporter(str(plugin))
zipped_module = zipped_package.load_module(str(module.parent))
plugin_module_name = zipped_module.__name__.split(os.path.sep)[-1]
if not plugins_config.is_enabled(plugin_module_name):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue
for key in dir(zipped_module):
if key.startswith("__"):
continue
@@ -257,7 +268,6 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
if (
"_abc_impl" in a_keys
and a_module.__name__ != "AutoGPTPluginTemplate"
and denylist_allowlist_check(a_module.__name__, cfg)
):
loaded_plugins.append(a_module())
@@ -269,40 +279,15 @@ def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate
manifests_specs, cfg, debug
)
for url, openai_plugin_meta in manifests_specs_clients.items():
if denylist_allowlist_check(url, cfg):
plugin = BaseOpenAIPlugin(openai_plugin_meta)
loaded_plugins.append(plugin)
if not plugins_config.is_enabled(url):
logger.warn(f"Plugin {plugin_module_name} found but not configured")
continue
plugin = BaseOpenAIPlugin(openai_plugin_meta)
loaded_plugins.append(plugin)
if loaded_plugins:
logger.info(f"\nPlugins found: {len(loaded_plugins)}\n" "--------------------")
for plugin in loaded_plugins:
logger.info(f"{plugin._name}: {plugin._version} - {plugin._description}")
return loaded_plugins
def denylist_allowlist_check(plugin_name: str, cfg: Config) -> bool:
"""Check if the plugin is in the allowlist or denylist.
Args:
plugin_name (str): Name of the plugin.
cfg (Config): Config object.
Returns:
True or False
"""
logger.debug(f"Checking if plugin {plugin_name} should be loaded")
if (
plugin_name in cfg.plugins_denylist
or "all" in cfg.plugins_denylist
or "none" in cfg.plugins_allowlist
):
logger.debug(f"Not loading plugin {plugin_name} as it was in the denylist.")
return False
if plugin_name in cfg.plugins_allowlist or "all" in cfg.plugins_allowlist:
logger.debug(f"Loading plugin {plugin_name} as it was in the allowlist.")
return True
ack = input(
f"WARNING: Plugin {plugin_name} found. But not in the"
f" allowlist... Load? ({cfg.authorise_key}/{cfg.exit_key}): "
)
return ack.lower() == cfg.authorise_key

14
autogpt/plugins/plugin_config.py Normal file
View File

@@ -0,0 +1,14 @@
from typing import Any
class PluginConfig:
"""Class for holding configuration of a single plugin"""
def __init__(self, name: str, enabled: bool = False, config: dict[str, Any] = None):
self.name = name
self.enabled = enabled
# Arbitray config options for this plugin. API keys or plugin-specific options live here.
self.config = config or {}
def __repr__(self):
return f"PluginConfig('{self.name}', {self.enabled}, {str(self.config)}"

81
autogpt/plugins/plugins_config.py Normal file
View File

@@ -0,0 +1,81 @@
import os
from typing import Any, Union
import yaml
from autogpt.config.config import Config
from autogpt.logs import logger
from autogpt.plugins.plugin_config import PluginConfig
class PluginsConfig:
"""Class for holding configuration of all plugins"""
def __init__(self, plugins_config: dict[str, Any]):
self.plugins = {}
for name, plugin in plugins_config.items():
if type(plugin) == dict:
self.plugins[name] = PluginConfig(
name,
plugin.get("enabled", False),
plugin.get("config", {}),
)
elif type(plugin) == PluginConfig:
self.plugins[name] = plugin
else:
raise ValueError(f"Invalid plugin config data type: {type(plugin)}")
def __repr__(self):
return f"PluginsConfig({self.plugins})"
def get(self, name: str) -> Union[PluginConfig, None]:
return self.plugins.get(name)
def is_enabled(self, name) -> bool:
plugin_config = self.plugins.get(name)
return plugin_config and plugin_config.enabled
@classmethod
def load_config(cls, global_config: Config) -> "PluginsConfig":
empty_config = cls({})
try:
config_data = cls.deserialize_config_file(global_config=global_config)
if type(config_data) != dict:
logger.error(
f"Expected plugins config to be a dict, got {type(config_data)}, continuing without plugins"
)
return empty_config
return cls(config_data)
except BaseException as e:
logger.error(
f"Plugin config is invalid, continuing without plugins. Error: {e}"
)
return empty_config
@classmethod
def deserialize_config_file(cls, global_config: Config) -> dict[str, Any]:
plugins_config_path = global_config.plugins_config_file
if not os.path.exists(plugins_config_path):
logger.warn("plugins_config.yaml does not exist, creating base config.")
cls.create_empty_plugins_config(global_config=global_config)
with open(plugins_config_path, "r") as f:
return yaml.load(f, Loader=yaml.FullLoader)
@staticmethod
def create_empty_plugins_config(global_config: Config):
"""Create an empty plugins_config.yaml file. Fill it with values from old env variables."""
base_config = {}
# Backwards-compatibility shim
for plugin_name in global_config.plugins_denylist:
base_config[plugin_name] = {"enabled": False, "config": {}}
for plugin_name in global_config.plugins_allowlist:
base_config[plugin_name] = {"enabled": True, "config": {}}
with open(global_config.plugins_config_file, "w+") as f:
f.write(yaml.dump(base_config))
return base_config

3
docs/configuration/options.md
View File

@@ -5,13 +5,11 @@ Configuration is controlled through the `Config` object. You can set configurati
## Environment Variables
- `AI_SETTINGS_FILE`: Location of AI Settings file. Default: ai_settings.yaml
- `ALLOWLISTED_PLUGINS`: List of plugins allowed. Optional.
- `AUDIO_TO_TEXT_PROVIDER`: Audio To Text Provider. Only option currently is `huggingface`. Default: huggingface
- `AUTHORISE_COMMAND_KEY`: Key response accepted when authorising commands. Default: y
- `BROWSE_CHUNK_MAX_LENGTH`: When browsing website, define the length of chunks to summarize. Default: 3000
- `BROWSE_SPACY_LANGUAGE_MODEL`: [spaCy language model](https://spacy.io/usage/models) to use when creating chunks. Default: en_core_web_sm
- `CHAT_MESSAGES_ENABLED`: Enable chat messages. Optional
- `DENYLISTED_PLUGINS`: List of plugins not allowed. Optional.
- `DISABLED_COMMAND_CATEGORIES`: Command categories to disable. Command categories are Python module names, e.g. autogpt.commands.analyze_code. See the directory `autogpt/commands` in the source for all command modules. Default: None
- `ELEVENLABS_API_KEY`: ElevenLabs API Key. Optional.
- `ELEVENLABS_VOICE_ID`: ElevenLabs Voice ID. Optional.
@@ -34,6 +32,7 @@ Configuration is controlled through the `Config` object. You can set configurati
- `OPENAI_API_KEY`: *REQUIRED*- Your [OpenAI API Key](https://platform.openai.com/account/api-keys).
- `OPENAI_ORGANIZATION`: Organization ID in OpenAI. Optional.
- `PLAIN_OUTPUT`: Plain output, which disables the spinner. Default: False
- `PLUGINS_CONFIG_FILE`: Path of plugins_config.yaml file. Default: plugins_config.yaml
- `PROMPT_SETTINGS_FILE`: Location of Prompt Settings file. Default: prompt_settings.yaml
- `REDIS_HOST`: Redis Host. Default: localhost
- `REDIS_PASSWORD`: Redis Password. Optional. Default:

12
docs/plugins.md
View File

@@ -2,6 +2,18 @@
⚠️💀 **WARNING** 💀⚠️: Review the code of any plugin you use thoroughly, as plugins can execute any Python code, potentially leading to malicious activities, such as stealing your API keys.
To configure plugins, you can create or edit the `plugins_config.yaml` file in the root directory of Auto-GPT. This file allows you to enable or disable plugins as desired. For specific configuration instructions, please refer to the documentation provided for each plugin. The file should be formatted in YAML. Here is an example for your reference:
```yaml
plugin_a:
config:
api_key: my-api-key
enabled: false
plugin_b:
config: {}
enabled: true
```
See our [Plugins Repo](https://github.com/Significant-Gravitas/Auto-GPT-Plugins) for more info on how to install all the amazing plugins the community has built!
Alternatively, developers can use the [Auto-GPT Plugin Template](https://github.com/Significant-Gravitas/Auto-GPT-Plugin-Template) as a starting point for creating your own plugins.

20
tests/conftest.py
View File

@@ -1,7 +1,9 @@
import os
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
import yaml
from pytest_mock import MockerFixture
from autogpt.agent.agent import Agent
@@ -32,9 +34,25 @@ def workspace(workspace_root: Path) -> Workspace:
return Workspace(workspace_root, restrict_to_workspace=True)
@pytest.fixture
def temp_plugins_config_file():
"""Create a plugins_config.yaml file in a temp directory so that it doesn't mess with existing ones"""
config_directory = TemporaryDirectory()
config_file = os.path.join(config_directory.name, "plugins_config.yaml")
with open(config_file, "w+") as f:
f.write(yaml.dump({}))
yield config_file
@pytest.fixture()
def config(mocker: MockerFixture, workspace: Workspace) -> Config:
def config(
temp_plugins_config_file: str, mocker: MockerFixture, workspace: Workspace
) -> Config:
config = Config()
config.plugins_dir = "tests/unit/data/test_plugins"
config.plugins_config_file = temp_plugins_config_file
config.load_plugins_config()
# Do a little setup and teardown since the config object is a singleton
mocker.patch.multiple(

71
tests/integration/test_plugins.py
View File

@@ -1,71 +0,0 @@
import pytest
from autogpt.config import Config
from autogpt.plugins import scan_plugins
PLUGINS_TEST_DIR = "tests/unit/data/test_plugins"
PLUGIN_TEST_OPENAI = "https://weathergpt.vercel.app/"
@pytest.fixture
def mock_config_denylist_allowlist_check():
class MockConfig:
"""Mock config object for testing the denylist_allowlist_check function"""
plugins_denylist = ["BadPlugin"]
plugins_allowlist = ["GoodPlugin"]
authorise_key = "y"
exit_key = "n"
return MockConfig()
@pytest.fixture
def config_with_plugins():
"""Mock config object for testing the scan_plugins function"""
# Test that the function returns the correct number of plugins
cfg = Config()
cfg.plugins_dir = PLUGINS_TEST_DIR
cfg.plugins_openai = ["https://weathergpt.vercel.app/"]
return cfg
@pytest.fixture
def mock_config_openai_plugin():
"""Mock config object for testing the scan_plugins function"""
class MockConfig:
"""Mock config object for testing the scan_plugins function"""
plugins_dir = PLUGINS_TEST_DIR
plugins_openai = [PLUGIN_TEST_OPENAI]
plugins_denylist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]
plugins_allowlist = [PLUGIN_TEST_OPENAI]
return MockConfig()
def test_scan_plugins_openai(mock_config_openai_plugin):
# Test that the function returns the correct number of plugins
result = scan_plugins(mock_config_openai_plugin, debug=True)
assert len(result) == 1
@pytest.fixture
def mock_config_generic_plugin():
"""Mock config object for testing the scan_plugins function"""
# Test that the function returns the correct number of plugins
class MockConfig:
plugins_dir = PLUGINS_TEST_DIR
plugins_openai = []
plugins_denylist = []
plugins_allowlist = ["AutoGPTPVicuna", "auto_gpt_guanaco"]
return MockConfig()
def test_scan_plugins_generic(mock_config_generic_plugin):
# Test that the function returns the correct number of plugins
result = scan_plugins(mock_config_generic_plugin, debug=True)
assert len(result) == 2

148
tests/unit/test_plugins.py
View File

@@ -1,10 +1,61 @@
import pytest
import os
from autogpt.plugins import denylist_allowlist_check, inspect_zip_for_modules
import yaml
from autogpt.config.config import Config
from autogpt.plugins import inspect_zip_for_modules, scan_plugins
from autogpt.plugins.plugin_config import PluginConfig
PLUGINS_TEST_DIR = "tests/unit/data/test_plugins"
PLUGIN_TEST_ZIP_FILE = "Auto-GPT-Plugin-Test-master.zip"
PLUGIN_TEST_INIT_PY = "Auto-GPT-Plugin-Test-master/src/auto_gpt_vicuna/__init__.py"
PLUGIN_TEST_OPENAI = "https://weathergpt.vercel.app/"
def test_scan_plugins_openai(config: Config):
config.plugins_openai = [PLUGIN_TEST_OPENAI]
plugins_config = config.plugins_config
plugins_config.plugins[PLUGIN_TEST_OPENAI] = PluginConfig(
name=PLUGIN_TEST_OPENAI, enabled=True
)
# Test that the function returns the correct number of plugins
result = scan_plugins(config, debug=True)
assert len(result) == 1
def test_scan_plugins_generic(config: Config):
# Test that the function returns the correct number of plugins
plugins_config = config.plugins_config
plugins_config.plugins["auto_gpt_guanaco"] = PluginConfig(
name="auto_gpt_guanaco", enabled=True
)
plugins_config.plugins["auto_gpt_vicuna"] = PluginConfig(
name="auto_gptp_vicuna", enabled=True
)
result = scan_plugins(config, debug=True)
plugin_class_names = [plugin.__class__.__name__ for plugin in result]
assert len(result) == 2
assert "AutoGPTGuanaco" in plugin_class_names
assert "AutoGPTPVicuna" in plugin_class_names
def test_scan_plugins_not_enabled(config: Config):
# Test that the function returns the correct number of plugins
plugins_config = config.plugins_config
plugins_config.plugins["auto_gpt_guanaco"] = PluginConfig(
name="auto_gpt_guanaco", enabled=True
)
plugins_config.plugins["auto_gpt_vicuna"] = PluginConfig(
name="auto_gptp_vicuna", enabled=False
)
result = scan_plugins(config, debug=True)
plugin_class_names = [plugin.__class__.__name__ for plugin in result]
assert len(result) == 1
assert "AutoGPTGuanaco" in plugin_class_names
assert "AutoGPTPVicuna" not in plugin_class_names
def test_inspect_zip_for_modules():
@@ -12,62 +63,49 @@ def test_inspect_zip_for_modules():
assert result == [PLUGIN_TEST_INIT_PY]
@pytest.fixture
def mock_config_denylist_allowlist_check():
class MockConfig:
"""Mock config object for testing the denylist_allowlist_check function"""
def test_create_base_config(config: Config):
"""Test the backwards-compatibility shim to convert old plugin allow/deny list to a config file"""
config.plugins_allowlist = ["a", "b"]
config.plugins_denylist = ["c", "d"]
plugins_denylist = ["BadPlugin"]
plugins_allowlist = ["GoodPlugin"]
authorise_key = "y"
exit_key = "n"
os.remove(config.plugins_config_file)
plugins_config = config.load_plugins_config()
return MockConfig()
# Check the structure of the plugins config data
assert len(plugins_config.plugins) == 4
assert plugins_config.get("a").enabled
assert plugins_config.get("b").enabled
assert not plugins_config.get("c").enabled
assert not plugins_config.get("d").enabled
# Check the saved config file
with open(config.plugins_config_file, "r") as saved_config_file:
saved_config = yaml.load(saved_config_file, Loader=yaml.FullLoader)
assert saved_config == {
"a": {"enabled": True, "config": {}},
"b": {"enabled": True, "config": {}},
"c": {"enabled": False, "config": {}},
"d": {"enabled": False, "config": {}},
}
def test_denylist_allowlist_check_denylist(
mock_config_denylist_allowlist_check, monkeypatch
):
# Test that the function returns False when the plugin is in the denylist
monkeypatch.setattr("builtins.input", lambda _: "y")
assert not denylist_allowlist_check(
"BadPlugin", mock_config_denylist_allowlist_check
)
def test_load_config(config: Config):
"""Test that the plugin config is loaded correctly from the plugins_config.yaml file"""
# Create a test config and write it to disk
test_config = {
"a": {"enabled": True, "config": {"api_key": "1234"}},
"b": {"enabled": False, "config": {}},
}
with open(config.plugins_config_file, "w+") as f:
f.write(yaml.dump(test_config))
# Load the config from disk
plugins_config = config.load_plugins_config()
def test_denylist_allowlist_check_allowlist(
mock_config_denylist_allowlist_check, monkeypatch
):
# Test that the function returns True when the plugin is in the allowlist
monkeypatch.setattr("builtins.input", lambda _: "y")
assert denylist_allowlist_check("GoodPlugin", mock_config_denylist_allowlist_check)
def test_denylist_allowlist_check_user_input_yes(
mock_config_denylist_allowlist_check, monkeypatch
):
# Test that the function returns True when the user inputs "y"
monkeypatch.setattr("builtins.input", lambda _: "y")
assert denylist_allowlist_check(
"UnknownPlugin", mock_config_denylist_allowlist_check
)
def test_denylist_allowlist_check_user_input_no(
mock_config_denylist_allowlist_check, monkeypatch
):
# Test that the function returns False when the user inputs "n"
monkeypatch.setattr("builtins.input", lambda _: "n")
assert not denylist_allowlist_check(
"UnknownPlugin", mock_config_denylist_allowlist_check
)
def test_denylist_allowlist_check_user_input_invalid(
mock_config_denylist_allowlist_check, monkeypatch
):
# Test that the function returns False when the user inputs an invalid value
monkeypatch.setattr("builtins.input", lambda _: "invalid")
assert not denylist_allowlist_check(
"UnknownPlugin", mock_config_denylist_allowlist_check
)
# Check that the loaded config is equal to the test config
assert len(plugins_config.plugins) == 2
assert plugins_config.get("a").enabled
assert plugins_config.get("a").config == {"api_key": "1234"}
assert not plugins_config.get("b").enabled
assert plugins_config.get("b").config == {}