diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d44f2a4b2..7f96529932 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: branches: [ master, ci-test* ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' pull_request: branches: [ stable, master ] pull_request_target: @@ -148,8 +148,9 @@ jobs: - name: Run pytest with coverage run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term - python tests/integration/challenges/utils/build_current_score.py + pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \ + tests/unit tests/integration tests/challenges + python tests/challenges/utils/build_current_score.py env: CI: true PROXY: ${{ secrets.PROXY }} @@ -179,7 +180,7 @@ jobs: - name: Push updated challenge scores if: github.event_name == 'push' run: | - score_file="tests/integration/challenges/current_score.json" + score_file="tests/challenges/current_score.json" if ! git diff --quiet $score_file; then git add $score_file diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index a61b707d8e..ff43666c44 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -5,7 +5,7 @@ on: branches: [ master ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' pull_request: branches: [ master, stable ] @@ -108,15 +108,18 @@ jobs: set +e test_output=$( docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \ - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1 + pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing \ + tests/unit tests/integration 2>&1 ) test_failure=$? - + echo "$test_output" - + cat << $EOF >> $GITHUB_STEP_SUMMARY # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌') \`\`\` $test_output \`\`\` $EOF + + exit $test_failure diff --git a/.github/workflows/pr-label.yml b/.github/workflows/pr-label.yml index 0bab56385b..62a197de53 100644 --- a/.github/workflows/pr-label.yml +++ b/.github/workflows/pr-label.yml @@ -6,7 +6,7 @@ on: branches: [ master ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - - 'tests/integration/challenges/current_score.json' + - 'tests/challenges/current_score.json' # So that the `dirtyLabel` is removed if conflicts are resolve # We recommend `pull_request_target` so that github secrets are available. # In `pull_request` we wouldn't be able to change labels of fork PRs diff --git a/autogpt/app.py b/autogpt/app.py index 0804b48271..525deddcb5 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -142,7 +142,7 @@ def get_text_summary(url: str, question: str, config: Config) -> str: Returns: str: The summary of the text """ - text = scrape_text(url) + text = scrape_text(url, config) summary, _ = summarize_text(text, question=question) return f""" "Result" : {summary}""" diff --git a/docs/challenges/building_challenges.md b/docs/challenges/building_challenges.md index 09ab3bf513..0bd416cc26 100644 --- a/docs/challenges/building_challenges.md +++ b/docs/challenges/building_challenges.md @@ -70,7 +70,7 @@ def kubernetes_agent( ``` ## Creating your challenge -Go to `tests/integration/challenges`and create a file that is called `test_your_test_description.py` and add it to the appropriate folder. If no category exists you can create a new one. +Go to `tests/challenges`and create a file that is called `test_your_test_description.py` and add it to the appropriate folder. If no category exists you can create a new one. Your test could look something like this @@ -84,7 +84,7 @@ import yaml from autogpt.commands.file_operations import read_file, write_to_file from tests.integration.agent_utils import run_interaction_loop -from tests.integration.challenges.utils import run_multiple_times +from tests.challenges.utils import run_multiple_times from tests.utils import requires_api_key diff --git a/docs/challenges/information_retrieval/challenge_a.md b/docs/challenges/information_retrieval/challenge_a.md index de21066ea5..bf1b7b104b 100644 --- a/docs/challenges/information_retrieval/challenge_a.md +++ b/docs/challenges/information_retrieval/challenge_a.md @@ -5,7 +5,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py --level=2 +pytest -s tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py --level=2 ``` ## Description diff --git a/docs/challenges/information_retrieval/challenge_b.md b/docs/challenges/information_retrieval/challenge_b.md index bf77a984f6..f4e68a151e 100644 --- a/docs/challenges/information_retrieval/challenge_b.md +++ b/docs/challenges/information_retrieval/challenge_b.md @@ -5,7 +5,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +pytest -s tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py ``` ## Description diff --git a/docs/challenges/memory/challenge_b.md b/docs/challenges/memory/challenge_b.md index 49c7c40f0a..abc6da6bef 100644 --- a/docs/challenges/memory/challenge_b.md +++ b/docs/challenges/memory/challenge_b.md @@ -4,7 +4,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/memory/test_memory_challenge_b.py --level=3 +pytest -s tests/challenges/memory/test_memory_challenge_b.py --level=3 `` ## Description @@ -41,4 +41,3 @@ Write all the task_ids into the file output.txt. The file has not been created y ## Objective The objective of this challenge is to test the agent's ability to follow instructions and maintain memory of the task IDs throughout the process. The agent successfully completed this challenge if it wrote the task ids in a file. - diff --git a/docs/challenges/memory/challenge_c.md b/docs/challenges/memory/challenge_c.md index fd02a4a56c..e197ddbd2c 100644 --- a/docs/challenges/memory/challenge_c.md +++ b/docs/challenges/memory/challenge_c.md @@ -4,7 +4,7 @@ **Command to try**: ``` -pytest -s tests/integration/challenges/memory/test_memory_challenge_c.py --level=2 +pytest -s tests/challenges/memory/test_memory_challenge_c.py --level=2 `` ## Description diff --git a/mypy.ini b/mypy.ini index b977deb04f..275cd2602b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,7 +2,7 @@ follow_imports = skip check_untyped_defs = True disallow_untyped_defs = True -files = tests/integration/challenges/**/*.py +files = tests/challenges/**/*.py [mypy-requests.*] ignore_missing_imports = True diff --git a/tests/integration/challenges/__init__.py b/tests/challenges/__init__.py similarity index 100% rename from tests/integration/challenges/__init__.py rename to tests/challenges/__init__.py diff --git a/tests/integration/challenges/basic_abilities/__init__.py b/tests/challenges/basic_abilities/__init__.py similarity index 100% rename from tests/integration/challenges/basic_abilities/__init__.py rename to tests/challenges/basic_abilities/__init__.py diff --git a/tests/integration/challenges/basic_abilities/goal_oriented_tasks.md b/tests/challenges/basic_abilities/goal_oriented_tasks.md similarity index 100% rename from tests/integration/challenges/basic_abilities/goal_oriented_tasks.md rename to tests/challenges/basic_abilities/goal_oriented_tasks.md diff --git a/tests/integration/challenges/basic_abilities/test_browse_website.py b/tests/challenges/basic_abilities/test_browse_website.py similarity index 76% rename from tests/integration/challenges/basic_abilities/test_browse_website.py rename to tests/challenges/basic_abilities/test_browse_website.py index 09e5ab2200..b918434c0e 100644 --- a/tests/integration/challenges/basic_abilities/test_browse_website.py +++ b/tests/challenges/basic_abilities/test_browse_website.py @@ -1,10 +1,8 @@ import pytest from autogpt.agent import Agent -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 2 diff --git a/tests/integration/challenges/basic_abilities/test_write_file.py b/tests/challenges/basic_abilities/test_write_file.py similarity index 81% rename from tests/integration/challenges/basic_abilities/test_write_file.py rename to tests/challenges/basic_abilities/test_write_file.py index 393dbfd05d..033f76e6bf 100644 --- a/tests/integration/challenges/basic_abilities/test_write_file.py +++ b/tests/challenges/basic_abilities/test_write_file.py @@ -4,10 +4,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/challenge_decorator/__init__.py b/tests/challenges/challenge_decorator/__init__.py similarity index 100% rename from tests/integration/challenges/challenge_decorator/__init__.py rename to tests/challenges/challenge_decorator/__init__.py diff --git a/tests/integration/challenges/challenge_decorator/challenge.py b/tests/challenges/challenge_decorator/challenge.py similarity index 100% rename from tests/integration/challenges/challenge_decorator/challenge.py rename to tests/challenges/challenge_decorator/challenge.py diff --git a/tests/integration/challenges/challenge_decorator/challenge_decorator.py b/tests/challenges/challenge_decorator/challenge_decorator.py similarity index 90% rename from tests/integration/challenges/challenge_decorator/challenge_decorator.py rename to tests/challenges/challenge_decorator/challenge_decorator.py index fe12317eed..1d1fbf9146 100644 --- a/tests/integration/challenges/challenge_decorator/challenge_decorator.py +++ b/tests/challenges/challenge_decorator/challenge_decorator.py @@ -4,11 +4,9 @@ from typing import Any, Callable, Optional import pytest -from tests.integration.challenges.challenge_decorator.challenge import Challenge -from tests.integration.challenges.challenge_decorator.challenge_utils import ( - create_challenge, -) -from tests.integration.challenges.challenge_decorator.score_utils import ( +from tests.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge_utils import create_challenge +from tests.challenges.challenge_decorator.score_utils import ( get_scores, update_new_score, ) diff --git a/tests/integration/challenges/challenge_decorator/challenge_utils.py b/tests/challenges/challenge_decorator/challenge_utils.py similarity index 96% rename from tests/integration/challenges/challenge_decorator/challenge_utils.py rename to tests/challenges/challenge_decorator/challenge_utils.py index 7db7648fa4..74f4cf5654 100644 --- a/tests/integration/challenges/challenge_decorator/challenge_utils.py +++ b/tests/challenges/challenge_decorator/challenge_utils.py @@ -1,7 +1,7 @@ import os from typing import Any, Callable, Dict, Optional, Tuple -from tests.integration.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge import Challenge CHALLENGE_PREFIX = "test_" diff --git a/tests/integration/challenges/challenge_decorator/score_utils.py b/tests/challenges/challenge_decorator/score_utils.py similarity index 95% rename from tests/integration/challenges/challenge_decorator/score_utils.py rename to tests/challenges/challenge_decorator/score_utils.py index 0a3b71a8cb..1a8be74439 100644 --- a/tests/integration/challenges/challenge_decorator/score_utils.py +++ b/tests/challenges/challenge_decorator/score_utils.py @@ -2,7 +2,7 @@ import json import os from typing import Any, Dict, Optional, Tuple -from tests.integration.challenges.challenge_decorator.challenge import Challenge +from tests.challenges.challenge_decorator.challenge import Challenge CURRENT_SCORE_LOCATION = "../current_score" NEW_SCORE_LOCATION = "../new_score" diff --git a/tests/integration/challenges/conftest.py b/tests/challenges/conftest.py similarity index 89% rename from tests/integration/challenges/conftest.py rename to tests/challenges/conftest.py index 5514a1293f..dff45f1132 100644 --- a/tests/integration/challenges/conftest.py +++ b/tests/challenges/conftest.py @@ -5,9 +5,8 @@ from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import FixtureRequest -from tests.integration.challenges.challenge_decorator.challenge import Challenge -from tests.integration.conftest import BASE_VCR_CONFIG -from tests.vcr.vcr_filter import before_record_response +from tests.challenges.challenge_decorator.challenge import Challenge +from tests.vcr import BASE_VCR_CONFIG, before_record_response def before_record_response_filter_errors( diff --git a/tests/integration/challenges/current_score.json b/tests/challenges/current_score.json similarity index 100% rename from tests/integration/challenges/current_score.json rename to tests/challenges/current_score.json diff --git a/tests/integration/challenges/debug_code/data/two_sum.py b/tests/challenges/debug_code/data/two_sum.py similarity index 100% rename from tests/integration/challenges/debug_code/data/two_sum.py rename to tests/challenges/debug_code/data/two_sum.py diff --git a/tests/integration/challenges/debug_code/data/two_sum_tests.py b/tests/challenges/debug_code/data/two_sum_tests.py similarity index 100% rename from tests/integration/challenges/debug_code/data/two_sum_tests.py rename to tests/challenges/debug_code/data/two_sum_tests.py diff --git a/tests/integration/challenges/debug_code/test_debug_code_challenge_a.py b/tests/challenges/debug_code/test_debug_code_challenge_a.py similarity index 89% rename from tests/integration/challenges/debug_code/test_debug_code_challenge_a.py rename to tests/challenges/debug_code/test_debug_code_challenge_a.py index 008e562ce3..93df754e74 100644 --- a/tests/integration/challenges/debug_code/test_debug_code_challenge_a.py +++ b/tests/challenges/debug_code/test_debug_code_challenge_a.py @@ -7,10 +7,8 @@ from autogpt.agent import Agent from autogpt.commands.execute_code import execute_python_file from autogpt.commands.file_operations import append_to_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 5 diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py similarity index 89% rename from tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py rename to tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 2f61fef390..eb3d0c949b 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -3,10 +3,8 @@ from pytest_mock import MockerFixture from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py similarity index 90% rename from tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py rename to tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py index 6461e13fbf..51195f77ff 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -6,10 +6,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py similarity index 88% rename from tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py rename to tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py index aa46ac4d4f..8ea20b94fa 100644 --- a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -5,10 +5,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key CYCLE_COUNT = 3 diff --git a/tests/integration/challenges/memory/__init__.py b/tests/challenges/memory/__init__.py similarity index 100% rename from tests/integration/challenges/memory/__init__.py rename to tests/challenges/memory/__init__.py diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/challenges/memory/test_memory_challenge_a.py similarity index 90% rename from tests/integration/challenges/memory/test_memory_challenge_a.py rename to tests/challenges/memory/test_memory_challenge_a.py index 08f461bdd8..3e3251ddde 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/challenges/memory/test_memory_challenge_a.py @@ -4,10 +4,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import run_interaction_loop from tests.utils import requires_api_key diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/challenges/memory/test_memory_challenge_b.py similarity index 91% rename from tests/integration/challenges/memory/test_memory_challenge_b.py rename to tests/challenges/memory/test_memory_challenge_b.py index c82763129a..011fa17a4d 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/challenges/memory/test_memory_challenge_b.py @@ -4,10 +4,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import generate_noise, run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import generate_noise, run_interaction_loop from tests.utils import requires_api_key NOISE = 1000 diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/challenges/memory/test_memory_challenge_c.py similarity index 95% rename from tests/integration/challenges/memory/test_memory_challenge_c.py rename to tests/challenges/memory/test_memory_challenge_c.py index ab8ece1052..d7cc699405 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/challenges/memory/test_memory_challenge_c.py @@ -4,10 +4,8 @@ from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file from autogpt.config import Config -from tests.integration.challenges.challenge_decorator.challenge_decorator import ( - challenge, -) -from tests.integration.challenges.utils import generate_noise, run_interaction_loop +from tests.challenges.challenge_decorator.challenge_decorator import challenge +from tests.challenges.utils import generate_noise, run_interaction_loop from tests.utils import requires_api_key NOISE = 1000 diff --git a/tests/integration/challenges/test_challenge_should_be_formatted_properly.py b/tests/challenges/test_challenge_should_be_formatted_properly.py similarity index 100% rename from tests/integration/challenges/test_challenge_should_be_formatted_properly.py rename to tests/challenges/test_challenge_should_be_formatted_properly.py diff --git a/tests/integration/challenges/utils.py b/tests/challenges/utils.py similarity index 100% rename from tests/integration/challenges/utils.py rename to tests/challenges/utils.py diff --git a/tests/integration/challenges/utils/build_current_score.py b/tests/challenges/utils/build_current_score.py similarity index 84% rename from tests/integration/challenges/utils/build_current_score.py rename to tests/challenges/utils/build_current_score.py index 743b132842..aec125b400 100644 --- a/tests/integration/challenges/utils/build_current_score.py +++ b/tests/challenges/utils/build_current_score.py @@ -26,12 +26,8 @@ def recursive_sort_dict(data: dict) -> dict: cwd = os.getcwd() # get current working directory -new_score_filename_pattern = os.path.join( - cwd, "tests/integration/challenges/new_score_*.json" -) -current_score_filename = os.path.join( - cwd, "tests/integration/challenges/current_score.json" -) +new_score_filename_pattern = os.path.join(cwd, "tests/challenges/new_score_*.json") +current_score_filename = os.path.join(cwd, "tests/challenges/current_score.json") merged_data: Dict[str, Any] = {} for filename in glob.glob(new_score_filename_pattern): diff --git a/tests/conftest.py b/tests/conftest.py index 98bebc9e34..0ee023b58d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import pytest @@ -8,15 +7,11 @@ from autogpt.config.config import Config from autogpt.llm.api_manager import ApiManager from autogpt.workspace import Workspace -pytest_plugins = ["tests.integration.agent_factory", "tests.integration.memory.utils"] - -PROXY = os.environ.get("PROXY") - - -@pytest.fixture() -def vcr_cassette_dir(request): - test_name = os.path.splitext(request.node.name)[0] - return os.path.join("tests/Auto-GPT-test-cassettes", test_name) +pytest_plugins = [ + "tests.integration.agent_factory", + "tests.integration.memory.utils", + "tests.vcr", +] @pytest.fixture() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py deleted file mode 100644 index 686f50be40..0000000000 --- a/tests/integration/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -import os - -import openai.api_requestor -import pytest -from pytest_mock import MockerFixture - -from tests.conftest import PROXY -from tests.vcr.vcr_filter import before_record_request, before_record_response - -BASE_VCR_CONFIG = { - "record_mode": "new_episodes", - "before_record_request": before_record_request, - "before_record_response": before_record_response, - "filter_headers": [ - "Authorization", - "X-OpenAI-Client-User-Agent", - "User-Agent", - ], - "match_on": ["method", "body"], -} - - -@pytest.fixture(scope="session") -def vcr_config(): - # this fixture is called by the pytest-recording vcr decorator. - return BASE_VCR_CONFIG - - -def patch_api_base(requestor): - new_api_base = f"{PROXY}/v1" - requestor.api_base = new_api_base - return requestor - - -@pytest.fixture -def patched_api_requestor(mocker: MockerFixture): - original_init = openai.api_requestor.APIRequestor.__init__ - original_validate_headers = openai.api_requestor.APIRequestor._validate_headers - - def patched_init(requestor, *args, **kwargs): - original_init(requestor, *args, **kwargs) - patch_api_base(requestor) - - def patched_validate_headers(self, supplied_headers): - headers = original_validate_headers(self, supplied_headers) - headers["AGENT-MODE"] = os.environ.get("AGENT_MODE") - headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE") - return headers - - if PROXY: - mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init) - mocker.patch.object( - openai.api_requestor.APIRequestor, - "_validate_headers", - new=patched_validate_headers, - ) diff --git a/tests/integration/test_commands.py b/tests/integration/test_commands.py deleted file mode 100644 index 1cbb392966..0000000000 --- a/tests/integration/test_commands.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Unit tests for the commands module""" -from unittest.mock import MagicMock, patch - -import pytest - -from autogpt.app import list_agents, start_agent -from tests.utils import requires_api_key - - -@pytest.mark.vcr -@pytest.mark.integration_test -@requires_api_key("OPENAI_API_KEY") -def test_make_agent(patched_api_requestor, config) -> None: - """Test that an agent can be created""" - # Use the mock agent manager to avoid creating a real agent - with patch("openai.ChatCompletion.create") as mock: - response = MagicMock() - # del response.error - response.choices[0].messages[0].content = "Test message" - response.usage.prompt_tokens = 1 - response.usage.completion_tokens = 1 - mock.return_value = response - start_agent( - "Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo" - ) - agents = list_agents(config) - assert "List of agents:\n0: chat" == agents - start_agent( - "Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo" - ) - agents = list_agents(config) - assert "List of agents:\n0: chat\n1: write" == agents diff --git a/tests/test_analyze_code.py b/tests/test_analyze_code.py deleted file mode 100644 index 98ab8b724d..0000000000 --- a/tests/test_analyze_code.py +++ /dev/null @@ -1,74 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. -import pytest - -from autogpt.commands.analyze_code import analyze_code -from autogpt.config import Config - - -@pytest.fixture -def mock_call_ai_function(mocker): - return mocker.patch("autogpt.commands.analyze_code.call_ai_function") - - -class TestAnalyzeCode: - def test_positive_analyze_code(self, mock_call_ai_function): - # Positive Test - mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"] - code = "def example_function():\n pass" - config = Config() - result = analyze_code(code, config) - assert result == ["Suggestion 1", "Suggestion 2"] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_negative_analyze_code( - self, - mock_call_ai_function, - config: Config, - ): - # Negative Test - mock_call_ai_function.return_value = [] - code = "def example_function():\n pass" - result = analyze_code(code, config) - assert result == [] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_error_analyze_code(self, mock_call_ai_function, config: Config): - # Error Test - mock_call_ai_function.side_effect = Exception("Error occurred") - code = "def example_function():\n pass" - with pytest.raises(Exception): - result = analyze_code(code, config) - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) - - def test_edge_analyze_code_empty_code( - self, - mock_call_ai_function, - config: Config, - ): - # Edge Test - mock_call_ai_function.return_value = ["Suggestion 1", "Suggestion 2"] - code = "" - result = analyze_code(code, config) - assert result == ["Suggestion 1", "Suggestion 2"] - mock_call_ai_function.assert_called_once_with( - "def analyze_code(code: str) -> list[str]:", - [code], - "Analyzes the given code and returns a list of suggestions for improvements.", - config=config, - ) diff --git a/tests/test_audio_text_read_audio.py b/tests/test_audio_text_read_audio.py deleted file mode 100644 index 4385da321c..0000000000 --- a/tests/test_audio_text_read_audio.py +++ /dev/null @@ -1,56 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. -import json -from unittest.mock import MagicMock, patch - -import pytest - -from autogpt.commands.audio_text import read_audio - - -class TestReadAudio: - @patch("requests.post") - def test_positive_read_audio(self, mock_post, config): - # Positive Test - audio_data = b"test_audio_data" - mock_response = MagicMock() - mock_response.content.decode.return_value = json.dumps( - {"text": "Hello, world!"} - ) - mock_post.return_value = mock_response - - config.huggingface_api_token = "testing-token" - result = read_audio(audio_data, config) - assert result == "The audio says: Hello, world!" - mock_post.assert_called_once_with( - f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}", - headers={"Authorization": f"Bearer {config.huggingface_api_token}"}, - data=audio_data, - ) - - @patch("requests.post") - def test_negative_read_audio(self, mock_post, config): - # Negative Test - audio_data = b"test_audio_data" - mock_response = MagicMock() - mock_response.content.decode.return_value = json.dumps({"text": ""}) - mock_post.return_value = mock_response - config.huggingface_api_token = "testing-token" - result = read_audio(audio_data, config) - assert result == "The audio says: " - mock_post.assert_called_once_with( - f"https://api-inference.huggingface.co/models/{config.huggingface_audio_to_text_model}", - headers={"Authorization": f"Bearer {config.huggingface_api_token}"}, - data=audio_data, - ) - - def test_error_read_audio(self, config): - # Error Test - config.huggingface_api_token = None - with pytest.raises(ValueError): - read_audio(b"test_audio_data", config) - - def test_edge_read_audio_empty_audio(self, config): - # Edge Test - with pytest.raises(ValueError): - read_audio(b"", config) diff --git a/tests/test_audio_text_read_audio_from_file.py b/tests/test_audio_text_read_audio_from_file.py deleted file mode 100644 index c8d66a0608..0000000000 --- a/tests/test_audio_text_read_audio_from_file.py +++ /dev/null @@ -1,55 +0,0 @@ -# Date: 2023-5-13 -# Author: Generated by GoCodeo. - - -from unittest.mock import mock_open, patch - -import pytest - -from autogpt.commands.audio_text import read_audio_from_file -from autogpt.config import Config - - -@pytest.fixture -def mock_read_audio(mocker): - return mocker.patch("autogpt.commands.audio_text.read_audio") - - -class TestReadAudioFromFile: - def test_positive_read_audio_from_file(self, mock_read_audio): - # Positive test - mock_read_audio.return_value = "This is a sample text." - mock_file_data = b"Audio data" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("test_audio.wav", Config()) - assert result == "This is a sample text." - m.assert_called_once_with("test_audio.wav", "rb") - - def test_negative_read_audio_from_file(self, mock_read_audio): - # Negative test - mock_read_audio.return_value = "This is a sample text." - mock_file_data = b"Audio data" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("test_audio.wav", Config()) - assert result != "Incorrect text." - m.assert_called_once_with("test_audio.wav", "rb") - - def test_error_read_audio_from_file(self): - # Error test - with pytest.raises(FileNotFoundError): - read_audio_from_file("non_existent_file.wav", Config()) - - def test_edge_empty_audio_file(self, mock_read_audio): - # Edge test - mock_read_audio.return_value = "" - mock_file_data = b"" - m = mock_open(read_data=mock_file_data) - - with patch("builtins.open", m): - result = read_audio_from_file("empty_audio.wav", Config()) - assert result == "" - m.assert_called_once_with("empty_audio.wav", "rb") diff --git a/tests/test_agent.py b/tests/unit/test_agent.py similarity index 100% rename from tests/test_agent.py rename to tests/unit/test_agent.py diff --git a/tests/test_agent_manager.py b/tests/unit/test_agent_manager.py similarity index 100% rename from tests/test_agent_manager.py rename to tests/unit/test_agent_manager.py diff --git a/tests/test_ai_config.py b/tests/unit/test_ai_config.py similarity index 100% rename from tests/test_ai_config.py rename to tests/unit/test_ai_config.py diff --git a/tests/test_api_manager.py b/tests/unit/test_api_manager.py similarity index 100% rename from tests/test_api_manager.py rename to tests/unit/test_api_manager.py diff --git a/tests/test_commands.py b/tests/unit/test_commands.py similarity index 100% rename from tests/test_commands.py rename to tests/unit/test_commands.py diff --git a/tests/test_config.py b/tests/unit/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/unit/test_config.py diff --git a/tests/integration/test_git_commands.py b/tests/unit/test_git_commands.py similarity index 100% rename from tests/integration/test_git_commands.py rename to tests/unit/test_git_commands.py diff --git a/tests/integration/test_google_search.py b/tests/unit/test_google_search.py similarity index 100% rename from tests/integration/test_google_search.py rename to tests/unit/test_google_search.py diff --git a/tests/test_logs.py b/tests/unit/test_logs.py similarity index 100% rename from tests/test_logs.py rename to tests/unit/test_logs.py diff --git a/tests/unit/test_make_agent.py b/tests/unit/test_make_agent.py new file mode 100644 index 0000000000..cff20ee399 --- /dev/null +++ b/tests/unit/test_make_agent.py @@ -0,0 +1,24 @@ +from unittest.mock import MagicMock + +from pytest_mock import MockerFixture + +from autogpt.app import list_agents, start_agent +from autogpt.config import Config + + +def test_make_agent(config: Config, mocker: MockerFixture) -> None: + """Test that an agent can be created""" + mock = mocker.patch("openai.ChatCompletion.create") + + response = MagicMock() + # del response.error + response.choices[0].messages[0].content = "Test message" + response.usage.prompt_tokens = 1 + response.usage.completion_tokens = 1 + mock.return_value = response + start_agent("Test Agent", "chat", "Hello, how are you?", config, "gpt-3.5-turbo") + agents = list_agents(config) + assert "List of agents:\n0: chat" == agents + start_agent("Test Agent 2", "write", "Hello, how are you?", config, "gpt-3.5-turbo") + agents = list_agents(config) + assert "List of agents:\n0: chat\n1: write" == agents diff --git a/tests/test_prompt_config.py b/tests/unit/test_prompt_config.py similarity index 100% rename from tests/test_prompt_config.py rename to tests/unit/test_prompt_config.py diff --git a/tests/test_prompt_generator.py b/tests/unit/test_prompt_generator.py similarity index 100% rename from tests/test_prompt_generator.py rename to tests/unit/test_prompt_generator.py diff --git a/tests/test_text_file_parsers.py b/tests/unit/test_text_file_parsers.py similarity index 100% rename from tests/test_text_file_parsers.py rename to tests/unit/test_text_file_parsers.py diff --git a/tests/unit/test_url_validation.py b/tests/unit/test_url_validation.py index 16eb8cd500..5d6e8124e9 100644 --- a/tests/unit/test_url_validation.py +++ b/tests/unit/test_url_validation.py @@ -49,25 +49,17 @@ def test_url_validation_succeeds(url): assert dummy_method(url) == url -bad_protocol_data = ( - ("htt://example.com"), - ("httppp://example.com"), - (" https://example.com"), +@pytest.mark.parametrize( + "url,expected_error", + [ + ("htt://example.com", "Invalid URL format"), + ("httppp://example.com", "Invalid URL format"), + (" https://example.com", "Invalid URL format"), + ("http://?query=q", "Missing Scheme or Network location"), + ], ) - - -@pytest.mark.parametrize("url", bad_protocol_data) -def test_url_validation_fails_bad_protocol(url): - with raises(ValueError, match="Invalid URL format"): - dummy_method(url) - - -missing_loc = (("http://?query=q"),) - - -@pytest.mark.parametrize("url", missing_loc) -def test_url_validation_fails_bad_protocol(url): - with raises(ValueError, match="Missing Scheme or Network location"): +def test_url_validation_fails_invalid_url(url, expected_error): + with raises(ValueError, match=expected_error): dummy_method(url) diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 98% rename from tests/test_utils.py rename to tests/unit/test_utils.py index c0ce28cc1f..099176baee 100644 --- a/tests/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,7 +1,6 @@ import os from unittest.mock import patch -import pytest import requests from autogpt.utils import ( @@ -151,7 +150,3 @@ def test_get_current_git_branch_failure(mock_repo): branch_name = get_current_git_branch() assert branch_name == "" - - -if __name__ == "__main__": - pytest.main() diff --git a/tests/test_workspace.py b/tests/unit/test_workspace.py similarity index 100% rename from tests/test_workspace.py rename to tests/unit/test_workspace.py diff --git a/tests/vcr/__init__.py b/tests/vcr/__init__.py index e69de29bb2..e1a2620ca7 100644 --- a/tests/vcr/__init__.py +++ b/tests/vcr/__init__.py @@ -0,0 +1,61 @@ +import os + +import openai.api_requestor +import pytest +from pytest_mock import MockerFixture + +from .vcr_filter import PROXY, before_record_request, before_record_response + +BASE_VCR_CONFIG = { + "record_mode": "new_episodes", + "before_record_request": before_record_request, + "before_record_response": before_record_response, + "filter_headers": [ + "Authorization", + "X-OpenAI-Client-User-Agent", + "User-Agent", + ], + "match_on": ["method", "body"], +} + + +@pytest.fixture(scope="session") +def vcr_config(): + # this fixture is called by the pytest-recording vcr decorator. + return BASE_VCR_CONFIG + + +@pytest.fixture() +def vcr_cassette_dir(request): + test_name = os.path.splitext(request.node.name)[0] + return os.path.join("tests/Auto-GPT-test-cassettes", test_name) + + +def patch_api_base(requestor): + new_api_base = f"{PROXY}/v1" + requestor.api_base = new_api_base + return requestor + + +@pytest.fixture +def patched_api_requestor(mocker: MockerFixture): + original_init = openai.api_requestor.APIRequestor.__init__ + original_validate_headers = openai.api_requestor.APIRequestor._validate_headers + + def patched_init(requestor, *args, **kwargs): + original_init(requestor, *args, **kwargs) + patch_api_base(requestor) + + def patched_validate_headers(self, supplied_headers): + headers = original_validate_headers(self, supplied_headers) + headers["AGENT-MODE"] = os.environ.get("AGENT_MODE") + headers["AGENT-TYPE"] = os.environ.get("AGENT_TYPE") + return headers + + if PROXY: + mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init) + mocker.patch.object( + openai.api_requestor.APIRequestor, + "_validate_headers", + new=patched_validate_headers, + ) diff --git a/tests/vcr/vcr_filter.py b/tests/vcr/vcr_filter.py index 4cc49fd301..1ba433a76e 100644 --- a/tests/vcr/vcr_filter.py +++ b/tests/vcr/vcr_filter.py @@ -1,8 +1,9 @@ import json +import os import re from typing import Any, Dict, List -from tests.conftest import PROXY +PROXY = os.environ.get("PROXY") REPLACEMENTS: List[Dict[str, str]] = [ {