Make benchmarks create cassettes without using them (#4664)

.github/workflows/benchmarks.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Run pytest with coverage
         run: |
           rm -rf tests/Auto-GPT-test-cassettes
-          pytest -n auto ${{ matrix.config.task }}
+          pytest -n auto --record-mode=all ${{ matrix.config.task }}
         env:
           CI: true
           PROXY: ${{ secrets.PROXY }}

tests/challenges/conftest.py

@@ -6,7 +6,7 @@ from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureRequest
 
 from tests.challenges.challenge_decorator.challenge import Challenge
-from tests.vcr import BASE_VCR_CONFIG, before_record_response
+from tests.vcr import before_record_response
 
 
 def before_record_response_filter_errors(
@@ -20,9 +20,9 @@ def before_record_response_filter_errors(
 
 
 @pytest.fixture(scope="module")
-def vcr_config() -> Dict[str, Any]:
+def vcr_config(get_base_vcr_config: Dict[str, Any]) -> Dict[str, Any]:
     # this fixture is called by the pytest-recording vcr decorator.
-    return BASE_VCR_CONFIG | {
+    return get_base_vcr_config | {
         "before_record_response": before_record_response_filter_errors,
     }
 

tests/vcr/__init__.py

@@ -6,8 +6,8 @@ from pytest_mock import MockerFixture
 
 from .vcr_filter import PROXY, before_record_request, before_record_response
 
+DEFAULT_RECORD_MODE = "new_episodes"
 BASE_VCR_CONFIG = {
-    "record_mode": "new_episodes",
     "before_record_request": before_record_request,
     "before_record_response": before_record_response,
     "filter_headers": [
@@ -20,9 +20,19 @@ BASE_VCR_CONFIG = {
 
 
 @pytest.fixture(scope="session")
-def vcr_config():
-    # this fixture is called by the pytest-recording vcr decorator.
-    return BASE_VCR_CONFIG
+def vcr_config(get_base_vcr_config):
+    return get_base_vcr_config
+
+
+@pytest.fixture(scope="session")
+def get_base_vcr_config(request):
+    record_mode = request.config.getoption("--record-mode", default="new_episodes")
+    config = BASE_VCR_CONFIG
+
+    if record_mode is None:
+        config["record_mode"] = DEFAULT_RECORD_MODE
+
+    return config
 
 
 @pytest.fixture()