refactor(agent, forge): Move tests from autogpt to forge (#7247)

- Move `autogpt/tests/vcr_cassettes` submodule to `forge/tests/vcr_cassettes`
- Remove not needed markers from `pyproject.toml`: `"requires_openai_api_key", "requires_huggingface_api_key"`
- Update relevant GitHub workflows

Moved relevant tests from `autogpt/tests` to appropiate directories:
- Component tests to their respective component dirs
- `autogpt/tests/unit/test_web_search.py` → `forge/components/web/test_search.py`
- `autogpt/tests/unit/test_git_commands.py` → `forge/components/git_operations/test_git_operations.py`
- `autogpt/tests/unit/test_file_operations.py` → `forge/components/file_manager/test_file_manager.py`
- `autogpt/tests/integration/test_image_gen.py` → `forge/components/image_gen/test_image_gen.py`
- `autogpt/tests/integration/test_web_selenium.py` → `forge/components/web/test_selenium.py`
- `autogpt/tests/integration/test_execute_code.py` → `forge/components/code_executor/test_code_executor.py`
- `autogpt/tests/unit/test_s3_file_storage.py` → `forge/file_storage/test_s3_file_storage.py`
- `autogpt/tests/unit/test_gcs_file_storage.py` → `forge/file_storage/test_gcs_file_storage.py`
- `autogpt/tests/unit/test_local_file_storage.py` → `forge/file_storage/test_local_file_storage.py`
- `autogpt/tests/unit/test_json.py` → `forge/json/test_parsing.py`
- `autogpt/tests/unit/test_logs.py` → `forge/logging/test_utils.py`
- `autogpt/tests/unit/test_url_validation.py` → `forge/utils/test_url_validator.py`
- `autogpt/tests/unit/test_text_file_parsers.py` → `forge/utils/test_file_operations.py`

- (Re)moved dependencies from `autogpt/pyproject.toml` that were only used in these test files.

Also:
- Added `load_env_vars` fixture to `forge/conftest.py`
- Fixed a type error in `forge/components/web/test_search.py`
- Merged `autogpt/.gitattributes` into root `.gitattributes`

---------

Co-authored-by: Reinier van der Leer <pwuts@agpt.co>

.github/workflows/forge-ci.yml

@@ -6,11 +6,13 @@ on:
     paths:
       - '.github/workflows/forge-ci.yml'
       - 'forge/**'
+      - '!forge/tests/vcr_cassettes'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
       - '.github/workflows/forge-ci.yml'
       - 'forge/**'
+      - '!forge/tests/vcr_cassettes'
 
 concurrency:
   group: ${{ format('forge-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
@@ -66,6 +68,37 @@ jobs:
           fetch-depth: 0
           submodules: true
 
+      - name: Checkout cassettes
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        env:
+          PR_BASE: ${{ github.event.pull_request.base.ref }}
+          PR_BRANCH: ${{ github.event.pull_request.head.ref }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          cassette_branch="${PR_AUTHOR}-${PR_BRANCH}"
+          cassette_base_branch="${PR_BASE}"
+          cd tests/vcr_cassettes
+
+          if ! git ls-remote --exit-code --heads origin $cassette_base_branch ; then
+            cassette_base_branch="master"
+          fi
+
+          if git ls-remote --exit-code --heads origin $cassette_branch ; then
+            git fetch origin $cassette_branch
+            git fetch origin $cassette_base_branch
+
+            git checkout $cassette_branch
+
+            # Pick non-conflicting cassette updates from the base branch
+            git merge --no-commit --strategy-option=ours origin/$cassette_base_branch
+            echo "Using cassettes from mirror branch '$cassette_branch'," \
+              "synced to upstream branch '$cassette_base_branch'."
+          else
+            git checkout -b $cassette_branch
+            echo "Branch '$cassette_branch' does not exist in cassette submodule." \
+              "Using cassettes from '$cassette_base_branch'."
+          fi
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -121,6 +154,80 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: forge,${{ runner.os }}
 
+      - id: setup_git_auth
+        name: Set up git token authentication
+        # Cassettes may be pushed even when tests fail
+        if: success() || failure()
+        run: |
+          config_key="http.${{ github.server_url }}/.extraheader"
+          if [ "${{ runner.os }}" = 'macOS' ]; then
+            base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64)
+          else
+            base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0)
+          fi
+
+          git config "$config_key" \
+            "Authorization: Basic $base64_pat"
+
+          cd tests/vcr_cassettes
+          git config "$config_key" \
+            "Authorization: Basic $base64_pat"
+
+          echo "config_key=$config_key" >> $GITHUB_OUTPUT
+
+      - id: push_cassettes
+        name: Push updated cassettes
+        # For pull requests, push updated cassettes even when tests fail
+        if: github.event_name == 'push' || (! github.event.pull_request.head.repo.fork && (success() || failure()))
+        env:
+          PR_BRANCH: ${{ github.event.pull_request.head.ref }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then
+            is_pull_request=true
+            cassette_branch="${PR_AUTHOR}-${PR_BRANCH}"
+          else
+            cassette_branch="${{ github.ref_name }}"
+          fi
+
+          cd tests/vcr_cassettes
+          # Commit & push changes to cassettes if any
+          if ! git diff --quiet; then
+            git add .
+            git commit -m "Auto-update cassettes"
+            git push origin HEAD:$cassette_branch
+            if [ ! $is_pull_request ]; then
+              cd ../..
+              git add tests/vcr_cassettes
+              git commit -m "Update cassette submodule"
+              git push origin HEAD:$cassette_branch
+            fi
+            echo "updated=true" >> $GITHUB_OUTPUT
+          else
+            echo "updated=false" >> $GITHUB_OUTPUT
+            echo "No cassette changes to commit"
+          fi
+
+      - name: Post Set up git token auth
+        if: steps.setup_git_auth.outcome == 'success'
+        run: |
+          git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
+          git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}'
+
+      - name: Apply "behaviour change" label and comment on PR
+        if: ${{ startsWith(github.event_name, 'pull_request') }}
+        run: |
+          PR_NUMBER="${{ github.event.pull_request.number }}"
+          TOKEN="${{ secrets.PAT_REVIEW }}"
+          REPO="${{ github.repository }}"
+
+          if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then
+            echo "Adding label and comment..."
+            echo $TOKEN | gh auth login --with-token
+            gh issue edit $PR_NUMBER --add-label "behaviour change"
+            gh issue comment $PR_NUMBER --body "You changed AutoGPT's behaviour on ${{ runner.os }}. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
+          fi
+
       - name: Upload logs to artifact
         if: always()
         uses: actions/upload-artifact@v4