feat(backend): Enable safe URL redirect on web requests for blocks (#9555)

We've disabled the request for security reasons, and the current request
library breaks on the 301 web response.

### Changes 🏗️

Add manually safe URL redirect on our web request layer.

### Checklist 📋

#### For code changes:
- [ ] I have clearly listed my changes in the PR description
- [ ] I have made a test plan
- [ ] I have tested my changes according to the test plan:
  <!-- Put your test plan here: -->
  - [ ] ...

<details>
  <summary>Example test plan</summary>
  
  - [ ] Create from scratch and execute an agent with at least 3 blocks
- [ ] Import an agent from file upload, and confirm it executes
correctly
  - [ ] Upload agent to marketplace
- [ ] Import an agent from marketplace and confirm it executes correctly
  - [ ] Edit an agent from monitor, and confirm it executes correctly
</details>

#### For configuration changes:
- [ ] `.env.example` is updated or already compatible with my changes
- [ ] `docker-compose.yml` is updated or already compatible with my
changes
- [ ] I have included a list of my configuration changes in the PR
description (under **Changes**)

<details>
  <summary>Examples of configuration changes</summary>

  - Changing ports
  - Adding new services that need to communicate with each other
  - Secrets or environment variable changes
  - New or infrastructure changes such as databases
</details>

autogpt_platform/backend/backend/util/request.py

@@ -2,7 +2,7 @@ import ipaddress
 import re
 import socket
 from typing import Callable
-from urllib.parse import urlparse, urlunparse
+from urllib.parse import urljoin, urlparse, urlunparse
 
 import idna
 import requests as req
@@ -128,7 +128,14 @@ class Requests:
         self.extra_headers = extra_headers
 
     def request(
-        self, method, url, headers=None, allow_redirects=False, *args, **kwargs
+        self,
+        method,
+        url,
+        headers=None,
+        allow_redirects=True,
+        max_redirects=10,
+        *args,
+        **kwargs,
     ) -> req.Response:
         # Merge any extra headers
         if self.extra_headers is not None:
@@ -139,18 +146,41 @@ class Requests:
         if self.extra_url_validator is not None:
             url = self.extra_url_validator(url)
 
-        # Perform the request
+        # Perform the request with redirects disabled for manual handling
         response = req.request(
             method,
             url,
             headers=headers,
-            allow_redirects=allow_redirects,
+            allow_redirects=False,
             *args,
             **kwargs,
         )
         if self.raise_for_status:
             response.raise_for_status()
 
+        # If allowed and a redirect is received, follow the redirect
+        if allow_redirects and response.is_redirect:
+            if max_redirects <= 0:
+                raise Exception("Too many redirects.")
+
+            location = response.headers.get("Location")
+            if not location:
+                return response
+
+            new_url = validate_url(urljoin(url, location), self.trusted_origins)
+            if self.extra_url_validator is not None:
+                new_url = self.extra_url_validator(new_url)
+
+            return self.request(
+                method,
+                new_url,
+                headers=headers,
+                allow_redirects=allow_redirects,
+                max_redirects=max_redirects - 1,
+                *args,
+                **kwargs,
+            )
+
         return response
 
     def get(self, url, *args, **kwargs) -> req.Response: