fix(backend): Handle HTTP errors in HTTP block by returning response objects (#11515)

### Changes 🏗️

- Modify the HTTP block to handle HTTP errors (4xx, 5xx) by returning
response objects instead of raising exceptions.
- This allows proper handling of client_error and server_error outputs.

Fixes
[AUTOGPT-SERVER-6VP](https://sentry.io/organizations/significant-gravitas/issues/7023985892/).
The issue was that: HTTP errors are raised as exceptions by `Requests`
default behavior, bypassing the block's intended error output handling,
resulting in `BlockUnknownError`.

This fix was generated by Seer in Sentry, triggered by Nicholas Tindle.
👁️ Run ID: 4902617

Not quite right? [Click here to continue debugging with
Seer.](https://sentry.io/organizations/significant-gravitas/issues/7023985892/?seerDrawer=true)

### Checklist 📋

#### For code changes:
- [x] I have clearly listed my changes in the PR description
- [x] I have made a test plan
- [x] I have tested my changes according to the test plan:
  <!-- Put your test plan here: -->
- [x] Tested with a service that will return 4XX and 5XX errors to make
sure the correct paths are followed



<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> HTTP block now returns 4xx/5xx responses instead of raising, and
Requests gains retry_max_attempts with last-result handling.
> 
> - **Backend**
>   - **HTTP block (`backend/blocks/http.py`)**:
> - Use `Requests(raise_for_status=False, retry_max_attempts=1)` so
4xx/5xx return response objects and route to
`client_error`/`server_error` outputs.
>   - **HTTP client util (`backend/util/request.py`)**:
> - Add `retry_max_attempts` option with `stop_after_attempt` and
`_return_last_result` to return the final response when retries stop.
> - Build `tenacity` retry config dynamically in `Requests.request()`;
validate `retry_max_attempts >= 1` when provided.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
fccae61c26. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: seer-by-sentry[bot] <157164994+seer-by-sentry[bot]@users.noreply.github.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: nicholas.tindle <nicholas.tindle@agpt.co>

autogpt_platform/backend/backend/blocks/http.py

@@ -184,7 +184,13 @@ class SendWebRequestBlock(Block):
             )
 
         # ─── Execute request ─────────────────────────────────────────
-        response = await Requests().request(
+        # Use raise_for_status=False so HTTP errors (4xx, 5xx) are returned
+        # as response objects instead of raising exceptions, allowing proper
+        # handling via client_error and server_error outputs
+        response = await Requests(
+            raise_for_status=False,
+            retry_max_attempts=1,  # allow callers to handle HTTP errors immediately
+        ).request(
             input_data.method.value,
             input_data.url,
             headers=input_data.headers,

autogpt_platform/backend/backend/util/request.py

@@ -11,7 +11,13 @@ from urllib.parse import quote, urljoin, urlparse
 import aiohttp
 import idna
 from aiohttp import FormData, abc
-from tenacity import retry, retry_if_result, wait_exponential_jitter
+from tenacity import (
+    RetryCallState,
+    retry,
+    retry_if_result,
+    stop_after_attempt,
+    wait_exponential_jitter,
+)
 
 from backend.util.json import loads
 
@@ -285,6 +291,20 @@ class Response:
         return 200 <= self.status < 300
 
 
+def _return_last_result(retry_state: RetryCallState) -> "Response":
+    """
+    Ensure the final attempt's response is returned when retrying stops.
+    """
+    if retry_state.outcome is None:
+        raise RuntimeError("Retry state is missing an outcome.")
+
+    exception = retry_state.outcome.exception()
+    if exception is not None:
+        raise exception
+
+    return retry_state.outcome.result()
+
+
 class Requests:
     """
     A wrapper around an aiohttp ClientSession that validates URLs before
@@ -299,6 +319,7 @@ class Requests:
         extra_url_validator: Callable[[URL], URL] | None = None,
         extra_headers: dict[str, str] | None = None,
         retry_max_wait: float = 300.0,
+        retry_max_attempts: int | None = None,
     ):
         self.trusted_origins = []
         for url in trusted_origins or []:
@@ -311,6 +332,9 @@ class Requests:
         self.extra_url_validator = extra_url_validator
         self.extra_headers = extra_headers
         self.retry_max_wait = retry_max_wait
+        if retry_max_attempts is not None and retry_max_attempts < 1:
+            raise ValueError("retry_max_attempts must be None or >= 1")
+        self.retry_max_attempts = retry_max_attempts
 
     async def request(
         self,
@@ -325,11 +349,17 @@ class Requests:
         max_redirects: int = 10,
         **kwargs,
     ) -> Response:
-        @retry(
-            wait=wait_exponential_jitter(max=self.retry_max_wait),
-            retry=retry_if_result(lambda r: r.status in THROTTLE_RETRY_STATUS_CODES),
-            reraise=True,
-        )
+        retry_kwargs: dict[str, Any] = {
+            "wait": wait_exponential_jitter(max=self.retry_max_wait),
+            "retry": retry_if_result(lambda r: r.status in THROTTLE_RETRY_STATUS_CODES),
+            "reraise": True,
+        }
+
+        if self.retry_max_attempts is not None:
+            retry_kwargs["stop"] = stop_after_attempt(self.retry_max_attempts)
+            retry_kwargs["retry_error_callback"] = _return_last_result
+
+        @retry(**retry_kwargs)
         async def _make_request() -> Response:
             return await self._request(
                 method=method,