From d5bae8759fcd93ee312ab65ce36deec3ed64b85b Mon Sep 17 00:00:00 2001
From: YeongJun <skywave@live.co.kr>
Date: Sat, 30 Nov 2024 23:31:30 +0900
Subject: [PATCH] follows redirects on checking robots.txt

---
 src/fetch/src/mcp_server_fetch/server.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py
index e172bcf5..72b294d0 100644
--- a/src/fetch/src/mcp_server_fetch/server.py
+++ b/src/fetch/src/mcp_server_fetch/server.py
@@ -74,7 +74,9 @@ async def check_may_autonomously_fetch_url(url: AnyUrl | str, user_agent: str) -
     async with AsyncClient() as client:
         try:
             response = await client.get(
-                robot_txt_url, headers={"User-Agent": user_agent}
+                robot_txt_url,
+                follow_redirects=True,
+                headers={"User-Agent": user_agent},
             )
         except HTTPError:
             raise McpError(