feat(blocks): Add error output pins to all Firecrawl blocks

- Added error field to Output schema of all Firecrawl blocks
- Wrapped execution logic in try-except blocks to catch and yield errors
- Ensures errors are properly surfaced through standard error pins

Resolves #11253

Co-authored-by: Toran Bruce Richards <Torantulino@users.noreply.github.com>

autogpt_platform/backend/backend/blocks/firecrawl/crawl.py

@@ -55,6 +55,10 @@ class FirecrawlCrawlBlock(Block):
         change_tracking: dict[str, Any] = SchemaField(
             description="The change tracking of the crawl"
         )
+        error: str = SchemaField(
+            description="Error message if the crawl failed",
+            default="",
+        )
 
     def __init__(self):
         super().__init__(
@@ -68,36 +72,39 @@ class FirecrawlCrawlBlock(Block):
     async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
+        try:
+            app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
 
-        # Sync call
-        crawl_result = app.crawl(
-            input_data.url,
-            limit=input_data.limit,
-            scrape_options=ScrapeOptions(
-                formats=convert_to_format_options(input_data.formats),
-                only_main_content=input_data.only_main_content,
-                max_age=input_data.max_age,
-                wait_for=input_data.wait_for,
-            ),
-        )
-        yield "data", crawl_result.data
+            # Sync call
+            crawl_result = app.crawl(
+                input_data.url,
+                limit=input_data.limit,
+                scrape_options=ScrapeOptions(
+                    formats=convert_to_format_options(input_data.formats),
+                    only_main_content=input_data.only_main_content,
+                    max_age=input_data.max_age,
+                    wait_for=input_data.wait_for,
+                ),
+            )
+            yield "data", crawl_result.data
 
-        for data in crawl_result.data:
-            for f in input_data.formats:
-                if f == ScrapeFormat.MARKDOWN:
-                    yield "markdown", data.markdown
-                elif f == ScrapeFormat.HTML:
-                    yield "html", data.html
-                elif f == ScrapeFormat.RAW_HTML:
-                    yield "raw_html", data.raw_html
-                elif f == ScrapeFormat.LINKS:
-                    yield "links", data.links
-                elif f == ScrapeFormat.SCREENSHOT:
-                    yield "screenshot", data.screenshot
-                elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
-                    yield "screenshot_full_page", data.screenshot
-                elif f == ScrapeFormat.CHANGE_TRACKING:
-                    yield "change_tracking", data.change_tracking
-                elif f == ScrapeFormat.JSON:
-                    yield "json", data.json
+            for data in crawl_result.data:
+                for f in input_data.formats:
+                    if f == ScrapeFormat.MARKDOWN:
+                        yield "markdown", data.markdown
+                    elif f == ScrapeFormat.HTML:
+                        yield "html", data.html
+                    elif f == ScrapeFormat.RAW_HTML:
+                        yield "raw_html", data.raw_html
+                    elif f == ScrapeFormat.LINKS:
+                        yield "links", data.links
+                    elif f == ScrapeFormat.SCREENSHOT:
+                        yield "screenshot", data.screenshot
+                    elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
+                        yield "screenshot_full_page", data.screenshot
+                    elif f == ScrapeFormat.CHANGE_TRACKING:
+                        yield "change_tracking", data.change_tracking
+                    elif f == ScrapeFormat.JSON:
+                        yield "json", data.json
+        except Exception as e:
+            yield "error", f"Crawl failed: {str(e)}"

autogpt_platform/backend/backend/blocks/firecrawl/extract.py

@@ -39,6 +39,10 @@ class FirecrawlExtractBlock(Block):
 
     class Output(BlockSchema):
         data: dict[str, Any] = SchemaField(description="The result of the crawl")
+        error: str = SchemaField(
+            description="Error message if the extract failed",
+            default="",
+        )
 
     def __init__(self):
         super().__init__(
@@ -52,13 +56,16 @@ class FirecrawlExtractBlock(Block):
     async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
+        try:
+            app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
 
-        extract_result = app.extract(
-            urls=input_data.urls,
-            prompt=input_data.prompt,
-            schema=input_data.output_schema,
-            enable_web_search=input_data.enable_web_search,
-        )
+            extract_result = app.extract(
+                urls=input_data.urls,
+                prompt=input_data.prompt,
+                schema=input_data.output_schema,
+                enable_web_search=input_data.enable_web_search,
+            )
 
-        yield "data", extract_result.data
+            yield "data", extract_result.data
+        except Exception as e:
+            yield "error", f"Extract failed: {str(e)}"

autogpt_platform/backend/backend/blocks/firecrawl/map.py

@@ -26,6 +26,10 @@ class FirecrawlMapWebsiteBlock(Block):
         results: list[dict[str, Any]] = SchemaField(
             description="List of search results with url, title, and description"
         )
+        error: str = SchemaField(
+            description="Error message if the map failed",
+            default="",
+        )
 
     def __init__(self):
         super().__init__(
@@ -39,22 +43,25 @@ class FirecrawlMapWebsiteBlock(Block):
     async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
+        try:
+            app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
 
-        # Sync call
-        map_result = app.map(
-            url=input_data.url,
-        )
+            # Sync call
+            map_result = app.map(
+                url=input_data.url,
+            )
 
-        # Convert SearchResult objects to dicts
-        results_data = [
-            {
-                "url": link.url,
-                "title": link.title,
-                "description": link.description,
-            }
-            for link in map_result.links
-        ]
+            # Convert SearchResult objects to dicts
+            results_data = [
+                {
+                    "url": link.url,
+                    "title": link.title,
+                    "description": link.description,
+                }
+                for link in map_result.links
+            ]
 
-        yield "links", [link.url for link in map_result.links]
-        yield "results", results_data
+            yield "links", [link.url for link in map_result.links]
+            yield "results", results_data
+        except Exception as e:
+            yield "error", f"Map failed: {str(e)}"

autogpt_platform/backend/backend/blocks/firecrawl/scrape.py

@@ -54,6 +54,10 @@ class FirecrawlScrapeBlock(Block):
         change_tracking: dict[str, Any] = SchemaField(
             description="The change tracking of the crawl"
         )
+        error: str = SchemaField(
+            description="Error message if the scrape failed",
+            default="",
+        )
 
     def __init__(self):
         super().__init__(
@@ -67,31 +71,34 @@ class FirecrawlScrapeBlock(Block):
     async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
+        try:
+            app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
 
-        scrape_result = app.scrape(
-            input_data.url,
-            formats=convert_to_format_options(input_data.formats),
-            only_main_content=input_data.only_main_content,
-            max_age=input_data.max_age,
-            wait_for=input_data.wait_for,
-        )
-        yield "data", scrape_result
+            scrape_result = app.scrape(
+                input_data.url,
+                formats=convert_to_format_options(input_data.formats),
+                only_main_content=input_data.only_main_content,
+                max_age=input_data.max_age,
+                wait_for=input_data.wait_for,
+            )
+            yield "data", scrape_result
 
-        for f in input_data.formats:
-            if f == ScrapeFormat.MARKDOWN:
-                yield "markdown", scrape_result.markdown
-            elif f == ScrapeFormat.HTML:
-                yield "html", scrape_result.html
-            elif f == ScrapeFormat.RAW_HTML:
-                yield "raw_html", scrape_result.raw_html
-            elif f == ScrapeFormat.LINKS:
-                yield "links", scrape_result.links
-            elif f == ScrapeFormat.SCREENSHOT:
-                yield "screenshot", scrape_result.screenshot
-            elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
-                yield "screenshot_full_page", scrape_result.screenshot
-            elif f == ScrapeFormat.CHANGE_TRACKING:
-                yield "change_tracking", scrape_result.change_tracking
-            elif f == ScrapeFormat.JSON:
-                yield "json", scrape_result.json
+            for f in input_data.formats:
+                if f == ScrapeFormat.MARKDOWN:
+                    yield "markdown", scrape_result.markdown
+                elif f == ScrapeFormat.HTML:
+                    yield "html", scrape_result.html
+                elif f == ScrapeFormat.RAW_HTML:
+                    yield "raw_html", scrape_result.raw_html
+                elif f == ScrapeFormat.LINKS:
+                    yield "links", scrape_result.links
+                elif f == ScrapeFormat.SCREENSHOT:
+                    yield "screenshot", scrape_result.screenshot
+                elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
+                    yield "screenshot_full_page", scrape_result.screenshot
+                elif f == ScrapeFormat.CHANGE_TRACKING:
+                    yield "change_tracking", scrape_result.change_tracking
+                elif f == ScrapeFormat.JSON:
+                    yield "json", scrape_result.json
+        except Exception as e:
+            yield "error", f"Scrape failed: {str(e)}"

autogpt_platform/backend/backend/blocks/firecrawl/search.py

@@ -38,6 +38,10 @@ class FirecrawlSearchBlock(Block):
     class Output(BlockSchema):
         data: dict[str, Any] = SchemaField(description="The result of the search")
         site: dict[str, Any] = SchemaField(description="The site of the search")
+        error: str = SchemaField(
+            description="Error message if the search failed",
+            default="",
+        )
 
     def __init__(self):
         super().__init__(
@@ -51,19 +55,22 @@ class FirecrawlSearchBlock(Block):
     async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
+        try:
+            app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
 
-        # Sync call
-        scrape_result = app.search(
-            input_data.query,
-            limit=input_data.limit,
-            scrape_options=ScrapeOptions(
-                formats=convert_to_format_options(input_data.formats) or None,
-                max_age=input_data.max_age,
-                wait_for=input_data.wait_for,
-            ),
-        )
-        yield "data", scrape_result
-        if hasattr(scrape_result, "web") and scrape_result.web:
-            for site in scrape_result.web:
-                yield "site", site
+            # Sync call
+            scrape_result = app.search(
+                input_data.query,
+                limit=input_data.limit,
+                scrape_options=ScrapeOptions(
+                    formats=convert_to_format_options(input_data.formats) or None,
+                    max_age=input_data.max_age,
+                    wait_for=input_data.wait_for,
+                ),
+            )
+            yield "data", scrape_result
+            if hasattr(scrape_result, "web") and scrape_result.web:
+                for site in scrape_result.web:
+                    yield "site", site
+        except Exception as e:
+            yield "error", f"Search failed: {str(e)}"