refactor(docs): restructure platform docs for GitBook and remove MkDo… (#11825)

<!-- Clearly explain the need for these changes: -->
we met some reality when merging into the docs site but this fixes it
### Changes 🏗️
updates paths, adds some guides
<!-- Concisely describe all of the changes made in this pull request:
-->
update to match reality
### Checklist 📋

#### For code changes:
- [x] I have clearly listed my changes in the PR description
- [x] I have made a test plan
- [x] I have tested my changes according to the test plan:
  <!-- Put your test plan here: -->
  - [x] deploy it and validate

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> Aligns block integrations documentation with GitBook.
> 
> - Changes generator default output to
`docs/integrations/block-integrations` and writes overview `README.md`
and `SUMMARY.md` at `docs/integrations/`
> - Adds GitBook frontmatter and hint syntax to overview; prefixes block
links with `block-integrations/`
> - Introduces `generate_summary_md` to build GitBook navigation
(including optional `guides/`)
> - Preserves per-block manual sections and adds optional `extras` +
file-level `additional_content`
> - Updates sync checker to validate parent `README.md` and `SUMMARY.md`
> - Rewrites `docs/integrations/README.md` with GitBook frontmatter and
updated links; adds `docs/integrations/SUMMARY.md`
> - Adds new guides: `guides/llm-providers.md`,
`guides/voice-providers.md`
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
fdb7ff8111. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: bobby.gaffin <bobby.gaffin@agpt.co>

autogpt_platform/backend/scripts/generate_block_docs.py

@@ -34,7 +34,10 @@ logger = logging.getLogger(__name__)
 
 # Default output directory relative to repo root
 DEFAULT_OUTPUT_DIR = (
-    Path(__file__).parent.parent.parent.parent / "docs" / "integrations"
+    Path(__file__).parent.parent.parent.parent
+    / "docs"
+    / "integrations"
+    / "block-integrations"
 )
 
 
@@ -421,6 +424,14 @@ def generate_block_markdown(
     lines.append("<!-- END MANUAL -->")
     lines.append("")
 
+    # Optional per-block extras (only include if has content)
+    extras = manual_content.get("extras", "")
+    if extras:
+        lines.append("<!-- MANUAL: extras -->")
+        lines.append(extras)
+        lines.append("<!-- END MANUAL -->")
+        lines.append("")
+
     lines.append("---")
     lines.append("")
 
@@ -456,25 +467,52 @@ def get_block_file_mapping(blocks: list[BlockDoc]) -> dict[str, list[BlockDoc]]:
     return dict(file_mapping)
 
 
-def generate_overview_table(blocks: list[BlockDoc]) -> str:
-    """Generate the overview table markdown (blocks.md)."""
+def generate_overview_table(blocks: list[BlockDoc], block_dir_prefix: str = "") -> str:
+    """Generate the overview table markdown (blocks.md).
+
+    Args:
+        blocks: List of block documentation objects
+        block_dir_prefix: Prefix for block file links (e.g., "block-integrations/")
+    """
     lines = []
 
+    # GitBook YAML frontmatter
+    lines.append("---")
+    lines.append("layout:")
+    lines.append("  width: default")
+    lines.append("  title:")
+    lines.append("    visible: true")
+    lines.append("  description:")
+    lines.append("    visible: true")
+    lines.append("  tableOfContents:")
+    lines.append("    visible: false")
+    lines.append("  outline:")
+    lines.append("    visible: true")
+    lines.append("  pagination:")
+    lines.append("    visible: true")
+    lines.append("  metadata:")
+    lines.append("    visible: true")
+    lines.append("---")
+    lines.append("")
+
     lines.append("# AutoGPT Blocks Overview")
     lines.append("")
     lines.append(
         'AutoGPT uses a modular approach with various "blocks" to handle different tasks. These blocks are the building blocks of AutoGPT workflows, allowing users to create complex automations by combining simple, specialized components.'
     )
     lines.append("")
-    lines.append('!!! info "Creating Your Own Blocks"')
-    lines.append("    Want to create your own custom blocks? Check out our guides:")
-    lines.append("    ")
+    lines.append('{% hint style="info" %}')
+    lines.append("**Creating Your Own Blocks**")
+    lines.append("")
+    lines.append("Want to create your own custom blocks? Check out our guides:")
+    lines.append("")
     lines.append(
-        "    - [Build your own Blocks](https://docs.agpt.co/platform/new_blocks/) - Step-by-step tutorial with examples"
+        "* [Build your own Blocks](https://docs.agpt.co/platform/new_blocks/) - Step-by-step tutorial with examples"
     )
     lines.append(
-        "    - [Block SDK Guide](https://docs.agpt.co/platform/block-sdk-guide/) - Advanced SDK patterns with OAuth, webhooks, and provider configuration"
+        "* [Block SDK Guide](https://docs.agpt.co/platform/block-sdk-guide/) - Advanced SDK patterns with OAuth, webhooks, and provider configuration"
     )
+    lines.append("{% endhint %}")
     lines.append("")
     lines.append(
         "Below is a comprehensive list of all available blocks, categorized by their primary function. Click on any block name to view its detailed documentation."
@@ -537,7 +575,8 @@ def generate_overview_table(blocks: list[BlockDoc]) -> str:
                     else "No description"
                 )
                 short_desc = short_desc.replace("\n", " ").replace("|", "\\|")
-                lines.append(f"| [{block.name}]({file_path}#{anchor}) | {short_desc} |")
+                link_path = f"{block_dir_prefix}{file_path}"
+                lines.append(f"| [{block.name}]({link_path}#{anchor}) | {short_desc} |")
             lines.append("")
             continue
 
@@ -563,13 +602,55 @@ def generate_overview_table(blocks: list[BlockDoc]) -> str:
             )
             short_desc = short_desc.replace("\n", " ").replace("|", "\\|")
 
-            lines.append(f"| [{block.name}]({file_path}#{anchor}) | {short_desc} |")
+            link_path = f"{block_dir_prefix}{file_path}"
+            lines.append(f"| [{block.name}]({link_path}#{anchor}) | {short_desc} |")
 
         lines.append("")
 
     return "\n".join(lines)
 
 
+def generate_summary_md(
+    blocks: list[BlockDoc], root_dir: Path, block_dir_prefix: str = ""
+) -> str:
+    """Generate SUMMARY.md for GitBook navigation.
+
+    Args:
+        blocks: List of block documentation objects
+        root_dir: The root docs directory (e.g., docs/integrations/)
+        block_dir_prefix: Prefix for block file links (e.g., "block-integrations/")
+    """
+    lines = []
+    lines.append("# Table of contents")
+    lines.append("")
+    lines.append("* [AutoGPT Blocks Overview](README.md)")
+    lines.append("")
+
+    # Check for guides/ directory at the root level (docs/integrations/guides/)
+    guides_dir = root_dir / "guides"
+    if guides_dir.exists():
+        lines.append("## Guides")
+        lines.append("")
+        for guide_file in sorted(guides_dir.glob("*.md")):
+            # Use just the file name for title (replace hyphens/underscores with spaces)
+            title = file_path_to_title(guide_file.stem.replace("-", "_") + ".md")
+            lines.append(f"* [{title}](guides/{guide_file.name})")
+        lines.append("")
+
+    lines.append("## Block Integrations")
+    lines.append("")
+
+    file_mapping = get_block_file_mapping(blocks)
+    for file_path in sorted(file_mapping.keys()):
+        title = file_path_to_title(file_path)
+        link_path = f"{block_dir_prefix}{file_path}"
+        lines.append(f"* [{title}]({link_path})")
+
+    lines.append("")
+
+    return "\n".join(lines)
+
+
 def load_all_blocks_for_docs() -> list[BlockDoc]:
     """Load all blocks and extract documentation."""
     from backend.blocks import load_all_blocks
@@ -653,6 +734,16 @@ def write_block_docs(
                 )
             )
 
+        # Add file-level additional_content section if present
+        file_additional = extract_manual_content(existing_content).get(
+            "additional_content", ""
+        )
+        if file_additional:
+            content_parts.append("<!-- MANUAL: additional_content -->")
+            content_parts.append(file_additional)
+            content_parts.append("<!-- END MANUAL -->")
+            content_parts.append("")
+
         full_content = file_header + "\n" + "\n".join(content_parts)
         generated_files[str(file_path)] = full_content
 
@@ -661,14 +752,28 @@ def write_block_docs(
 
         full_path.write_text(full_content)
 
-    # Generate overview file
-    overview_content = generate_overview_table(blocks)
-    overview_path = output_dir / "README.md"
+    # Generate overview file at the parent directory (docs/integrations/)
+    # with links prefixed to point into block-integrations/
+    root_dir = output_dir.parent
+    block_dir_name = output_dir.name  # "block-integrations"
+    block_dir_prefix = f"{block_dir_name}/"
+
+    overview_content = generate_overview_table(blocks, block_dir_prefix)
+    overview_path = root_dir / "README.md"
     generated_files["README.md"] = overview_content
     overview_path.write_text(overview_content)
 
     if verbose:
-        print("  Writing README.md (overview)")
+        print("  Writing README.md (overview) to parent directory")
+
+    # Generate SUMMARY.md for GitBook navigation at the parent directory
+    summary_content = generate_summary_md(blocks, root_dir, block_dir_prefix)
+    summary_path = root_dir / "SUMMARY.md"
+    generated_files["SUMMARY.md"] = summary_content
+    summary_path.write_text(summary_content)
+
+    if verbose:
+        print("  Writing SUMMARY.md (navigation) to parent directory")
 
     return generated_files
 
@@ -748,6 +853,16 @@ def check_docs_in_sync(output_dir: Path, blocks: list[BlockDoc]) -> bool:
             elif block_match.group(1).strip() != expected_block_content.strip():
                 mismatched_blocks.append(block.name)
 
+        # Add file-level additional_content to expected content (matches write_block_docs)
+        file_additional = extract_manual_content(existing_content).get(
+            "additional_content", ""
+        )
+        if file_additional:
+            content_parts.append("<!-- MANUAL: additional_content -->")
+            content_parts.append(file_additional)
+            content_parts.append("<!-- END MANUAL -->")
+            content_parts.append("")
+
         expected_content = file_header + "\n" + "\n".join(content_parts)
 
         if existing_content.strip() != expected_content.strip():
@@ -757,11 +872,15 @@ def check_docs_in_sync(output_dir: Path, blocks: list[BlockDoc]) -> bool:
             out_of_sync_details.append((file_path, mismatched_blocks))
             all_match = False
 
-    # Check overview
-    overview_path = output_dir / "README.md"
+    # Check overview at the parent directory (docs/integrations/)
+    root_dir = output_dir.parent
+    block_dir_name = output_dir.name  # "block-integrations"
+    block_dir_prefix = f"{block_dir_name}/"
+
+    overview_path = root_dir / "README.md"
     if overview_path.exists():
         existing_overview = overview_path.read_text()
-        expected_overview = generate_overview_table(blocks)
+        expected_overview = generate_overview_table(blocks, block_dir_prefix)
         if existing_overview.strip() != expected_overview.strip():
             print("OUT OF SYNC: README.md (overview)")
             print("  The blocks overview table needs regeneration")
@@ -772,6 +891,21 @@ def check_docs_in_sync(output_dir: Path, blocks: list[BlockDoc]) -> bool:
         out_of_sync_details.append(("README.md", ["overview table"]))
         all_match = False
 
+    # Check SUMMARY.md at the parent directory
+    summary_path = root_dir / "SUMMARY.md"
+    if summary_path.exists():
+        existing_summary = summary_path.read_text()
+        expected_summary = generate_summary_md(blocks, root_dir, block_dir_prefix)
+        if existing_summary.strip() != expected_summary.strip():
+            print("OUT OF SYNC: SUMMARY.md (navigation)")
+            print("  The GitBook navigation needs regeneration")
+            out_of_sync_details.append(("SUMMARY.md", ["navigation"]))
+            all_match = False
+    else:
+        print("MISSING: SUMMARY.md (navigation)")
+        out_of_sync_details.append(("SUMMARY.md", ["navigation"]))
+        all_match = False
+
     # Check for unfilled manual sections
     unfilled_patterns = [
         "_Add a description of this category of blocks._",

docs/integrations/SUMMARY.md

@@ -0,0 +1,133 @@
+# Table of contents
+
+* [AutoGPT Blocks Overview](README.md)
+
+## Guides
+
+* [LLM Providers](guides/llm-providers.md)
+* [Voice Providers](guides/voice-providers.md)
+
+## Block Integrations
+
+* [Airtable Bases](block-integrations/airtable/bases.md)
+* [Airtable Records](block-integrations/airtable/records.md)
+* [Airtable Schema](block-integrations/airtable/schema.md)
+* [Airtable Triggers](block-integrations/airtable/triggers.md)
+* [Apollo Organization](block-integrations/apollo/organization.md)
+* [Apollo People](block-integrations/apollo/people.md)
+* [Apollo Person](block-integrations/apollo/person.md)
+* [Ayrshare Post To Bluesky](block-integrations/ayrshare/post_to_bluesky.md)
+* [Ayrshare Post To Facebook](block-integrations/ayrshare/post_to_facebook.md)
+* [Ayrshare Post To GMB](block-integrations/ayrshare/post_to_gmb.md)
+* [Ayrshare Post To Instagram](block-integrations/ayrshare/post_to_instagram.md)
+* [Ayrshare Post To LinkedIn](block-integrations/ayrshare/post_to_linkedin.md)
+* [Ayrshare Post To Pinterest](block-integrations/ayrshare/post_to_pinterest.md)
+* [Ayrshare Post To Reddit](block-integrations/ayrshare/post_to_reddit.md)
+* [Ayrshare Post To Snapchat](block-integrations/ayrshare/post_to_snapchat.md)
+* [Ayrshare Post To Telegram](block-integrations/ayrshare/post_to_telegram.md)
+* [Ayrshare Post To Threads](block-integrations/ayrshare/post_to_threads.md)
+* [Ayrshare Post To TikTok](block-integrations/ayrshare/post_to_tiktok.md)
+* [Ayrshare Post To X](block-integrations/ayrshare/post_to_x.md)
+* [Ayrshare Post To YouTube](block-integrations/ayrshare/post_to_youtube.md)
+* [Baas Bots](block-integrations/baas/bots.md)
+* [Bannerbear Text Overlay](block-integrations/bannerbear/text_overlay.md)
+* [Basic](block-integrations/basic.md)
+* [Compass Triggers](block-integrations/compass/triggers.md)
+* [Data](block-integrations/data.md)
+* [Dataforseo Keyword Suggestions](block-integrations/dataforseo/keyword_suggestions.md)
+* [Dataforseo Related Keywords](block-integrations/dataforseo/related_keywords.md)
+* [Discord Bot Blocks](block-integrations/discord/bot_blocks.md)
+* [Discord OAuth Blocks](block-integrations/discord/oauth_blocks.md)
+* [Enrichlayer LinkedIn](block-integrations/enrichlayer/linkedin.md)
+* [Exa Answers](block-integrations/exa/answers.md)
+* [Exa Code Context](block-integrations/exa/code_context.md)
+* [Exa Contents](block-integrations/exa/contents.md)
+* [Exa Research](block-integrations/exa/research.md)
+* [Exa Search](block-integrations/exa/search.md)
+* [Exa Similar](block-integrations/exa/similar.md)
+* [Exa Webhook Blocks](block-integrations/exa/webhook_blocks.md)
+* [Exa Websets](block-integrations/exa/websets.md)
+* [Exa Websets Enrichment](block-integrations/exa/websets_enrichment.md)
+* [Exa Websets Import Export](block-integrations/exa/websets_import_export.md)
+* [Exa Websets Items](block-integrations/exa/websets_items.md)
+* [Exa Websets Monitor](block-integrations/exa/websets_monitor.md)
+* [Exa Websets Polling](block-integrations/exa/websets_polling.md)
+* [Exa Websets Search](block-integrations/exa/websets_search.md)
+* [Fal AI Video Generator](block-integrations/fal/ai_video_generator.md)
+* [Firecrawl Crawl](block-integrations/firecrawl/crawl.md)
+* [Firecrawl Extract](block-integrations/firecrawl/extract.md)
+* [Firecrawl Map](block-integrations/firecrawl/map.md)
+* [Firecrawl Scrape](block-integrations/firecrawl/scrape.md)
+* [Firecrawl Search](block-integrations/firecrawl/search.md)
+* [Generic Webhook Triggers](block-integrations/generic_webhook/triggers.md)
+* [GitHub Checks](block-integrations/github/checks.md)
+* [GitHub CI](block-integrations/github/ci.md)
+* [GitHub Issues](block-integrations/github/issues.md)
+* [GitHub Pull Requests](block-integrations/github/pull_requests.md)
+* [GitHub Repo](block-integrations/github/repo.md)
+* [GitHub Reviews](block-integrations/github/reviews.md)
+* [GitHub Statuses](block-integrations/github/statuses.md)
+* [GitHub Triggers](block-integrations/github/triggers.md)
+* [Google Calendar](block-integrations/google/calendar.md)
+* [Google Docs](block-integrations/google/docs.md)
+* [Google Gmail](block-integrations/google/gmail.md)
+* [Google Sheets](block-integrations/google/sheets.md)
+* [HubSpot Company](block-integrations/hubspot/company.md)
+* [HubSpot Contact](block-integrations/hubspot/contact.md)
+* [HubSpot Engagement](block-integrations/hubspot/engagement.md)
+* [Jina Chunking](block-integrations/jina/chunking.md)
+* [Jina Embeddings](block-integrations/jina/embeddings.md)
+* [Jina Fact Checker](block-integrations/jina/fact_checker.md)
+* [Jina Search](block-integrations/jina/search.md)
+* [Linear Comment](block-integrations/linear/comment.md)
+* [Linear Issues](block-integrations/linear/issues.md)
+* [Linear Projects](block-integrations/linear/projects.md)
+* [LLM](block-integrations/llm.md)
+* [Logic](block-integrations/logic.md)
+* [Misc](block-integrations/misc.md)
+* [Multimedia](block-integrations/multimedia.md)
+* [Notion Create Page](block-integrations/notion/create_page.md)
+* [Notion Read Database](block-integrations/notion/read_database.md)
+* [Notion Read Page](block-integrations/notion/read_page.md)
+* [Notion Read Page Markdown](block-integrations/notion/read_page_markdown.md)
+* [Notion Search](block-integrations/notion/search.md)
+* [Nvidia Deepfake](block-integrations/nvidia/deepfake.md)
+* [Replicate Flux Advanced](block-integrations/replicate/flux_advanced.md)
+* [Replicate Replicate Block](block-integrations/replicate/replicate_block.md)
+* [Search](block-integrations/search.md)
+* [Slant3D Filament](block-integrations/slant3d/filament.md)
+* [Slant3D Order](block-integrations/slant3d/order.md)
+* [Slant3D Slicing](block-integrations/slant3d/slicing.md)
+* [Slant3D Webhook](block-integrations/slant3d/webhook.md)
+* [Smartlead Campaign](block-integrations/smartlead/campaign.md)
+* [Stagehand Blocks](block-integrations/stagehand/blocks.md)
+* [System Library Operations](block-integrations/system/library_operations.md)
+* [System Store Operations](block-integrations/system/store_operations.md)
+* [Text](block-integrations/text.md)
+* [Todoist Comments](block-integrations/todoist/comments.md)
+* [Todoist Labels](block-integrations/todoist/labels.md)
+* [Todoist Projects](block-integrations/todoist/projects.md)
+* [Todoist Sections](block-integrations/todoist/sections.md)
+* [Todoist Tasks](block-integrations/todoist/tasks.md)
+* [Twitter Blocks](block-integrations/twitter/blocks.md)
+* [Twitter Bookmark](block-integrations/twitter/bookmark.md)
+* [Twitter Follows](block-integrations/twitter/follows.md)
+* [Twitter Hide](block-integrations/twitter/hide.md)
+* [Twitter Like](block-integrations/twitter/like.md)
+* [Twitter List Follows](block-integrations/twitter/list_follows.md)
+* [Twitter List Lookup](block-integrations/twitter/list_lookup.md)
+* [Twitter List Members](block-integrations/twitter/list_members.md)
+* [Twitter List Tweets Lookup](block-integrations/twitter/list_tweets_lookup.md)
+* [Twitter Manage](block-integrations/twitter/manage.md)
+* [Twitter Manage Lists](block-integrations/twitter/manage_lists.md)
+* [Twitter Mutes](block-integrations/twitter/mutes.md)
+* [Twitter Pinned Lists](block-integrations/twitter/pinned_lists.md)
+* [Twitter Quote](block-integrations/twitter/quote.md)
+* [Twitter Retweet](block-integrations/twitter/retweet.md)
+* [Twitter Search Spaces](block-integrations/twitter/search_spaces.md)
+* [Twitter Spaces Lookup](block-integrations/twitter/spaces_lookup.md)
+* [Twitter Timeline](block-integrations/twitter/timeline.md)
+* [Twitter Tweet Lookup](block-integrations/twitter/tweet_lookup.md)
+* [Twitter User Lookup](block-integrations/twitter/user_lookup.md)
+* [Wolfram LLM API](block-integrations/wolfram/llm_api.md)
+* [Zerobounce Validate Emails](block-integrations/zerobounce/validate_emails.md)