From aa2577be91c55b9b3cc188c2267f65722115a26b Mon Sep 17 00:00:00 2001
From: Aditya Tripathi <aditya@climactic.co>
Date: Tue, 10 Jun 2025 21:02:37 +0530
Subject: [PATCH] feat(llms): add LLM text processing and routing for MDX and
 TXT files (#470)

---
 apps/docs/app/llms.mdx/[[...slug]]/route.ts | 18 +++++++++++++++++
 apps/docs/app/llms.txt/route.ts             | 12 +++++++++++
 apps/docs/lib/llms.ts                       | 22 +++++++++++++++++++++
 apps/docs/next.config.ts                    |  5 +++++
 4 files changed, 57 insertions(+)
 create mode 100644 apps/docs/app/llms.mdx/[[...slug]]/route.ts
 create mode 100644 apps/docs/app/llms.txt/route.ts
 create mode 100644 apps/docs/lib/llms.ts

diff --git a/apps/docs/app/llms.mdx/[[...slug]]/route.ts b/apps/docs/app/llms.mdx/[[...slug]]/route.ts
new file mode 100644
index 000000000..837dce08d
--- /dev/null
+++ b/apps/docs/app/llms.mdx/[[...slug]]/route.ts
@@ -0,0 +1,18 @@
+import { notFound } from 'next/navigation'
+import { type NextRequest, NextResponse } from 'next/server'
+import { getLLMText } from '@/lib/llms'
+import { source } from '@/lib/source'
+
+export const revalidate = false
+
+export async function GET(_req: NextRequest, { params }: { params: Promise<{ slug?: string[] }> }) {
+  const { slug } = await params
+  const page = source.getPage(slug)
+  if (!page) notFound()
+
+  return new NextResponse(await getLLMText(page))
+}
+
+export function generateStaticParams() {
+  return source.generateParams()
+}
diff --git a/apps/docs/app/llms.txt/route.ts b/apps/docs/app/llms.txt/route.ts
new file mode 100644
index 000000000..6496e0d5a
--- /dev/null
+++ b/apps/docs/app/llms.txt/route.ts
@@ -0,0 +1,12 @@
+import { getLLMText } from '@/lib/llms'
+import { source } from '@/lib/source'
+
+// cached forever
+export const revalidate = false
+
+export async function GET() {
+  const scan = source.getPages().map(getLLMText)
+  const scanned = await Promise.all(scan)
+
+  return new Response(scanned.join('\n\n'))
+}
diff --git a/apps/docs/lib/llms.ts b/apps/docs/lib/llms.ts
new file mode 100644
index 000000000..86e221c76
--- /dev/null
+++ b/apps/docs/lib/llms.ts
@@ -0,0 +1,22 @@
+import type { InferPageType } from 'fumadocs-core/source'
+import { remarkInclude } from 'fumadocs-mdx/config'
+import { remark } from 'remark'
+import remarkGfm from 'remark-gfm'
+import remarkMdx from 'remark-mdx'
+import type { source } from '@/lib/source'
+
+const processor = remark().use(remarkMdx).use(remarkInclude).use(remarkGfm)
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: page.data.content,
+  })
+
+  return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`
+}
diff --git a/apps/docs/next.config.ts b/apps/docs/next.config.ts
index 89c0ccd5d..fbaa560eb 100644
--- a/apps/docs/next.config.ts
+++ b/apps/docs/next.config.ts
@@ -12,6 +12,11 @@ const config = {
         destination: '/introduction',
         permanent: true,
       },
+      {
+        source: '/docs/:path*.mdx',
+        destination: '/llms.mdx/:path*',
+        permanent: true,
+      },
     ]
   },
 }