chore: Routing Refactor (#3)

* scrape with 1:1 mapping to origin repo

* exclude .md extension from file path in URLs inside MDs

* removed legacy static files

* remove image path manipulation

* move scrapper to new folder

* sidebar custom ordering implemented

docusaurus-utils/scrapper/fetch-content.mjs

@@ -43,7 +43,8 @@ export async function fetchDirectoryContents(dirUrl, basePath, prefixToRemove) {
     for (const file of files) {
       const prefixRemovalRegex = new RegExp(`^${prefixToRemove}`)
       const relativePath = file.path.replace(prefixRemovalRegex, '')
-      const filePath = path.join(basePath, adjustPathForMarkdown(relativePath))
+      // const filePath = path.join(basePath, adjustPathForMarkdown(relativePath))
+      const filePath = path.join(basePath, relativePath)
 
       if (file.type === 'file') {
         await downloadAndModifyFile(file.download_url, filePath)

docusaurus-utils/scrapper/markdown-convertor.mjs

@@ -86,10 +86,16 @@ function unescapeHtmlComments(htmlString) {
   return htmlString.replace(/\\<\!--/g, '\n<!--').replace(/--\\>/g, '-->\n')
 }
 
-function updateMarkdownImagePath(content, number) {
+function updateMarkdownLinksToExcludeMD(content) {
-  const regex = /(!\[.*?\]\(\.\/)images/g
+  function replaceLinks(match, p1, p2, p3) {
+    let url = p2.replace(/\.md$/, ''); // Remove .md extension from URL
+    let anchor = p3.replace(/^\//, ''); // Remove preceding '/' from anchor if exists
+    return `[${p1}](${url}${anchor ? '#' + anchor : ''})`;
+  }
 
-  return content.replace(regex, `$1${number}/images`)
+  const regex = /\[((?:(?!\]).)+)\]\(([^)]*?\.md)(?:\/#|\/#)?([^)]*)\)/g
+
+  return content.replace(regex, replaceLinks)
 }
 
 export function vacMarkdownToDocusaurusMarkdown(fileContent) {
@@ -113,23 +119,9 @@ export function vacMarkdownToDocusaurusMarkdown(fileContent) {
 
   convertedContent = unescapeHtmlComments(convertedContent)
 
-  // // parse sidebarPosition from the slug in the frontmatter
-  const sidebarPosition = parseSlugFromFrontmatter(convertedContent) || 1
-
   convertedContent = enhanceMarkdownWithBulletPointsCorrected(convertedContent)
 
-  convertedContent = updateMarkdownImagePath(convertedContent, sidebarPosition)
+  convertedContent = updateMarkdownLinksToExcludeMD(convertedContent)
-
-  // Insert sidebar_position at the end of frontmatter if it doesn't exist
-  if (
-    /^---\s*[\s\S]+?---/.test(convertedContent) &&
-    !/sidebar_position: \d+/.test(convertedContent)
-  ) {
-    convertedContent = convertedContent.replace(
-      /^---\s*([\s\S]+?)---/,
-      `---\n$1sidebar_position: ${sidebarPosition}\n---`,
-    )
-  }
 
   return convertedContent;
 }

docusaurus-utils/sidebar/generator.js

@@ -0,0 +1,28 @@
+const { compose } = require("./utils");
+const {
+  positionDefaultReadmeToTop,
+  removeRFCNumberedDirectories,
+  separateFoldersAndFilesOrder,
+  orderAlphabeticallyAndByNumber
+} = require("./modifiers")
+
+async function sidebarItemsGenerator({defaultSidebarItemsGenerator, ...args}) {
+  const defaultSidebarItems = await defaultSidebarItemsGenerator(args);
+
+  /*
+  We'll have multiple O(N) passes through the items depending on the reducer implementation,
+  but we'll sacrifice very small performance for sake of easier maintainability
+  */
+  const sidebarModifier = compose(
+    positionDefaultReadmeToTop,
+    separateFoldersAndFilesOrder,
+    removeRFCNumberedDirectories,
+    orderAlphabeticallyAndByNumber
+  )
+
+  return sidebarModifier(defaultSidebarItems)
+}
+
+module.exports = {
+  sidebarItemsGenerator
+}

docusaurus-utils/sidebar/helpers.js

@@ -0,0 +1,15 @@
+function isIndexDocument(documentId, parentDirectory) {
+  if (!documentId) {
+    return false
+  }
+
+  return (
+    documentId.toUpperCase() === "README" ||
+    documentId.toUpperCase() === "INDEX" ||
+    (!!parentDirectory && documentId.toUpperCase() === parentDirectory.toUpperCase())
+  )
+}
+
+module.exports = {
+  isIndexDocument
+}

docusaurus-utils/sidebar/modifiers.js

@@ -0,0 +1,89 @@
+const { isNumber } = require('./utils')
+const {
+  isIndexDocument
+} = require("./helpers")
+
+function orderAlphabeticallyAndByNumber(sidebarItems) {
+  let newSidebarItems = [...sidebarItems]
+
+  for (let i = 0; i < newSidebarItems.length; i++) {
+    const sidebarItem = newSidebarItems[i];
+
+    if (sidebarItem.type === 'category' && sidebarItem.items && sidebarItem.items.length > 1) {
+      newSidebarItems[i] = {
+        ...sidebarItem,
+        items: orderAlphabeticallyAndByNumber(sidebarItem.items)
+      }
+    }
+  }
+
+  const collator = new Intl.Collator(undefined, {numeric: true, sensitivity: 'base'});
+  const compareFunction = (a, b) => {
+    return collator.compare(a.label, b.label)
+  }
+
+  return newSidebarItems.sort(compareFunction)
+}
+
+function removeRFCNumberedDirectories(sidebarItems) {
+  let newSidebarItems = []
+
+  sidebarItems.forEach(sidebarItem => {
+    if (sidebarItem.type === 'category') {
+      const isRFCNumberedCategory = isNumber(sidebarItem.label);
+
+      if (isRFCNumberedCategory) {
+        newSidebarItems = [...newSidebarItems, ...sidebarItem.items]
+      } else {
+        const newSidebarItem = {
+          ...sidebarItem,
+          items: removeRFCNumberedDirectories(sidebarItem.items)
+        }
+        newSidebarItems = [...newSidebarItems, newSidebarItem]
+      }
+    } else {
+      newSidebarItems = [...newSidebarItems, sidebarItem]
+    }
+  })
+
+  return newSidebarItems
+}
+
+function separateFoldersAndFilesOrder(sidebarItems) {
+  let categories = [];
+  let pages = [];
+
+  sidebarItems.forEach(sidebarItem => {
+    if (sidebarItem.type === 'category') {
+      categories = [...categories, sidebarItem]
+    } else {
+      pages = [...pages, sidebarItem]
+    }
+  })
+
+  return [
+    ...categories,
+    ...pages
+  ]
+}
+
+function positionDefaultReadmeToTop(sidebarItems) {
+  let newSidebarItems = []
+
+  sidebarItems.forEach(sidebarItem => {
+    if (sidebarItem.type === "doc" && isIndexDocument(sidebarItem.id)) {
+      newSidebarItems = [sidebarItem, ...newSidebarItems]
+    } else {
+      newSidebarItems = [...newSidebarItems, sidebarItem]
+    }
+  })
+
+  return newSidebarItems
+}
+
+module.exports = {
+  orderAlphabeticallyAndByNumber,
+  removeRFCNumberedDirectories,
+  separateFoldersAndFilesOrder,
+  positionDefaultReadmeToTop,
+}

docusaurus-utils/sidebar/utils.js

@@ -0,0 +1,27 @@
+function isNumber(value) {
+  if (true === Array.isArray(value)) {
+    return false;
+  }
+  return !isNaN(parseInt(value, 10));
+}
+
+/*
+ Composes multiple functions with same arguments into a single one
+ NOTE: Functions are executed from end of array to start (right to left)
+ */
+function compose(...funcs) {
+  if (funcs.length === 1) {
+    return funcs[0]
+  }
+
+  return funcs.reduce(
+    (firstFunction, nextFunction) =>
+      (...args) =>
+        firstFunction(nextFunction(...args))
+  )
+}
+
+module.exports = {
+  isNumber,
+  compose
+}

docusaurus.config.js

@@ -1,6 +1,7 @@
 // @ts-check
 // Note: type annotations allow type checking and IDEs autocompletion
 require('dotenv').config()
+const { sidebarItemsGenerator } = require("./docusaurus-utils/sidebar/generator")
 
 /** @type {import('@docusaurus/types').Config} */
 const config = {
@@ -29,6 +30,7 @@ const config = {
         id: 'codex',
         path: 'codex',
         routeBasePath: 'codex',
+        sidebarItemsGenerator,
       },
     ],
     [
@@ -37,6 +39,7 @@ const config = {
         id: 'nomos',
         path: 'nomos',
         routeBasePath: 'nomos',
+        sidebarItemsGenerator,
       },
     ],
     [
@@ -45,6 +48,7 @@ const config = {
         id: 'status',
         path: 'status',
         routeBasePath: 'status',
+        sidebarItemsGenerator,
       },
     ],
     [
@@ -53,6 +57,7 @@ const config = {
         id: 'vac',
         path: 'vac',
         routeBasePath: 'vac',
+        sidebarItemsGenerator,
       },
     ],
     [
@@ -61,6 +66,7 @@ const config = {
         id: 'waku',
         path: 'waku',
         routeBasePath: 'waku',
+        sidebarItemsGenerator,
       },
     ],
   ],

package.json

@@ -13,7 +13,7 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
-    "scrape": "node ./scrapper/main.mjs"
+    "scrape": "node ./docusaurus-utils/scrapper/main.mjs"
   },
   "dependencies": {
     "@acid-info/logos-docusaurus-preset": "^1.0.0-alpha.14",