From d60464374db0152f3141d8ec111f19a4e8ade7e9 Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Sat, 2 Mar 2024 09:27:11 -0500 Subject: [PATCH] Upgrade Quarto and use notebook metadata for frontmatter (#1836) * Update process_notebook to use metadata instead of a yaml comment * upgrade quarto and version check in tool * formatting * address comments --- .devcontainer/Dockerfile | 8 +- .devcontainer/dev/Dockerfile | 8 +- .github/workflows/deploy-website.yml | 12 +- notebook/agentchat_RetrieveChat.ipynb | 14 +- ...at_auto_feedback_from_code_execution.ipynb | 13 +- notebook/agentchat_function_call_async.ipynb | 10 +- notebook/agentchat_groupchat.ipynb | 10 +- notebook/agentchat_groupchat_RAG.ipynb | 18 +- ...tchat_groupchat_finite_state_machine.ipynb | 14 +- notebook/agentchat_logging.ipynb | 15 +- .../agentchat_multi_task_async_chats.ipynb | 12 +- notebook/agentchat_multi_task_chats.ipynb | 12 +- notebook/agentchat_nestedchat.ipynb | 10 +- notebook/agentchat_society_of_mind.ipynb | 10 +- notebook/agentchats_sequential_chats.ipynb | 22 +- notebook/contributing.md | 42 +++- website/docs/llm_configuration.ipynb | 4 - website/process_notebooks.py | 216 ++++++++---------- 18 files changed, 207 insertions(+), 243 deletions(-) mode change 100644 => 100755 website/process_notebooks.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0e24c6cd5..4431ee58b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,13 +14,13 @@ RUN apt-get update \ && apt-get -y install --no-install-recommends build-essential npm \ && apt-get autoremove -y \ && apt-get clean -y \ - && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.deb \ - && dpkg -i quarto-1.4.549-linux-amd64.deb \ - && rm -rf /var/lib/apt/lists/* quarto-1.4.549-linux-amd64.deb + && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.deb \ + && dpkg -i quarto-1.5.23-linux-amd64.deb \ + && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-amd64.deb ENV DEBIAN_FRONTEND=dialog # For docs RUN npm install --global yarn RUN pip install pydoc-markdown RUN pip install pyyaml -RUN pip install colored +RUN pip install colored diff --git a/.devcontainer/dev/Dockerfile b/.devcontainer/dev/Dockerfile index 482157437..4749e41ba 100644 --- a/.devcontainer/dev/Dockerfile +++ b/.devcontainer/dev/Dockerfile @@ -33,12 +33,12 @@ RUN cd website RUN yarn install --frozen-lockfile --ignore-engines RUN arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \ - wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-${arch}.tar.gz && \ + wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-${arch}.tar.gz && \ mkdir -p /home/autogen/quarto/ && \ - tar -xzf quarto-1.4.549-linux-${arch}.tar.gz --directory /home/autogen/quarto/ && \ - rm quarto-1.4.549-linux-${arch}.tar.gz + tar -xzf quarto-1.5.23-linux-${arch}.tar.gz --directory /home/autogen/quarto/ && \ + rm quarto-1.5.23-linux-${arch}.tar.gz -ENV PATH="${PATH}:/home/autogen/quarto/quarto-1.4.549/bin/" +ENV PATH="${PATH}:/home/autogen/quarto/quarto-1.5.23/bin/" # Exposes the Yarn port for Docusaurus EXPOSE 3000 diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index c9c7deede..ae866ffca 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -44,9 +44,9 @@ jobs: - name: quarto install working-directory: ${{ runner.temp }} run: | - wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.tar.gz - tar -xzf quarto-1.4.549-linux-amd64.tar.gz - echo "$(pwd)/quarto-1.4.549/bin/" >> $GITHUB_PATH + wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz + tar -xzf quarto-1.5.23-linux-amd64.tar.gz + echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH - name: quarto run run: | quarto render . @@ -90,9 +90,9 @@ jobs: - name: quarto install working-directory: ${{ runner.temp }} run: | - wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.549/quarto-1.4.549-linux-amd64.tar.gz - tar -xzf quarto-1.4.549-linux-amd64.tar.gz - echo "$(pwd)/quarto-1.4.549/bin/" >> $GITHUB_PATH + wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz + tar -xzf quarto-1.5.23-linux-amd64.tar.gz + echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH - name: quarto run run: | quarto render . diff --git a/notebook/agentchat_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat.ipynb index 6c965d0e7..92620a55b 100644 --- a/notebook/agentchat_RetrieveChat.ipynb +++ b/notebook/agentchat_RetrieveChat.ipynb @@ -5,12 +5,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "\n", "# Using RetrieveChat for Retrieve Augmented Code Generation and Question Answering\n", "\n", "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", @@ -3020,6 +3014,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["RAG"], + "description": "Explore the use of AutoGen's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning." + }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", @@ -3036,8 +3034,8 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" -}, -"test_skip": "Requires interactive usage" + }, + "skip_test": "Requires interactive usage" }, "nbformat": 4, "nbformat_minor": 4 diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb index 8f232975b..1598c66f8 100644 --- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb +++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb @@ -8,12 +8,6 @@ } }, "source": [ - "\n", - "\n", "# Task Solving with Code Generation, Execution and Debugging\n", "\n", "AutoGen offers conversable LLM agents, which can be used to solve various tasks with human or automatic feedback, including tasks that require using tools via code.\n", @@ -1098,6 +1092,13 @@ } ], "metadata": { + "front_matter": { + "tags": [ + "code generation", + "debugging" + ], + "description": "Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes." + }, "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/notebook/agentchat_function_call_async.ipynb b/notebook/agentchat_function_call_async.ipynb index e7da183f5..b30357259 100644 --- a/notebook/agentchat_function_call_async.ipynb +++ b/notebook/agentchat_function_call_async.ipynb @@ -5,12 +5,6 @@ "id": "ae1f50ec", "metadata": {}, "source": [ - "\n", - "\n", "# Task Solving with Provided Tools as Functions (Asynchronous Function Calls)\n" ] }, @@ -366,6 +360,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["code generation", "function call", "async"], + "description": "Learn how to implement both synchronous and asynchronous function calls using AssistantAgent and UserProxyAgent in AutoGen, with examples of their application in individual and group chat settings for task execution with language models." + }, "kernelspec": { "display_name": "flaml_dev", "language": "python", diff --git a/notebook/agentchat_groupchat.ipynb b/notebook/agentchat_groupchat.ipynb index 058b687b3..bb45ade83 100644 --- a/notebook/agentchat_groupchat.ipynb +++ b/notebook/agentchat_groupchat.ipynb @@ -5,12 +5,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "\n", "# Group Chat\n", "\n", "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", @@ -223,6 +217,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["orchestration", "group chat"], + "description": "Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources." + }, "kernelspec": { "display_name": "flaml", "language": "python", diff --git a/notebook/agentchat_groupchat_RAG.ipynb b/notebook/agentchat_groupchat_RAG.ipynb index 6e06a0ec2..50d35f5a9 100644 --- a/notebook/agentchat_groupchat_RAG.ipynb +++ b/notebook/agentchat_groupchat_RAG.ipynb @@ -1,17 +1,5 @@ { "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -1120,6 +1108,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["group chat", "orchestration", "RAG"], + "description": "Implement and manage a multi-agent chat system using AutoGen, where AI assistants retrieve information, generate code, and interact collaboratively to solve complex tasks, especially in areas not covered by their training data." + }, "kernelspec": { "display_name": "flaml", "language": "python", @@ -1135,7 +1127,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/notebook/agentchat_groupchat_finite_state_machine.ipynb b/notebook/agentchat_groupchat_finite_state_machine.ipynb index 3a05af280..e24bb5bbb 100644 --- a/notebook/agentchat_groupchat_finite_state_machine.ipynb +++ b/notebook/agentchat_groupchat_finite_state_machine.ipynb @@ -5,13 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "\n", - "# FSM - User can input speaker transition contraints.\n", + "# FSM - User can input speaker transition constraints\n", "\n", "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", @@ -597,6 +591,12 @@ } ], "metadata": { + "front_matter": { + "description": "Explore the demonstration of the Finite State Machine implementation, which allows the user to input speaker transition constraints.", + "tags": [ + "orchestration" + ] + }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", diff --git a/notebook/agentchat_logging.ipynb b/notebook/agentchat_logging.ipynb index 5d8a824e9..c06a8fd74 100644 --- a/notebook/agentchat_logging.ipynb +++ b/notebook/agentchat_logging.ipynb @@ -4,12 +4,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "\n", "# Runtime Logging with AutoGen \n", "\n", "AutoGen offers utilities to log data for debugging and performance analysis. This notebook demonstrates how to use them. \n", @@ -294,6 +288,13 @@ } ], "metadata": { + "front_matter": { + "description": "Provide capabilities of runtime logging for debugging and performance analysis.", + "tags": [ + "logging", + "debugging" + ] + }, "kernelspec": { "display_name": "autog", "language": "python", @@ -309,7 +310,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/notebook/agentchat_multi_task_async_chats.ipynb b/notebook/agentchat_multi_task_async_chats.ipynb index 16b46ab16..bc7d35dab 100644 --- a/notebook/agentchat_multi_task_async_chats.ipynb +++ b/notebook/agentchat_multi_task_async_chats.ipynb @@ -9,12 +9,6 @@ } }, "source": [ - "\n", - "\n", "# Solving Multiple Tasks in a Sequence of Async Chats\n", "\n", "This notebook showcases how to use the new chat interface of conversational agents in AutoGen: a_initiate_chats, to conduct a series of tasks. Similar to \"notebook/agentchat_microsoft_fabric.ipynb\", this new interface allows one to pass multiple tasks and their corresponding dedicated agents and execute concurrently. Depending on the prerequisite task(s), the tasks will be solved concurrently, with the summaries from prerequisite task(s) provided to subsequent tasks as context, if the `summary_method` argument is specified.\n", @@ -1484,6 +1478,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["sequential chat"], + "description": "Use conversational agents to solve a set of tasks with a sequence of async chats." + }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -1499,7 +1497,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.11.7" }, "vscode": { "interpreter": { diff --git a/notebook/agentchat_multi_task_chats.ipynb b/notebook/agentchat_multi_task_chats.ipynb index ab65f6e0f..3f10bb072 100644 --- a/notebook/agentchat_multi_task_chats.ipynb +++ b/notebook/agentchat_multi_task_chats.ipynb @@ -9,12 +9,6 @@ } }, "source": [ - "\n", - "\n", "# Solving Multiple Tasks in a Sequence of Chats\n", "\n", "This notebook showcases how to use the new chat interface of conversational agents in AutoGen: initiate_chats, to conduct a series of tasks. This new interface allows one to pass multiple tasks and their corresponding dedicated agents. Once initiate_chats is invoked, the tasks will be solved sequentially, with the summaries from previous tasks provided to subsequent tasks as context, if the `summary_method` argument is specified.\n", @@ -1536,6 +1530,12 @@ } ], "metadata": { + "front_matter": { + "description": "Use conversational agents to solve a set of tasks with a sequence of chats.", + "tags": [ + "sequential chat" + ] + }, "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/notebook/agentchat_nestedchat.ipynb b/notebook/agentchat_nestedchat.ipynb index f9a7f78ef..03a8afa8d 100644 --- a/notebook/agentchat_nestedchat.ipynb +++ b/notebook/agentchat_nestedchat.ipynb @@ -9,12 +9,6 @@ } }, "source": [ - "\n", - "\n", "# Solving Complex Tasks with Nested Chats\n", "\n", "This notebook shows how you can leverage \"nested chats\" to solve complex task with AutoGen. Nested chats allow AutoGen agents to use other agents as their inner monologue to accomplish tasks. This abstraction is powerful as it allows you to compose agents in rich ways. This notebook shows how you can nest a pretty complex sequence of chats among _inner_ agents inside an _outer_ agent.\n", @@ -814,6 +808,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["nested chat"], + "description": "Solve complex tasks with one or more sequence chats nested as inner monologue." + }, "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/notebook/agentchat_society_of_mind.ipynb b/notebook/agentchat_society_of_mind.ipynb index b395c1433..be9ef5521 100644 --- a/notebook/agentchat_society_of_mind.ipynb +++ b/notebook/agentchat_society_of_mind.ipynb @@ -5,12 +5,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "\n", "# SocietyOfMindAgent\n", "\n", "This notebook demonstrates the SocietyOfMindAgent, which runs a group chat as an internal monologue, but appears to the external world as a single agent. This confers three distinct advantages:\n", @@ -362,6 +356,10 @@ } ], "metadata": { + "front_matter": { + "tags": ["orchestration"], + "description": "Explore the demonstration of the SocietyOfMindAgent in the AutoGen library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints." + }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", diff --git a/notebook/agentchats_sequential_chats.ipynb b/notebook/agentchats_sequential_chats.ipynb index 22a3eafa9..9b5904050 100644 --- a/notebook/agentchats_sequential_chats.ipynb +++ b/notebook/agentchats_sequential_chats.ipynb @@ -1,16 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -20,11 +9,6 @@ } }, "source": [ - "\n", "# Solving Multiple Tasks in a Sequence of Chats\n", "\n", "This notebook showcases how to use the new chat interface `autogen.initiate_chats` to solve a set of tasks with a sequence of chats. \n", @@ -837,6 +821,12 @@ } ], "metadata": { + "front_matter": { + "description": "Use AutoGen to solve a set of tasks with a sequence of chats.", + "tags": [ + "sequential chats" + ] + }, "kernelspec": { "display_name": "Python 3", "language": "python", diff --git a/notebook/contributing.md b/notebook/contributing.md index 4234bce6d..c717939b2 100644 --- a/notebook/contributing.md +++ b/notebook/contributing.md @@ -2,16 +2,23 @@ ## How to get a notebook displayed on the website -Ensure the first cell is markdown and before absolutely anything else include the following yaml within a comment. +In the notebook metadata set the `tags` and `description` `front_matter` properties. For example: -```markdown - +```json +{ + "...": "...", + "metadata": { + "...": "...", + "front_matter": { + "tags": ["code generation", "debugging"], + "description": "Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes." + } + } +} ``` +**Note**: Notebook metadata can be edited by opening the notebook in a text editor (Or "Open With..." -> "Text Editor" in VSCode) + The `tags` field is a list of tags that will be used to categorize the notebook. The `description` field is a brief description of the notebook. ## Best practices for authoring notebooks @@ -102,9 +109,26 @@ If a notebook needs to be skipped then add to the notebook metadata: { "...": "...", "metadata": { - "test_skip": "REASON" + "skip_test": "REASON" } } ``` -Note: Notebook metadata can be edited by opening the notebook in a text editor (Or "Open With..." -> "Text Editor" in VSCode) +## Metadata fields + +All possible metadata fields are as follows: +```json +{ + "...": "...", + "metadata": { + "...": "...", + "front_matter": { + "tags": "List[str] - List of tags to categorize the notebook", + "description": "str - Brief description of the notebook", + }, + "skip_test": "str - Reason for skipping the test. If present, the notebook will be skipped during testing", + "skip_render": "str - Reason for skipping rendering the notebook. If present, the notebook will be left out of the website.", + "extra_files_to_copy": "List[str] - List of files to copy to the website. The paths are relative to the notebook directory", + } +} +``` diff --git a/website/docs/llm_configuration.ipynb b/website/docs/llm_configuration.ipynb index 5691f1d79..e7fad3cf3 100644 --- a/website/docs/llm_configuration.ipynb +++ b/website/docs/llm_configuration.ipynb @@ -4,10 +4,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "---\n", - "custom_edit_url: https://github.com/microsoft/autogen/edit/main/website/docs/llm_configuration.ipynb\n", - "---\n", - "\n", "# LLM Configuration\n", "\n", "In AutoGen, agents use LLMs as key components to understand and react. To configure an agent's access to LLMs, you can specify an `llm_config` argument in its constructor. For example, the following snippet shows a configuration that uses `gpt-4`:" diff --git a/website/process_notebooks.py b/website/process_notebooks.py old mode 100644 new mode 100755 index 835a69a35..8a2875c22 --- a/website/process_notebooks.py +++ b/website/process_notebooks.py @@ -15,9 +15,10 @@ import typing import concurrent.futures import os -from typing import Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union from dataclasses import dataclass + from multiprocessing import current_process try: @@ -26,6 +27,7 @@ except ImportError: print("pyyaml not found.\n\nPlease install pyyaml:\n\tpip install pyyaml\n") sys.exit(1) + try: import nbclient from nbclient.client import ( @@ -65,7 +67,12 @@ class Result: def check_quarto_bin(quarto_bin: str = "quarto") -> None: """Check if quarto is installed.""" try: - subprocess.check_output([quarto_bin, "--version"]) + version = subprocess.check_output([quarto_bin, "--version"], text=True).strip() + version = tuple(map(int, version.split("."))) + if version < (1, 5, 23): + print("Quarto version is too old. Please upgrade to 1.5.23 or later.") + sys.exit(1) + except FileNotFoundError: print("Quarto is not installed. Please install it from https://quarto.org") sys.exit(1) @@ -76,32 +83,9 @@ def notebooks_target_dir(website_directory: Path) -> Path: return website_directory / "docs" / "notebooks" -def extract_yaml_from_notebook(notebook: Path) -> typing.Optional[typing.Dict]: - with open(notebook, "r", encoding="utf-8") as f: - content = f.read() - - json_content = json.loads(content) - first_cell = json_content["cells"][0] - - # must exists on lines on their own - if first_cell["cell_type"] != "markdown": - return None - - lines = first_cell["source"] - if "" not in lines: - return None - - closing_arrow_idx = lines.index("-->") - - front_matter_lines = lines[1:closing_arrow_idx] - front_matter = yaml.safe_load("\n".join(front_matter_lines)) - return front_matter +def load_metadata(notebook: Path) -> typing.Dict: + content = json.load(notebook.open()) + return content["metadata"] def skip_reason_or_none_if_ok(notebook: Path) -> typing.Optional[str]: @@ -125,29 +109,17 @@ def skip_reason_or_none_if_ok(notebook: Path) -> typing.Optional[str]: first_cell = json_content["cells"][0] # must exists on lines on their own - if first_cell["cell_type"] != "markdown": - return "first cell is not markdown" + if first_cell["cell_type"] == "markdown" and first_cell["source"][0].strip() == "" not in lines: - return "no closing --> found, or it is not on a line on its own" - - try: - front_matter = extract_yaml_from_notebook(notebook) - except yaml.YAMLError as e: - return colored(f"Failed to parse front matter in {notebook.name}: {e}", "red") - - # Should not be none at this point as we have already done the same checks as in extract_yaml_from_notebook - assert front_matter is not None, f"Front matter is None for {notebook.name}" - - if "skip" in front_matter and front_matter["skip"] is True: - return "skip is set to true" + front_matter = metadata["front_matter"] if "tags" not in front_matter: return "tags is not in front matter" @@ -171,16 +143,20 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path, in_notebook_dir = "notebook" in src_notebook.parts + metadata = load_metadata(src_notebook) + if "skip_render" in metadata: + return fmt_skip(src_notebook, "skip_render is in notebook metadata") + if in_notebook_dir: - relative_notebook = src_notebook.relative_to(notebook_dir) + relative_notebook = src_notebook.resolve().relative_to(notebook_dir.resolve()) dest_dir = notebooks_target_dir(website_directory=website_dir) - target_mdx_file = dest_dir / relative_notebook.with_suffix(".mdx") + target_file = dest_dir / relative_notebook.with_suffix(".mdx") intermediate_notebook = dest_dir / relative_notebook # If the intermediate_notebook already exists, check if it is newer than the source file - if target_mdx_file.exists(): - if target_mdx_file.stat().st_mtime > src_notebook.stat().st_mtime: - return colored(f"Skipping {src_notebook.name}, as target file is newer", "blue") + if target_file.exists(): + if target_file.stat().st_mtime > src_notebook.stat().st_mtime: + return fmt_skip(src_notebook, f"target file ({target_file.name}) is newer ☑️") if dry_run: return colored(f"Would process {src_notebook.name}", "green") @@ -191,11 +167,8 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path, # Check if another file has to be copied too # Solely added for the purpose of agent_library_example.json - front_matter = extract_yaml_from_notebook(src_notebook) - # Should not be none at this point as we have already done the same checks as in extract_yaml_from_notebook - assert front_matter is not None, f"Front matter is None for {src_notebook.name}" - if "extra_files_to_copy" in front_matter: - for file in front_matter["extra_files_to_copy"]: + if "extra_files_to_copy" in metadata: + for file in metadata["extra_files_to_copy"]: shutil.copy(src_notebook.parent / file, dest_dir / file) # Capture output @@ -203,28 +176,19 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path, [quarto_bin, "render", intermediate_notebook], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) if result.returncode != 0: - return ( - colored(f"Failed to render {intermediate_notebook}", "red") - + f"\n{result.stderr}" - + f"\n{result.stdout}" + return fmt_error( + src_notebook, f"Failed to render {src_notebook}\n\nstderr:\n{result.stderr}\nstdout:\n{result.stdout}" ) # Unlink intermediate files intermediate_notebook.unlink() - - if "extra_files_to_copy" in front_matter: - for file in front_matter["extra_files_to_copy"]: - (dest_dir / file).unlink() - - # Post process the file - post_process_mdx(target_mdx_file) else: - target_mdx_file = src_notebook.with_suffix(".mdx") + target_file = src_notebook.with_suffix(".mdx") # If the intermediate_notebook already exists, check if it is newer than the source file - if target_mdx_file.exists(): - if target_mdx_file.stat().st_mtime > src_notebook.stat().st_mtime: - return colored(f"Skipping {src_notebook.name}, as target file is newer", "blue") + if target_file.exists(): + if target_file.stat().st_mtime > src_notebook.stat().st_mtime: + return fmt_skip(src_notebook, f"target file ({target_file.name}) is newer ☑️") if dry_run: return colored(f"Would process {src_notebook.name}", "green") @@ -233,9 +197,17 @@ def process_notebook(src_notebook: Path, website_dir: Path, notebook_dir: Path, [quarto_bin, "render", src_notebook], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) if result.returncode != 0: - return colored(f"Failed to render {src_notebook}", "red") + f"\n{result.stderr}" + f"\n{result.stdout}" + return fmt_error( + src_notebook, f"Failed to render {src_notebook}\n\nstderr:\n{result.stderr}\nstdout:\n{result.stdout}" + ) - return colored(f"Processed {src_notebook.name}", "green") + front_matter = {} + if "front_matter" in metadata: + front_matter = metadata["front_matter"] + + post_process_mdx(target_file, front_matter) + + return fmt_ok(src_notebook) # Notebook execution based on nbmake: https://github.com/treebeardtech/nbmakes @@ -258,21 +230,14 @@ NB_VERSION = 4 def test_notebook(notebook_path: Path, timeout: int = 300) -> Tuple[Path, Optional[Union[NotebookError, NotebookSkip]]]: nb = nbformat.read(str(notebook_path), NB_VERSION) - allow_errors = False - if "execution" in nb.metadata: - if "timeout" in nb.metadata.execution: - timeout = nb.metadata.execution.timeout - if "allow_errors" in nb.metadata.execution: - allow_errors = nb.metadata.execution.allow_errors - - if "test_skip" in nb.metadata: - return notebook_path, NotebookSkip(reason=nb.metadata.test_skip) + if "skip_test" in nb.metadata: + return notebook_path, NotebookSkip(reason=nb.metadata.skip_test) try: c = NotebookClient( nb, timeout=timeout, - allow_errors=allow_errors, + allow_errors=False, record_timing=True, ) os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1" @@ -327,24 +292,19 @@ def get_error_info(nb: NotebookNode) -> Optional[NotebookError]: # rendered_notebook is the final mdx file -def post_process_mdx(rendered_mdx: Path) -> None: +def post_process_mdx(rendered_mdx: Path, front_matter: Dict) -> None: notebook_name = f"{rendered_mdx.stem}.ipynb" with open(rendered_mdx, "r", encoding="utf-8") as f: content = f.read() - # Check for existence of "export const quartoRawHtml", this indicates there was a front matter line in the file - if "export const quartoRawHtml" not in content: - raise ValueError(f"File {rendered_mdx} does not contain 'export const quartoRawHtml'") + # If there is front matter in the mdx file, we need to remove it + if content.startswith("---"): + front_matter_end = content.find("---", 3) + front_matter = yaml.safe_load(content[4:front_matter_end]) + content = content[front_matter_end + 3 :] - # Extract the text between - front_matter = content.split("")[0] - # Strip empty lines before and after - front_matter = "\n".join([line for line in front_matter.split("\n") if line.strip() != ""]) - - # add file path - front_matter += f"\nsource_notebook: /notebook/{notebook_name}" - # Custom edit url - front_matter += f"\ncustom_edit_url: https://github.com/microsoft/autogen/edit/main/notebook/{notebook_name}" + front_matter["source_notebook"] = f"/notebook/{notebook_name}" + front_matter["custom_edit_url"] = f"https://github.com/microsoft/autogen/edit/main/notebook/{notebook_name}" # inject in content directly after the markdown title the word done # Find the end of the line with the title @@ -352,8 +312,11 @@ def post_process_mdx(rendered_mdx: Path) -> None: # Extract page title title = content[content.find("#") + 1 : content.find("\n", content.find("#"))].strip() + # If there is a { in the title we trim off the { and everything after it + if "{" in title: + title = title[: title.find("{")].strip() - front_matter += f"\ntitle: {title}" + front_matter["title"] = title github_link = f"https://github.com/microsoft/autogen/blob/main/notebook/{notebook_name}" content = ( @@ -374,12 +337,15 @@ def post_process_mdx(rendered_mdx: Path) -> None: + content[title_end:] ) + # Dump front_matter to ysaml + front_matter = yaml.dump(front_matter, default_flow_style=False) + # Rewrite the content as # --- # front_matter # --- # content - new_content = f"---\n{front_matter}\n---\n{content}" + new_content = f"---\n{front_matter}---\n{content}" with open(rendered_mdx, "w", encoding="utf-8") as f: f.write(new_content) @@ -395,6 +361,18 @@ def collect_notebooks(notebook_directory: Path, website_directory: Path) -> typi return notebooks +def fmt_skip(notebook: Path, reason: str) -> None: + return f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {reason}" + + +def fmt_ok(notebook: Path) -> None: + return f"{colored('[OK]', 'green')} {colored(notebook.name, 'blue')} ✅" + + +def fmt_error(notebook: Path, error: NotebookError) -> None: + return f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {error.error_name} - {error.error_value}" + + def start_thread_to_terminate_when_parent_process_dies(ppid: int): pid = os.getpid() @@ -424,7 +402,6 @@ def main() -> None: parser.add_argument( "--website-directory", type=path, help="Root directory of docusarus website", default=script_dir ) - parser.add_argument("--workers", help="Number of workers to use", type=int, default=-1) render_parser = subparsers.add_parser("render") render_parser.add_argument("--quarto-bin", help="Path to quarto binary", default="quarto") @@ -435,10 +412,9 @@ def main() -> None: test_parser.add_argument("--timeout", help="Timeout for each notebook", type=int, default=60) test_parser.add_argument("--exit-on-first-fail", "-e", help="Exit after first test fail", action="store_true") test_parser.add_argument("notebooks", type=path, nargs="*", default=None) + test_parser.add_argument("--workers", help="Number of workers to use", type=int, default=-1) args = parser.parse_args() - if args.workers == -1: - args.workers = None if args.subcommand is None: print("No subcommand specified") @@ -453,13 +429,13 @@ def main() -> None: for notebook in collected_notebooks: reason = skip_reason_or_none_if_ok(notebook) if reason: - print(f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {reason}") + print(fmt_skip(notebook, reason)) else: filtered_notebooks.append(notebook) - print(f"Processing {len(filtered_notebooks)} notebook{'s' if len(filtered_notebooks) != 1 else ''}...") - if args.subcommand == "test": + if args.workers == -1: + args.workers = None failure = False with concurrent.futures.ProcessPoolExecutor( max_workers=args.workers, @@ -471,14 +447,15 @@ def main() -> None: notebook, optional_error_or_skip = future.result() if isinstance(optional_error_or_skip, NotebookError): if optional_error_or_skip.error_name == "timeout": - print( - f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.error_name}" - ) + print(fmt_error(notebook, optional_error_or_skip.error_name)) else: print("-" * 80) + print( - f"{colored('[Error]', 'red')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.error_name} - {optional_error_or_skip.error_value}" + fmt_error( + notebook, f"{optional_error_or_skip.error_name} - {optional_error_or_skip.error_value}" + ) ) print(optional_error_or_skip.traceback) print("-" * 80) @@ -486,11 +463,9 @@ def main() -> None: sys.exit(1) failure = True elif isinstance(optional_error_or_skip, NotebookSkip): - print( - f"{colored('[Skip]', 'yellow')} {colored(notebook.name, 'blue')}: {optional_error_or_skip.reason}" - ) + print(fmt_skip(notebook, optional_error_or_skip.reason)) else: - print(f"{colored('[OK]', 'green')} {colored(notebook.name, 'blue')}") + print(fmt_ok(notebook)) if failure: sys.exit(1) @@ -501,15 +476,12 @@ def main() -> None: if not notebooks_target_dir(args.website_directory).exists(): notebooks_target_dir(args.website_directory).mkdir(parents=True) - with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as executor: - futures = [ - executor.submit( - process_notebook, f, args.website_directory, args.notebook_directory, args.quarto_bin, args.dry_run + for notebook in filtered_notebooks: + print( + process_notebook( + notebook, args.website_directory, args.notebook_directory, args.quarto_bin, args.dry_run ) - for f in filtered_notebooks - ] - for future in concurrent.futures.as_completed(futures): - print(future.result()) + ) else: print("Unknown subcommand") sys.exit(1)