Move docs from python/packages/autogen-core to python/docs (#6757)

.gitattributes

@@ -91,5 +91,5 @@ python/packages/autogen-ext/src/autogen_ext/runtimes/grpc/protos/*.pyi linguist-
 python/packages/autogen-ext/tests/protos/*.py linguist-generated
 python/packages/autogen-ext/tests/protos/*.pyi linguist-generated
 docs/** linguist-documentation
-python/packages/autogen-core/docs/** linguist-documentation
+python/docs/** linguist-documentation
 dotnet/website/** linguist-documentation

.github/workflows/checks.yml

@@ -75,6 +75,24 @@ jobs:
           poe --directory ${{ matrix.package }} mypy
         working-directory: ./python
 
+  docs-mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: uv sync --locked --all-extras
+        working-directory: ./python
+      - name: Run task
+        run: |
+          source ${{ github.workspace }}/python/.venv/bin/activate
+          poe docs-mypy
+        working-directory: ./python
+
   pyright:
     runs-on: ubuntu-latest
     strategy:
@@ -244,9 +262,6 @@ jobs:
 
   docs:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        package: ["./packages/autogen-core"]
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5
@@ -260,14 +275,11 @@ jobs:
       - name: Run task
         run: |
           source ${{ github.workspace }}/python/.venv/bin/activate
-          poe --directory ${{ matrix.package }} docs-check
+          poe docs-check
         working-directory: ./python
 
   docs-example-check:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        package: ["./packages/autogen-core"]
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5
@@ -281,7 +293,7 @@ jobs:
       - name: Run task
         run: |
           source ${{ github.workspace }}/python/.venv/bin/activate
-          poe --directory ${{ matrix.package }} docs-check-examples
+          poe docs-check-examples
         working-directory: ./python
 
   samples-code-check:

.github/workflows/docs.yml

@@ -31,28 +31,153 @@ jobs:
       matrix:
         version:
           [
-            # For main use the workflow target
-            { ref: "${{github.ref}}", dest-dir: dev, uv-version: "0.7.13", sphinx-release-override: "dev" },
-            { ref: "python-v0.6.2", dest-dir: stable, uv-version: "0.7.13", sphinx-release-override: "stable" },
-            { ref: "v0.4.0.post1", dest-dir: "0.4.0", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "v0.4.1", dest-dir: "0.4.1", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "v0.4.2", dest-dir: "0.4.2", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "v0.4.3", dest-dir: "0.4.3", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "v0.4.4", dest-dir: "0.4.4", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.4.5", dest-dir: "0.4.5", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.4.6", dest-dir: "0.4.6", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.4.7", dest-dir: "0.4.7", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.4.8", dest-dir: "0.4.8", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.4.9-website", dest-dir: "0.4.9", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.1", dest-dir: "0.5.1", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.2", dest-dir: "0.5.2", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.3", dest-dir: "0.5.3", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.4", dest-dir: "0.5.4", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.5", dest-dir: "0.5.5", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.6", dest-dir: "0.5.6", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.5.7", dest-dir: "0.5.7", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.6.1", dest-dir: "0.6.1", uv-version: "0.5.13", sphinx-release-override: "" },
-            { ref: "python-v0.6.2", dest-dir: "0.6.2", uv-version: "0.7.13", sphinx-release-override: "" },
+            {
+              ref: "${{github.ref}}",
+              dest-dir: dev,
+              uv-version: "0.7.13",
+              sphinx-release-override: "dev",
+              poe-dir: ".",
+            },
+            {
+              ref: "python-v0.6.2",
+              dest-dir: stable,
+              uv-version: "0.7.13",
+              sphinx-release-override: "stable",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "v0.4.0.post1",
+              dest-dir: "0.4.0",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "v0.4.1",
+              dest-dir: "0.4.1",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "v0.4.2",
+              dest-dir: "0.4.2",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "v0.4.3",
+              dest-dir: "0.4.3",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "v0.4.4",
+              dest-dir: "0.4.4",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.4.5",
+              dest-dir: "0.4.5",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.4.6",
+              dest-dir: "0.4.6",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.4.7",
+              dest-dir: "0.4.7",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.4.8",
+              dest-dir: "0.4.8",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.4.9-website",
+              dest-dir: "0.4.9",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.1",
+              dest-dir: "0.5.1",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.2",
+              dest-dir: "0.5.2",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.3",
+              dest-dir: "0.5.3",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.4",
+              dest-dir: "0.5.4",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.5",
+              dest-dir: "0.5.5",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.6",
+              dest-dir: "0.5.6",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.5.7",
+              dest-dir: "0.5.7",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.6.1",
+              dest-dir: "0.6.1",
+              uv-version: "0.5.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
+            {
+              ref: "python-v0.6.2",
+              dest-dir: "0.6.2",
+              uv-version: "0.7.13",
+              sphinx-release-override: "",
+              poe-dir: "./packages/autogen-core",
+            },
           ]
     steps:
       - name: Checkout
@@ -63,16 +188,16 @@ jobs:
       - uses: astral-sh/setup-uv@v5
         with:
           enable-cache: true
-          version: ${{ matrix.version.uv-version }}    
+          version: ${{ matrix.version.uv-version }}
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - run: |
           uv sync --locked --all-extras
           source .venv/bin/activate
-          poe --directory ./packages/autogen-core docs-build
+          poe --directory ${{ matrix.version.poe-dir }} docs-build
           mkdir -p docs-staging/${{ matrix.version.dest-dir }}/
-          mv ./packages/autogen-core/docs/build/* docs-staging/${{ matrix.version.dest-dir }}/
+          mv ${{ matrix.version.poe-dir }}/docs/build/* docs-staging/${{ matrix.version.dest-dir }}/
         working-directory: ./python
         env:
           PY_DOCS_DIR: ${{ matrix.version.dest-dir }}/
@@ -96,7 +221,7 @@ jobs:
       - name: generate redirects
         run: |
           mkdir -p python/docs-staging/
-          python python/packages/autogen-core/docs/redirects/redirects.py python/docs-staging
+          python python/docs/redirects/redirects.py python/docs-staging
       - uses: actions/upload-artifact@v4
         with:
           path: "./python/docs-staging"
@@ -241,7 +366,8 @@ jobs:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
-    needs: [build-02, build-04, build-04-dotnet, gen-redirects, gen-component-schema]
+    needs:
+      [build-02, build-04, build-04-dotnet, gen-redirects, gen-component-schema]
     if: ${{ needs.build-02.result == 'success' && needs.build-04.result == 'success' && needs.gen-redirects.result == 'success' && github.ref == 'refs/heads/main' }}
     steps:
       - uses: actions/download-artifact@v4

python/.gitignore

@@ -174,3 +174,6 @@ tmp_code_*.py
 
 # .NET Development settings
 appsettings.Development.json
+
+# Documentation reference files
+docs/src/reference

python/README.md

@@ -1,15 +1,21 @@
-# AutoGen Python packages
+# AutoGen Python Development Guide
 
-[![0.4 Docs](https://img.shields.io/badge/Docs-0.4-blue)](https://microsoft.github.io/autogen/dev/)
+[![Docs (dev)](https://img.shields.io/badge/Docs-dev-blue)](https://microsoft.github.io/autogen/dev/)
+[![Docs (latest release)](https://img.shields.io/badge/Docs-latest%20release-blue)](https://microsoft.github.io/autogen/dev/)
 [![PyPi autogen-core](https://img.shields.io/badge/PyPi-autogen--core-blue?logo=pypi)](https://pypi.org/project/autogen-core/) [![PyPi autogen-agentchat](https://img.shields.io/badge/PyPi-autogen--agentchat-blue?logo=pypi)](https://pypi.org/project/autogen-agentchat/) [![PyPi autogen-ext](https://img.shields.io/badge/PyPi-autogen--ext-blue?logo=pypi)](https://pypi.org/project/autogen-ext/)
 
-This directory works as a single `uv` workspace containing all project packages. See [`packages`](./packages/) to discover all project packages.
+This directory works as a single `uv` workspace containing all project packages, including:
+
+- `packages/autogen-core`: interface definitions and reference implementations of agent runtime, model, tool, workbench, memory, tracing.
+- `packages/autogen-agentchat`: single and multi-agent workflows built on top of `autogen-core`.
+- `packages/autogen-ext`: implementations for ecosystem integrations. For example, `autogen-ext[openai]` provides the OpenAI model client.
+- `packages/autogen-studio`: a web-based IDE for building and running AutoGen agents.
 
 ## Migrating from 0.2.x?
 
 Please refer to the [migration guide](./migration_guide.md) for how to migrate your code from 0.2.x to 0.4.x.
 
-## Development
+## Quick Start
 
 **TL;DR**, run all checks with:
 
@@ -19,7 +25,7 @@ source .venv/bin/activate
 poe check
 ```
 
-### Setup
+## Setup
 
 `uv` is a package manager that assists in creating the necessary environment and installing packages to run AutoGen.
 
@@ -31,14 +37,7 @@ To upgrade `uv` to the latest version, run:
 uv self update
 ```
 
-<!-- **Note:** To prevent incompatibilities between versions the same UV version as is running in CI should be used. Check the version in CI by looking the `setup-uv` action, [here](https://github.com/microsoft/autogen/blob/main/.github/workflows/checks.yml#L40) for example.
-
-For example, to change your version to `0.5.18`, run:
-```sh
-uv self update 0.5.18
-``` -->
-
-### Virtual Environment
+## Virtual Environment
 
 During development, you may need to test changes made to any of the packages.\
 To do so, create a virtual environment where the AutoGen packages are installed based on the current state of the directory.\
@@ -52,7 +51,7 @@ source .venv/bin/activate
 - `uv sync --all-extras` will create a `.venv` directory at the current level and install packages from the current directory along with any other dependencies. The `all-extras` flag adds optional dependencies.
 - `source .venv/bin/activate` activates the virtual environment.
 
-### Common Tasks
+## Common Tasks
 
 To create a pull request (PR), ensure the following checks are met. You can run each check individually:
 
@@ -61,16 +60,19 @@ To create a pull request (PR), ensure the following checks are met. You can run
 - Test: `poe test`
 - Mypy: `poe mypy`
 - Pyright: `poe pyright`
-- Build docs: `poe --directory ./packages/autogen-core/ docs-build`
-- Auto rebuild+serve docs: `poe --directory ./packages/autogen-core/ docs-serve`
+- Build docs: `poe docs-build`
+- Check docs: `poe docs-check`
+- Clean docs: `poe docs-clean`
+- Check code blocks in API references: `poe docs-check-examples`
+- Auto rebuild+serve docs: `poe docs-serve`
 - Check samples in `python/samples`: `poe samples-code-check`
-Alternatively, you can run all the checks with:
+  Alternatively, you can run all the checks with:
 - `poe check`
 
 > [!NOTE]
 > These need to be run in the virtual environment.
 
-### Syncing Dependencies
+## Syncing Dependencies
 
 When you pull new changes, you may need to update the dependencies.
 To do so, first make sure you are in the virtual environment, and then in the `python` directory, run:
@@ -81,7 +83,135 @@ uv sync --all-extras
 
 This will update the dependencies in the virtual environment.
 
-### Creating a New Package
+## Building Documentation
+
+The documentation source directory is located at `docs/src/`.
+
+To build the documentation, run this from the root of the Python directory:
+
+```sh
+poe docs-build
+```
+
+To serve the documentation locally, run:
+
+```sh
+poe docs-serve
+```
+
+When you make changes to the doc strings or add new modules, you may need to
+refresh the API references in the documentation by first cleaning the docs and
+then building them again:
+
+```sh
+poe docs-clean # This will remove the build directory and the reference directory
+poe docs-build # This will rebuild the documentation from scratch
+```
+
+## Writing Documentation
+
+When you add a new public class or function, you should always add a docstring
+to it. The docstring should follow the
+[Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) layout
+and the Sphinx RST format for Python docstrings.
+
+The docstring for a public class or function should include:
+
+- A short description of the class or function at the beginning immediately after the `"""`.
+- A longer description if necessary, explaining the purpose and usage.
+- A list of arguments with their types and descriptions, using the `Args` section.
+  Each argument should be listed with its name, type, and a brief description.
+- A description of the return value and its type, using the `Returns` section.
+  If the function does not return anything, you can omit this section.
+- A list of exceptions that the function may raise, with descriptions,
+  using the `Raises` section. This is optional but recommended if the function can raise exceptions that users should be aware of.
+- Examples of how to use the class or function, using the `Examples` section,
+  and formatted using `.. code-block:: python` directive. Optionally, also include the output of the example using
+  `.. code-block:: text` directive.
+
+Here is an example of a docstring for `McpWorkbench` class:
+
+```python
+class McpWorkbench(Workbench, Component[McpWorkbenchConfig]):
+    """A workbench that wraps an MCP server and provides an interface
+    to list and call tools provided by the server.
+
+    This workbench should be used as a context manager to ensure proper
+    initialization and cleanup of the underlying MCP session.
+
+    Args:
+        server_params (McpServerParams): The parameters to connect to the MCP server.
+            This can be either a :class:`StdioServerParams` or :class:`SseServerParams`.
+        tool_overrides (Optional[Dict[str, ToolOverride]]): Optional mapping of original tool
+            names to override configurations for name and/or description. This allows
+            customizing how server tools appear to consumers while maintaining the underlying
+            tool functionality.
+
+    Raises:
+        ValueError: If there are conflicts in tool override names.
+
+    Examples:
+
+        Here is a simple example of how to use the workbench with a `mcp-server-fetch` server:
+
+        .. code-block:: python
+
+            import asyncio
+
+            from autogen_ext.tools.mcp import McpWorkbench, StdioServerParams
+
+
+            async def main() -> None:
+                params = StdioServerParams(
+                    command="uvx",
+                    args=["mcp-server-fetch"],
+                    read_timeout_seconds=60,
+                )
+
+                # You can also use `start()` and `stop()` to manage the session.
+                async with McpWorkbench(server_params=params) as workbench:
+                    tools = await workbench.list_tools()
+                    print(tools)
+                    result = await workbench.call_tool(tools[0]["name"], {"url": "https://github.com/"})
+                    print(result)
+
+
+            asyncio.run(main())
+```
+
+The code blocks with `.. code-block:: python` is checked by the `docs-check-examples` task using Pyright,
+so make sure the code is valid. Running the code as a script and checking it using `pyright`
+is a good way to ensure the code examples are correct.
+
+When you reference a class, method, or function in the docstring, you should always
+use the `:class:`, `:meth:`, or `:func:` directive to create a link to the class or function.
+Always use the fully qualified name of the class or function, including the package name, but
+prefix it with a `~` for shorter rendering in the documentation.
+For example, if you are referencing the `AssistantAgent` class in the `autogen-agentchat` package,
+you should write it as `:class:~autogen_agentchat.AssistantAgent`.
+
+For a public data class, including those that are Pydantic models, you should also include docstrings
+for each field in the class.
+
+## Writing Tests
+
+When you add a new public class or function, you should also always add tests for it.
+We track test coverage and aim for not reducing the coverage percentage with new changes.
+
+We use `pytest` for testing, and you should always use fixtures to set up the test dependencies.
+
+Use mock objects to simulate dependencies and avoid making real API calls or database queries in tests.
+See existing tests for examples of how to use fixtures and mocks.
+
+For model clients, use `autogen_ext.models.replay.ReplayChatCompletionClient` as a
+drop-in replacement for the model client to simulate responses without making real API calls.
+
+When certain tests requires interaction with actual model APIs or other external services,
+you should configure the tests to be skipped if the required services are not available.
+For example, if you are testing a model client that requires an OpenAI API key,
+you can use the `pytest.mark.skipif` decorator to skip the test if the environment variable for the API key is not set.
+
+## Creating a New Package
 
 To create a new package, similar to `autogen-core` or `autogen-chat`, use the following:
 

python/docs/README.md

@@ -0,0 +1,29 @@
+## Building the AutoGen Documentation
+
+AutoGen documentation is based on the sphinx documentation system and uses the myst-parser to render markdown files. It uses the [pydata-sphinx-theme](https://pydata-sphinx-theme.readthedocs.io/en/latest/) to style the documentation.
+
+### Prerequisites
+
+Ensure you have all of the dev dependencies for the `autogen-core` package installed. You can install them by running the following command from the root of the python repository:
+
+```bash
+uv sync
+source .venv/bin/activate
+```
+
+## Building Docs
+
+To build the documentation, run the following command from the root of the python directory:
+
+```bash
+poe docs-build
+```
+
+To serve the documentation locally, run the following command from the root of the python directory:
+
+```bash
+poe docs-serve
+```
+
+[!NOTE]
+Sphinx will only rebuild files that have changed since the last build. If you want to force a full rebuild, you can delete the `./docs/build` directory before running the `docs-build` command.

python/docs/src/conf.py

@@ -150,7 +150,7 @@ html_context = {
     "github_user": "microsoft",
     "github_repo": "autogen",
     "github_version": "main",
-    "doc_path": "python/packages/autogen-core/docs/src/",
+    "doc_path": "python/docs/src/",
 }
 
 autodoc_default_options = {
@@ -183,9 +183,16 @@ rediraffe_redirects = {
 }
 
 
-def generate_api_docs() -> None:
+def generate_api_reference() -> None:
     """Generate API documentation before building."""
-    script_path = Path(__file__).parent / "generate_api_toc.py"
+    reference_dir = Path(__file__).parent / "reference"
+    
+    # Only generate if reference directory doesn't exist
+    if reference_dir.exists():
+        print("📁 Reference directory already exists, skipping API generation")
+        return
+    
+    script_path = Path(__file__).parent / "generate_api_reference.py"
     if script_path.exists():
         print("🔄 Generating API documentation...")
         try:
@@ -242,7 +249,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
         the 2 parallel parameters set to ``True``.
     """
     # Generate API documentation before building
-    app.connect("builder-inited", lambda app: generate_api_docs())
+    app.connect("builder-inited", lambda app: generate_api_reference())
     
     app.connect("html-page-context", setup_to_main)
 

python/docs/src/generate_api_reference.py

@@ -47,7 +47,7 @@ def is_private_module(module_parts: List[str]) -> bool:
 
 def find_python_packages() -> List[Path]:
     """Find documented Python packages in the workspace."""
-    packages_dir = Path(__file__).parent.parent.parent.parent.parent / "packages"
+    packages_dir = Path(__file__).parent.parent.parent / "packages"
     python_packages = []
     
     for package_dir in packages_dir.iterdir():