Fixup ruff config and inclusions (#495)

* add tests to ruff for core

* fmt

* lint

* lint fixes

* fixup more dirs

* dont include non python

* lint fixes

* lint fixes

* fix dir name

* dont relative include

python/packages/team-one/pyproject.toml

@@ -68,9 +68,10 @@ exclude = ["build", "dist", "page_script.js"]
 include = [
     "src/**",
     "examples/*.py",
+    "tests/**/*.py",
 ]
 
 
 [tool.pyright]
 extend = "../../pyproject.toml"
-include = ["src", "tests"]
+include = ["src", "tests", "examples"]

python/packages/team-one/tests/browser_utils/test_bing_markdown_search.py

@@ -2,8 +2,8 @@
 import os
 
 import pytest
-
 from team_one.markdown_browser import BingMarkdownSearch
+
 skip_all = False
 
 bing_api_key = None

python/packages/team-one/tests/browser_utils/test_mdconvert.py

@@ -5,8 +5,8 @@ import shutil
 
 import pytest
 import requests
-
 from team_one.markdown_browser import MarkdownConverter
+
 skip_all = False
 
 skip_exiftool = shutil.which("exiftool") is None
@@ -119,30 +119,30 @@ def test_mdconvert_local() -> None:
     # Test XLSX processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test.xlsx"))
     for test_string in XLSX_TEST_STRINGS:
-        text_content = result.text_content.replace('\\','')
+        text_content = result.text_content.replace("\\", "")
         assert test_string in text_content
 
     # Test DOCX processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test.docx"))
     for test_string in DOCX_TEST_STRINGS:
-        text_content = result.text_content.replace('\\','')
+        text_content = result.text_content.replace("\\", "")
         assert test_string in text_content
 
     # Test PPTX processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))
     for test_string in PPTX_TEST_STRINGS:
-        text_content = result.text_content.replace('\\','')
+        text_content = result.text_content.replace("\\", "")
         assert test_string in text_content
 
     # Test HTML processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test_blog.html"), url=BLOG_TEST_URL)
     for test_string in BLOG_TEST_STRINGS:
-        text_content = result.text_content.replace('\\','')
+        text_content = result.text_content.replace("\\", "")
         assert test_string in text_content
 
     # Test Wikipedia processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test_wikipedia.html"), url=WIKIPEDIA_TEST_URL)
-    text_content = result.text_content.replace('\\','')
+    text_content = result.text_content.replace("\\", "")
     for test_string in WIKIPEDIA_TEST_EXCLUDES:
         assert test_string not in text_content
     for test_string in WIKIPEDIA_TEST_STRINGS:
@@ -150,7 +150,7 @@ def test_mdconvert_local() -> None:
 
     # Test Bing processing
     result = mdconvert.convert(os.path.join(TEST_FILES_DIR, "test_serp.html"), url=SERP_TEST_URL)
-    text_content = result.text_content.replace('\\','')
+    text_content = result.text_content.replace("\\", "")
     for test_string in SERP_TEST_EXCLUDES:
         assert test_string not in text_content
     for test_string in SERP_TEST_STRINGS:

python/packages/team-one/tests/browser_utils/test_requests_markdown_browser.py

@@ -8,6 +8,7 @@ import re
 
 import pytest
 import requests
+from team_one.markdown_browser import BingMarkdownSearch, RequestsMarkdownBrowser
 
 BLOG_POST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
 BLOG_POST_TITLE = "Does Model and Inference Parameter Matter in LLM Applications? - A Case Study for MATH | AutoGen"
@@ -37,7 +38,7 @@ LOCAL_FILE_TEST_STRINGS = [
     BLOG_POST_FIND_ON_PAGE_MATCH,
 ]
 
-from team_one.markdown_browser import BingMarkdownSearch, RequestsMarkdownBrowser
+
 skip_all = False
 
 
@@ -49,10 +50,12 @@ def _rm_folder(path: str) -> None:
             os.unlink(fpath)
     os.rmdir(path)
 
+
 def normalize_text(text: str) -> str:
     text = "\n".join([line.rstrip() for line in re.split(r"\r?\n", text)])
     return re.sub(r"\n{3,}", "\n\n", text)
 
+
 @pytest.mark.skipif(
     skip_all,
     reason="do not run if dependency is not installed",
@@ -77,7 +80,7 @@ def test_requests_markdown_browser() -> None:
     assert browser.viewport == top_viewport
     assert browser.page_title is not None
     assert browser.page_title.strip() == BLOG_POST_TITLE.strip()
-    page_content = browser.page_content.replace('\\','')
+    page_content = browser.page_content.replace("\\", "")
     assert BLOG_POST_STRING in page_content
 
     # Check if page splitting works
@@ -98,7 +101,7 @@ def test_requests_markdown_browser() -> None:
         browser.page_down()
         assert browser.viewport_current_page == i
     # Test scrolloing beyond the limits
-    for i in range(0, 5):
+    for _ in range(0, 5):
         browser.page_down()
         assert browser.viewport_current_page == len(browser.viewport_pages) - 1
 
@@ -107,7 +110,7 @@ def test_requests_markdown_browser() -> None:
         browser.page_up()
         assert browser.viewport_current_page == i
     # Test scrolloing beyond the limits
-    for i in range(0, 5):
+    for _ in range(0, 5):
         browser.page_up()
         assert browser.viewport_current_page == 0
 
@@ -118,12 +121,12 @@ def test_requests_markdown_browser() -> None:
     # Visit a plain-text file
     response = requests.get(PLAIN_TEXT_URL)
     response.raise_for_status()
-    expected_results = re.sub(r"\s+", " ", response.text, re.DOTALL).strip()
+    expected_results = re.sub(r"\s+", " ", string=response.text, flags=re.DOTALL).strip()
     # Run the normalize code that the markdown request module uses
     expected_results = normalize_text(expected_results)
 
     browser.visit_page(PLAIN_TEXT_URL)
-    assert re.sub(r"\s+", " ", browser.page_content, re.DOTALL).strip() == expected_results
+    assert re.sub(r"\s+", " ", string=browser.page_content, flags=re.DOTALL).strip() == expected_results
 
     # Disrectly download a ZIP file and compute its md5
     response = requests.get(DOWNLOAD_URL, stream=True)
@@ -157,7 +160,7 @@ def test_requests_markdown_browser() -> None:
     assert find_viewport is not None
 
     # Find next using the same query
-    for i in range(0, 10):
+    for _ in range(0, 10):
         find_viewport = browser.find_on_page("LLM app*")
         assert find_viewport is not None
 
@@ -166,7 +169,7 @@ def test_requests_markdown_browser() -> None:
         loc = new_loc
 
     # Find next using find_next
-    for i in range(0, 10):
+    for _ in range(0, 10):
         find_viewport = browser.find_next()
         assert find_viewport is not None
 
@@ -207,25 +210,25 @@ def test_local_file_browsing() -> None:
     # Directory listing via open_local_file
     viewport = browser.open_local_file(directory)
     for target_string in DIR_TEST_STRINGS:
-        viewport = viewport.replace('\\','')
+        viewport = viewport.replace("\\", "")
         assert target_string in viewport
 
     # Directory listing via file URI
     viewport = browser.visit_page(pathlib.Path(os.path.abspath(directory)).as_uri())
     for target_string in DIR_TEST_STRINGS:
-        viewport = viewport.replace('\\','')
+        viewport = viewport.replace("\\", "")
         assert target_string in viewport
 
     # File access via file open_local_file
     browser.open_local_file(test_file)
     for target_string in LOCAL_FILE_TEST_STRINGS:
-        page_content = browser.page_content.replace('\\','')
+        page_content = browser.page_content.replace("\\", "")
         assert target_string in page_content
 
     # File access via file URI
     browser.visit_page(pathlib.Path(os.path.abspath(test_file)).as_uri())
     for target_string in LOCAL_FILE_TEST_STRINGS:
-        page_content = browser.page_content.replace('\\','')
+        page_content = browser.page_content.replace("\\", "")
         assert target_string in page_content
 
 

python/packages/team-one/tests/headless_web_surfer/conftest.py

@@ -8,4 +8,4 @@ MOCK_CHAT_COMPLETION_KWARGS: str = """
   "api_key": "sk-mockopenaiAPIkeyinexpectedformatfortestingonly",
   "model": "gpt-4o-2024-05-13"
 }
-"""
+"""

python/packages/team-one/tests/headless_web_surfer/test_web_surfer.py

@@ -1,45 +1,21 @@
 #!/usr/bin/env python3 -m pytest
 
+import asyncio
 import os
 import re
-import sys
-from math import ceil
-
-import asyncio
-import pytest
-
-from autogen_core.base import AgentId
-from autogen_core.base import AgentProxy
-pytest_plugins = ('pytest_asyncio',)
 from json import dumps
+from math import ceil
+from typing import Mapping
 
-from team_one.utils import (
-  ENVIRON_KEY_CHAT_COMPLETION_PROVIDER,
-  ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON,
-  create_completion_client_from_env
-)
-from team_one.agents.user_proxy import UserProxy
-from team_one.agents.orchestrator import RoundRobinOrchestrator
-from team_one.messages import BroadcastMessage
+import pytest
 from autogen_core.application import SingleThreadedAgentRuntime
+from autogen_core.base import AgentId, AgentProxy
 from autogen_core.components import FunctionCall
 from autogen_core.components.models import (
     UserMessage,
 )
 from autogen_core.components.tools._base import ToolSchema
-
 from openai import AuthenticationError
-
-sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
-from conftest import MOCK_CHAT_COMPLETION_KWARGS, reason
-
-sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
-#from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST  # noqa: E402
-
-BLOG_POST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
-BLOG_POST_TITLE = "Does Model and Inference Parameter Matter in LLM Applications? - A Case Study for MATH | AutoGen"
-BING_QUERY = "Microsoft"
-
 from team_one.agents.multimodal_web_surfer import MultimodalWebSurfer
 from team_one.agents.multimodal_web_surfer.tool_definitions import (
     TOOL_PAGE_DOWN,
@@ -49,18 +25,37 @@ from team_one.agents.multimodal_web_surfer.tool_definitions import (
     TOOL_VISIT_URL,
     TOOL_WEB_SEARCH,
 )
+from team_one.agents.orchestrator import RoundRobinOrchestrator
+from team_one.agents.user_proxy import UserProxy
+from team_one.messages import BroadcastMessage
+from team_one.utils import (
+    ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON,
+    ENVIRON_KEY_CHAT_COMPLETION_PROVIDER,
+    create_completion_client_from_env,
+)
+
+from conftest import MOCK_CHAT_COMPLETION_KWARGS, reason
+
+pytest_plugins = ("pytest_asyncio",)
+
+
+BLOG_POST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
+BLOG_POST_TITLE = "Does Model and Inference Parameter Matter in LLM Applications? - A Case Study for MATH | AutoGen"
+BING_QUERY = "Microsoft"
+
+
 skip_all = False
 
-#except ImportError:
+# except ImportError:
 #    skip_all = True
-#else:
+# else:
 #    skip_all = False
 
-#try:
+# try:
 #    BING_API_KEY = os.environ["BING_API_KEY"]
-#except KeyError:
+# except KeyError:
 #    skip_bing = True
-#else:
+# else:
 #    skip_bing = False
 # Search currently does not require an API key
 skip_bing = False
@@ -70,16 +65,19 @@ if os.getenv(ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON):
 else:
     skip_openai = True
 
-def generate_tool_request(tool: ToolSchema, args: dict[str, str]) -> list[FunctionCall]:
-    ret = [FunctionCall(id='', arguments='', name=tool["name"])]
+
+def generate_tool_request(tool: ToolSchema, args: Mapping[str, str]) -> list[FunctionCall]:
+    ret = [FunctionCall(id="", arguments="", name=tool["name"])]
     ret[0].arguments = dumps(args)
     return ret
 
-async def make_browser_request(browser: MultimodalWebSurfer, tool: ToolSchema, args: dict[str, str]={}) -> str:
-    rects = await browser._get_interactive_rects() # type: ignore
+
+async def make_browser_request(browser: MultimodalWebSurfer, tool: ToolSchema, args: Mapping[str, str] = {}) -> str:
+    rects = await browser._get_interactive_rects()  # type: ignore
 
     req = generate_tool_request(tool, args)
-    return str((await browser._execute_tool(req, rects, "", use_ocr=False))[1][0]) # type: ignore
+    return str((await browser._execute_tool(req, rects, "", use_ocr=False))[1][0])  # type: ignore
+
 
 @pytest.mark.skipif(
     skip_all,
@@ -89,7 +87,7 @@ async def make_browser_request(browser: MultimodalWebSurfer, tool: ToolSchema, a
 async def test_web_surfer() -> None:
     env = {
         ENVIRON_KEY_CHAT_COMPLETION_PROVIDER: "openai",
-        ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON: MOCK_CHAT_COMPLETION_KWARGS
+        ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON: MOCK_CHAT_COMPLETION_KWARGS,
     }
 
     runtime = SingleThreadedAgentRuntime()
@@ -111,7 +109,7 @@ async def test_web_surfer() -> None:
 
     # Test some basic navigations
     tool_resp = await make_browser_request(actual_surfer, TOOL_VISIT_URL, {"url": BLOG_POST_URL})
-    metadata = await actual_surfer._get_page_metadata() # type: ignore
+    metadata = await actual_surfer._get_page_metadata()  # type: ignore
     assert f"{BLOG_POST_URL}".strip() in metadata["meta_tags"]["og:url"]
     assert f"{BLOG_POST_TITLE}".strip() in metadata["meta_tags"]["og:title"]
 
@@ -122,36 +120,43 @@ async def test_web_surfer() -> None:
 
     tool_resp = await make_browser_request(actual_surfer, TOOL_PAGE_DOWN)
     assert (
-        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned {viewport_percentage}% down from the top of the page." in tool_resp
+        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned {viewport_percentage}% down from the top of the page."
+        in tool_resp
     )  # Assumes the content is longer than one screen
 
     tool_resp = await make_browser_request(actual_surfer, TOOL_PAGE_UP)
     assert (
-        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the top of the page" in tool_resp
+        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the top of the page"
+        in tool_resp
     )  # Assumes the content is longer than one screen
 
-#        # Try to scroll too far back up
+    #        # Try to scroll too far back up
     tool_resp = await make_browser_request(actual_surfer, TOOL_PAGE_UP)
     assert (
-        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the top of the page" in tool_resp
+        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the top of the page"
+        in tool_resp
     )
 
     # Try to scroll too far down
-    total_pages = ceil(100/viewport_percentage)
+    total_pages = ceil(100 / viewport_percentage)
     for _ in range(0, total_pages + 1):
         tool_resp = await make_browser_request(actual_surfer, TOOL_PAGE_DOWN)
     assert (
-        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the bottom of the page" in tool_resp
+        f"The viewport shows {viewport_percentage}% of the webpage, and is positioned at the bottom of the page"
+        in tool_resp
     )
 
     # Test Q&A and summarization -- we don't have a key so we expect it to fail #(but it means the code path is correct)
     with pytest.raises(AuthenticationError):
-        tool_resp = await make_browser_request(actual_surfer, TOOL_READ_PAGE_AND_ANSWER, {"question": "When was it founded?"})
+        tool_resp = await make_browser_request(
+            actual_surfer, TOOL_READ_PAGE_AND_ANSWER, {"question": "When was it founded?"}
+        )
 
     with pytest.raises(AuthenticationError):
         tool_resp = await make_browser_request(actual_surfer, TOOL_SUMMARIZE_PAGE)
     await runtime.stop_when_idle()
 
+
 @pytest.mark.skipif(
     skip_all or skip_openai,
     reason="dependency is not installed OR" + reason,
@@ -182,32 +187,37 @@ async def test_web_surfer_oai() -> None:
     await actual_surfer.init(model_client=client, downloads_folder=os.getcwd(), browser_channel="chromium")
 
     await runtime.send_message(
-        BroadcastMessage(content=UserMessage(content="Please visit the page 'https://en.wikipedia.org/wiki/Microsoft'", source="user")),
+        BroadcastMessage(
+            content=UserMessage(
+                content="Please visit the page 'https://en.wikipedia.org/wiki/Microsoft'", source="user"
+            )
+        ),
         recipient=web_surfer.id,
-        sender=user_proxy.id
+        sender=user_proxy.id,
     )
     await runtime.send_message(
         BroadcastMessage(content=UserMessage(content="Please scroll down.", source="user")),
         recipient=web_surfer.id,
-        sender=user_proxy.id
+        sender=user_proxy.id,
     )
     await runtime.send_message(
         BroadcastMessage(content=UserMessage(content="Please scroll up.", source="user")),
         recipient=web_surfer.id,
-        sender=user_proxy.id
+        sender=user_proxy.id,
     )
     await runtime.send_message(
         BroadcastMessage(content=UserMessage(content="When was it founded?", source="user")),
         recipient=web_surfer.id,
-        sender=user_proxy.id
+        sender=user_proxy.id,
     )
     await runtime.send_message(
         BroadcastMessage(content=UserMessage(content="What's this page about?", source="user")),
         recipient=web_surfer.id,
-        sender=user_proxy.id
+        sender=user_proxy.id,
     )
     await runtime.stop_when_idle()
 
+
 @pytest.mark.skipif(
     skip_bing,
     reason="do not run if bing api key is not available",
@@ -216,7 +226,7 @@ async def test_web_surfer_oai() -> None:
 async def test_web_surfer_bing() -> None:
     env = {
         ENVIRON_KEY_CHAT_COMPLETION_PROVIDER: "openai",
-        ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON: MOCK_CHAT_COMPLETION_KWARGS
+        ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON: MOCK_CHAT_COMPLETION_KWARGS,
     }
 
     runtime = SingleThreadedAgentRuntime()
@@ -239,16 +249,17 @@ async def test_web_surfer_bing() -> None:
     # Test some basic navigations
     tool_resp = await make_browser_request(actual_surfer, TOOL_WEB_SEARCH, {"query": BING_QUERY})
 
-    metadata = await actual_surfer._get_page_metadata() # type: ignore
+    metadata = await actual_surfer._get_page_metadata()  # type: ignore
     assert f"{BING_QUERY}".strip() in metadata["meta_tags"]["og:url"]
     assert f"{BING_QUERY}".strip() in metadata["meta_tags"]["og:title"]
-    assert f"I typed '{BING_QUERY}' into the browser search bar." in tool_resp.replace("\\","")
+    assert f"I typed '{BING_QUERY}' into the browser search bar." in tool_resp.replace("\\", "")
 
     tool_resp = await make_browser_request(actual_surfer, TOOL_WEB_SEARCH, {"query": BING_QUERY + " Wikipedia"})
-    markdown = await actual_surfer._get_page_markdown() # type: ignore
+    markdown = await actual_surfer._get_page_markdown()  # type: ignore
     assert "https://en.wikipedia.org/wiki/" in markdown
     await runtime.stop_when_idle()
 
+
 if __name__ == "__main__":
     """Runs this file's tests from the command line."""
     asyncio.run(test_web_surfer())

python/packages/team-one/tests/test_example.py

@@ -1,6 +1,7 @@
 import team_one
 
+
 def test_about() -> None:
     about = team_one.ABOUT
 
-    assert isinstance(about, str)
+    assert isinstance(about, str)