diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py index 77a3581f7..3d55ff154 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_code_executor_agent.py @@ -2,7 +2,7 @@ import re from typing import List, Sequence from autogen_core import CancellationToken -from autogen_core.components.code_executor import CodeBlock, CodeExecutor +from autogen_core.code_executor import CodeBlock, CodeExecutor from ..base import Response from ..messages import ChatMessage, TextMessage diff --git a/python/packages/autogen-agentchat/tests/test_code_executor_agent.py b/python/packages/autogen-agentchat/tests/test_code_executor_agent.py index ca09a203b..89476586a 100644 --- a/python/packages/autogen-agentchat/tests/test_code_executor_agent.py +++ b/python/packages/autogen-agentchat/tests/test_code_executor_agent.py @@ -3,7 +3,7 @@ from autogen_agentchat.agents import CodeExecutorAgent from autogen_agentchat.base import Response from autogen_agentchat.messages import TextMessage from autogen_core import CancellationToken -from autogen_core.components.code_executor import LocalCommandLineCodeExecutor +from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor @pytest.mark.asyncio diff --git a/python/packages/autogen-agentchat/tests/test_group_chat.py b/python/packages/autogen-agentchat/tests/test_group_chat.py index 12115664e..689be6238 100644 --- a/python/packages/autogen-agentchat/tests/test_group_chat.py +++ b/python/packages/autogen-agentchat/tests/test_group_chat.py @@ -33,9 +33,9 @@ from autogen_agentchat.teams._group_chat._selector_group_chat import SelectorGro from autogen_agentchat.teams._group_chat._swarm_group_chat import SwarmGroupChatManager from autogen_agentchat.ui import Console from autogen_core import AgentId, CancellationToken, FunctionCall -from autogen_core.components.code_executor import LocalCommandLineCodeExecutor from autogen_core.components.models import FunctionExecutionResult from autogen_core.components.tools import FunctionTool +from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor from autogen_ext.models import OpenAIChatCompletionClient, ReplayChatCompletionClient from openai.resources.chat.completions import AsyncCompletions from openai.types.chat.chat_completion import ChatCompletion, Choice diff --git a/python/packages/autogen-core/docs/src/reference/index.md b/python/packages/autogen-core/docs/src/reference/index.md index ccdffee61..c3e8dc104 100644 --- a/python/packages/autogen-core/docs/src/reference/index.md +++ b/python/packages/autogen-core/docs/src/reference/index.md @@ -26,8 +26,8 @@ python/autogen_agentchat.state :caption: AutoGen Core python/autogen_core +python/autogen_core.code_executor python/autogen_core.components.models -python/autogen_core.components.code_executor python/autogen_core.components.model_context python/autogen_core.components.tools python/autogen_core.components.tool_agent diff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst new file mode 100644 index 000000000..f05bdd68d --- /dev/null +++ b/python/packages/autogen-core/docs/src/reference/python/autogen_core.code_executor.rst @@ -0,0 +1,7 @@ +autogen\_core.code_executor +=========================== + +.. automodule:: autogen_core.code_executor + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst deleted file mode 100644 index 54d0cb095..000000000 --- a/python/packages/autogen-core/docs/src/reference/python/autogen_core.components.code_executor.rst +++ /dev/null @@ -1,8 +0,0 @@ -autogen\_core.components.code\_executor -======================================= - - -.. automodule:: autogen_core.components.code_executor - :members: - :undoc-members: - :show-inheritance: diff --git a/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb b/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb index 19d0ad489..152079594 100644 --- a/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb @@ -11,7 +11,7 @@ "This means that each code block is executed in a new process. There are two forms of this executor:\n", "\n", "- Docker ({py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n", - "- Local ({py:class}`~autogen_core.components.code_executor.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n", + "- Local ({py:class}`~autogen_core.code_executor.local.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n", "\n", "## Docker\n", "\n", @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -55,7 +55,7 @@ "from pathlib import Path\n", "\n", "from autogen_core import CancellationToken\n", - "from autogen_core.components.code_executor import CodeBlock\n", + "from autogen_core.code_executor import CodeBlock\n", "from autogen_ext.code_executors import DockerCommandLineCodeExecutor\n", "\n", "work_dir = Path(\"coding\")\n", @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -126,7 +126,8 @@ "from pathlib import Path\n", "\n", "from autogen_core import CancellationToken\n", - "from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor\n", + "from autogen_core.code_executor import CodeBlock\n", + "from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor\n", "\n", "work_dir = Path(\"coding\")\n", "work_dir.mkdir(exist_ok=True)\n", @@ -153,7 +154,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -172,7 +173,8 @@ "from pathlib import Path\n", "\n", "from autogen_core import CancellationToken\n", - "from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor\n", + "from autogen_core.code_executor import CodeBlock\n", + "from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor\n", "\n", "work_dir = Path(\"coding\")\n", "work_dir.mkdir(exist_ok=True)\n", @@ -197,20 +199,6 @@ "source": [ "As we can see, the code has executed successfully, and the installation has been isolated to the newly created virtual environment, without affecting our global environment." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -229,7 +217,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py b/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py new file mode 100644 index 000000000..f1789a546 --- /dev/null +++ b/python/packages/autogen-core/src/autogen_core/code_executor/__init__.py @@ -0,0 +1,21 @@ +from ._base import CodeBlock, CodeExecutor, CodeResult +from ._func_with_reqs import ( + Alias, + FunctionWithRequirements, + FunctionWithRequirementsStr, + Import, + ImportFromModule, + with_requirements, +) + +__all__ = [ + "CodeBlock", + "CodeExecutor", + "CodeResult", + "Alias", + "ImportFromModule", + "Import", + "FunctionWithRequirements", + "FunctionWithRequirementsStr", + "with_requirements", +] diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/_base.py b/python/packages/autogen-core/src/autogen_core/code_executor/_base.py new file mode 100644 index 000000000..16f023a21 --- /dev/null +++ b/python/packages/autogen-core/src/autogen_core/code_executor/_base.py @@ -0,0 +1,59 @@ +# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/base.py +# Credit to original authors + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Protocol, runtime_checkable + +from .._cancellation_token import CancellationToken + + +@dataclass +class CodeBlock: + """A code block extracted fromm an agent message.""" + + code: str + language: str + + +@dataclass +class CodeResult: + """Result of a code execution.""" + + exit_code: int + output: str + + +@runtime_checkable +class CodeExecutor(Protocol): + """Executes code blocks and returns the result.""" + + async def execute_code_blocks( + self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken + ) -> CodeResult: + """Execute code blocks and return the result. + + This method should be implemented by the code executor. + + Args: + code_blocks (List[CodeBlock]): The code blocks to execute. + + Returns: + CodeResult: The result of the code execution. + + Raises: + ValueError: Errors in user inputs + asyncio.TimeoutError: Code execution timeouts + asyncio.CancelledError: CancellationToken evoked during execution + """ + ... + + async def restart(self) -> None: + """Restart the code executor. + + This method should be implemented by the code executor. + + This method is called when the agent is reset. + """ + ... diff --git a/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py b/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py new file mode 100644 index 000000000..b7f4fcaef --- /dev/null +++ b/python/packages/autogen-core/src/autogen_core/code_executor/_func_with_reqs.py @@ -0,0 +1,201 @@ +# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/func_with_reqs.py +# Credit to original authors + +from __future__ import annotations + +import functools +import inspect +from dataclasses import dataclass, field +from importlib.abc import SourceLoader +from importlib.util import module_from_spec, spec_from_loader +from textwrap import dedent, indent +from typing import Any, Callable, Generic, List, Sequence, Set, TypeVar, Union + +from typing_extensions import ParamSpec + +T = TypeVar("T") +P = ParamSpec("P") + + +def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str: + if isinstance(func, FunctionWithRequirementsStr): + return func.func + + code = inspect.getsource(func) + # Strip the decorator + if code.startswith("@"): + code = code[code.index("\n") + 1 :] + return code + + +@dataclass +class Alias: + name: str + alias: str + + +@dataclass +class ImportFromModule: + module: str + imports: List[Union[str, Alias]] + + +Import = Union[str, ImportFromModule, Alias] + + +def _import_to_str(im: Import) -> str: + if isinstance(im, str): + return f"import {im}" + elif isinstance(im, Alias): + return f"import {im.name} as {im.alias}" + else: + + def to_str(i: Union[str, Alias]) -> str: + if isinstance(i, str): + return i + else: + return f"{i.name} as {i.alias}" + + imports = ", ".join(map(to_str, im.imports)) + return f"from {im.module} import {imports}" + + +class _StringLoader(SourceLoader): + def __init__(self, data: str): + self.data = data + + def get_source(self, fullname: str) -> str: + return self.data + + def get_data(self, path: str) -> bytes: + return self.data.encode("utf-8") + + def get_filename(self, fullname: str) -> str: + return "/" + fullname + ".py" + + +@dataclass +class FunctionWithRequirementsStr: + func: str + compiled_func: Callable[..., Any] + _func_name: str + python_packages: Sequence[str] = field(default_factory=list) + global_imports: Sequence[Import] = field(default_factory=list) + + def __init__(self, func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = []): + self.func = func + self.python_packages = python_packages + self.global_imports = global_imports + + module_name = "func_module" + loader = _StringLoader(func) + spec = spec_from_loader(module_name, loader) + if spec is None: + raise ValueError("Could not create spec") + module = module_from_spec(spec) + if spec.loader is None: + raise ValueError("Could not create loader") + + try: + spec.loader.exec_module(module) + except Exception as e: + raise ValueError(f"Could not compile function: {e}") from e + + functions = inspect.getmembers(module, inspect.isfunction) + if len(functions) != 1: + raise ValueError("The string must contain exactly one function") + + self._func_name, self.compiled_func = functions[0] + + def __call__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError("String based function with requirement objects are not directly callable") + + +@dataclass +class FunctionWithRequirements(Generic[T, P]): + func: Callable[P, T] + python_packages: Sequence[str] = field(default_factory=list) + global_imports: Sequence[Import] = field(default_factory=list) + + @classmethod + def from_callable( + cls, func: Callable[P, T], python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] + ) -> FunctionWithRequirements[T, P]: + return cls(python_packages=python_packages, global_imports=global_imports, func=func) + + @staticmethod + def from_str( + func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] + ) -> FunctionWithRequirementsStr: + return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports) + + # Type this based on F + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + return self.func(*args, **kwargs) + + +def with_requirements( + python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] +) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: + """Decorate a function with package and import requirements + + Args: + python_packages (List[str], optional): Packages required to function. Can include version info.. Defaults to []. + global_imports (List[Import], optional): Required imports. Defaults to []. + + Returns: + Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: The decorated function + """ + + def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]: + func_with_reqs = FunctionWithRequirements( + python_packages=python_packages, global_imports=global_imports, func=func + ) + + functools.update_wrapper(func_with_reqs, func) + return func_with_reqs + + return wrapper + + +def build_python_functions_file( + funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]], +) -> str: + """:meta private:""" + # First collect all global imports + global_imports: Set[Import] = set() + for func in funcs: + if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)): + global_imports.update(func.global_imports) + + content = "\n".join(map(_import_to_str, global_imports)) + "\n\n" + + for func in funcs: + content += _to_code(func) + "\n\n" + + return content + + +def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str: + """Generate a stub for a function as a string + + Args: + func (Callable[..., Any]): The function to generate a stub for + + Returns: + str: The stub for the function + """ + if isinstance(func, FunctionWithRequirementsStr): + return to_stub(func.compiled_func) + + content = f"def {func.__name__}{inspect.signature(func)}:\n" + docstring = func.__doc__ + + if docstring: + docstring = dedent(docstring) + docstring = '"""' + docstring + '"""' + docstring = indent(docstring, " ") + content += docstring + "\n" + + content += " ..." + return content diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py index 805b718ff..e1265a81c 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py @@ -1,17 +1,14 @@ -from ._base import CodeBlock, CodeExecutor, CodeResult +from ._base import CodeBlock, CodeExecutor, CodeResult # type: ignore from ._func_with_reqs import ( - Alias, - FunctionWithRequirements, - FunctionWithRequirementsStr, + Alias, # type: ignore + FunctionWithRequirements, # type: ignore + FunctionWithRequirementsStr, # type: ignore Import, - ImportFromModule, - build_python_functions_file, - to_stub, - with_requirements, + ImportFromModule, # type: ignore + with_requirements, # type: ignore ) -from ._impl.command_line_code_result import CommandLineCodeResult -from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor -from ._impl.utils import get_file_name_from_content, get_required_packages, lang_to_cmd, silence_pip +from ._impl.command_line_code_result import CommandLineCodeResult # type: ignore +from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor # type: ignore __all__ = [ "LocalCommandLineCodeExecutor", @@ -25,11 +22,4 @@ __all__ = [ "FunctionWithRequirements", "FunctionWithRequirementsStr", "with_requirements", - "to_stub", - "get_required_packages", - "build_python_functions_file", - "get_required_packages", - "lang_to_cmd", - "get_file_name_from_content", - "silence_pip", ] diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py index 4c12f5735..420a96447 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_base.py @@ -6,9 +6,12 @@ from __future__ import annotations from dataclasses import dataclass from typing import List, Protocol, runtime_checkable +from typing_extensions import deprecated + from ... import CancellationToken +@deprecated("Moved to autogen_core.code_executor.CodeBlock. This alias will be removed in 0.4.0.") @dataclass class CodeBlock: """A code block extracted fromm an agent message.""" @@ -17,6 +20,7 @@ class CodeBlock: language: str +@deprecated("Moved to autogen_core.code_executor.CodeResult. This alias will be removed in 0.4.0.") @dataclass class CodeResult: """Result of a code execution.""" @@ -25,13 +29,16 @@ class CodeResult: output: str +@deprecated("Moved to autogen_core.code_executor.CodeExecutor. This alias will be removed in 0.4.0.") @runtime_checkable class CodeExecutor(Protocol): """Executes code blocks and returns the result.""" async def execute_code_blocks( - self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken - ) -> CodeResult: + self, + code_blocks: List[CodeBlock], # type: ignore + cancellation_token: CancellationToken, # type: ignore + ) -> CodeResult: # type: ignore """Execute code blocks and return the result. This method should be implemented by the code executor. diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py index b7f4fcaef..98c39518e 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_func_with_reqs.py @@ -11,14 +11,14 @@ from importlib.util import module_from_spec, spec_from_loader from textwrap import dedent, indent from typing import Any, Callable, Generic, List, Sequence, Set, TypeVar, Union -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, deprecated T = TypeVar("T") P = ParamSpec("P") -def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str: - if isinstance(func, FunctionWithRequirementsStr): +def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str: # type: ignore + if isinstance(func, FunctionWithRequirementsStr): # type: ignore return func.func code = inspect.getsource(func) @@ -28,29 +28,31 @@ def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], Functio return code +@deprecated("Moved to autogen_core.code_executor.Alias. This alias will be removed in 0.4.0.") @dataclass class Alias: name: str alias: str +@deprecated("Moved to autogen_core.code_executor.ImportFromModule. This alias will be removed in 0.4.0.") @dataclass class ImportFromModule: module: str - imports: List[Union[str, Alias]] + imports: List[Union[str, Alias]] # type: ignore -Import = Union[str, ImportFromModule, Alias] +Import = Union[str, ImportFromModule, Alias] # type: ignore def _import_to_str(im: Import) -> str: if isinstance(im, str): return f"import {im}" - elif isinstance(im, Alias): + elif isinstance(im, Alias): # type: ignore return f"import {im.name} as {im.alias}" else: - def to_str(i: Union[str, Alias]) -> str: + def to_str(i: Union[str, Alias]) -> str: # type: ignore if isinstance(i, str): return i else: @@ -74,6 +76,7 @@ class _StringLoader(SourceLoader): return "/" + fullname + ".py" +@deprecated("Moved to autogen_core.code_executor.CodeBlock. This alias will be removed in 0.4.0.") @dataclass class FunctionWithRequirementsStr: func: str @@ -111,6 +114,7 @@ class FunctionWithRequirementsStr: raise NotImplementedError("String based function with requirement objects are not directly callable") +@deprecated("Moved to autogen_core.code_executor.FunctionWithRequirements. This alias will be removed in 0.4.0.") @dataclass class FunctionWithRequirements(Generic[T, P]): func: Callable[P, T] @@ -120,23 +124,24 @@ class FunctionWithRequirements(Generic[T, P]): @classmethod def from_callable( cls, func: Callable[P, T], python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] - ) -> FunctionWithRequirements[T, P]: + ) -> FunctionWithRequirements[T, P]: # type: ignore return cls(python_packages=python_packages, global_imports=global_imports, func=func) @staticmethod def from_str( func: str, python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] - ) -> FunctionWithRequirementsStr: - return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports) + ) -> FunctionWithRequirementsStr: # type: ignore + return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports) # type: ignore # Type this based on F def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: return self.func(*args, **kwargs) +@deprecated("Moved to autogen_core.code_executor.with_requirements. This alias will be removed in 0.4.0.") def with_requirements( python_packages: Sequence[str] = [], global_imports: Sequence[Import] = [] -) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: +) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: # type: ignore """Decorate a function with package and import requirements Args: @@ -147,8 +152,8 @@ def with_requirements( Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: The decorated function """ - def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]: - func_with_reqs = FunctionWithRequirements( + def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]: # type: ignore + func_with_reqs = FunctionWithRequirements( # type: ignore python_packages=python_packages, global_imports=global_imports, func=func ) @@ -159,13 +164,13 @@ def with_requirements( def build_python_functions_file( - funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]], + funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]], # type: ignore ) -> str: """:meta private:""" # First collect all global imports global_imports: Set[Import] = set() for func in funcs: - if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)): + if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)): # type: ignore global_imports.update(func.global_imports) content = "\n".join(map(_import_to_str, global_imports)) + "\n\n" @@ -176,7 +181,7 @@ def build_python_functions_file( return content -def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str: +def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str: # type: ignore """Generate a stub for a function as a string Args: @@ -185,7 +190,7 @@ def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str Returns: str: The stub for the function """ - if isinstance(func, FunctionWithRequirementsStr): + if isinstance(func, FunctionWithRequirementsStr): # type: ignore return to_stub(func.compiled_func) content = f"def {func.__name__}{inspect.signature(func)}:\n" diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/base_code_executor.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/base_code_executor.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py index cafc73f14..879ef8af3 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/command_line_code_result.py @@ -1,9 +1,14 @@ from dataclasses import dataclass from typing import Optional -from .._base import CodeResult +from typing_extensions import deprecated + +from ....code_executor._base import CodeResult +@deprecated( + "CommandLineCodeResult moved to autogen_ext.code_executors.CommandLineCodeResult. This alias will be removed in 0.4.0." +) @dataclass class CommandLineCodeResult(CodeResult): """A code result class for command line code executor.""" diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py index 01e098881..e6a71d65b 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/local_commandline_code_executor.py @@ -12,17 +12,17 @@ from string import Template from types import SimpleNamespace from typing import Any, Callable, ClassVar, List, Optional, Sequence, Union -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, deprecated from .... import CancellationToken -from .._base import CodeBlock, CodeExecutor -from .._func_with_reqs import ( +from ....code_executor._base import CodeBlock, CodeExecutor +from ....code_executor._func_with_reqs import ( FunctionWithRequirements, FunctionWithRequirementsStr, build_python_functions_file, to_stub, ) -from .command_line_code_result import CommandLineCodeResult +from .command_line_code_result import CommandLineCodeResult # type: ignore from .utils import PYTHON_VARIANTS, get_file_name_from_content, lang_to_cmd, silence_pip # type: ignore __all__ = ("LocalCommandLineCodeExecutor",) @@ -30,6 +30,9 @@ __all__ = ("LocalCommandLineCodeExecutor",) A = ParamSpec("A") +@deprecated( + "LocalCommandLineCodeExecutor moved to autogen_ext.code_executors.local.LocalCommandLineCodeExecutor. This alias will be removed in 0.4.0." +) class LocalCommandLineCodeExecutor(CodeExecutor): """A code executor class that executes code through a local command line environment. @@ -70,7 +73,8 @@ class LocalCommandLineCodeExecutor(CodeExecutor): import asyncio from autogen_core import CancellationToken - from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor + from autogen_core.code_executor import CodeBlock + from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor async def example(): @@ -243,7 +247,7 @@ $functions""" async def execute_code_blocks( self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken - ) -> CommandLineCodeResult: + ) -> CommandLineCodeResult: # type: ignore """(Experimental) Execute the code blocks and return the result. Args: @@ -260,7 +264,7 @@ $functions""" async def _execute_code_dont_check_setup( self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken - ) -> CommandLineCodeResult: + ) -> CommandLineCodeResult: # type: ignore logs_all: str = "" file_names: List[Path] = [] exitcode = 0 @@ -283,7 +287,7 @@ $functions""" # Check if there is a filename comment filename = get_file_name_from_content(code, self._work_dir) except ValueError: - return CommandLineCodeResult( + return CommandLineCodeResult( # type: ignore exit_code=1, output="Filename is not in the workspace", code_file=None, @@ -347,7 +351,7 @@ $functions""" break code_file = str(file_names[0]) if len(file_names) > 0 else None - return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file) + return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file) # type: ignore async def restart(self) -> None: """(Experimental) Restart the code executor.""" diff --git a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py index 98ce3e2de..3fc39054a 100644 --- a/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py +++ b/python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/utils.py @@ -46,24 +46,6 @@ def silence_pip(code: str, lang: str) -> str: return "\n".join(lines) -def get_required_packages(code: str, lang: str) -> set[str]: - ret: set[str] = set() - if lang == "python": - regex = r"^! ?pip install(.*)$" - else: - return ret - - # Find lines that start with pip install and make sure "-qqq" flag is added. - lines = code.split("\n") - for _, line in enumerate(lines): - # use regex to find lines that start with pip install. - match = re.search(regex, line) - if match is not None: - reqs = match.group(1).split(",") - ret = {req.strip(" ") for req in reqs} - return ret - - PYTHON_VARIANTS = ["python", "Python", "py"] diff --git a/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py b/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py index 5b3f0bcad..eb792a32f 100644 --- a/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py +++ b/python/packages/autogen-core/src/autogen_core/components/tools/_code_execution.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field, model_serializer from ... import CancellationToken -from ..code_executor import CodeBlock, CodeExecutor +from ...code_executor import CodeBlock, CodeExecutor from ._base import BaseTool diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py index f984cdf3a..c54d7c54c 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_azure_container_code_executor.py @@ -14,18 +14,17 @@ import aiohttp # async functions shouldn't use open() from anyio import open_file from autogen_core import CancellationToken -from autogen_core.components.code_executor import ( +from autogen_core.code_executor import ( CodeBlock, CodeExecutor, CodeResult, FunctionWithRequirements, FunctionWithRequirementsStr, - build_python_functions_file, - get_required_packages, - to_stub, ) from typing_extensions import ParamSpec +from ._common import build_python_functions_file, get_required_packages, to_stub + if TYPE_CHECKING: from azure.core.credentials import AccessToken diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py new file mode 100644 index 000000000..ce656e93f --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py @@ -0,0 +1,190 @@ +import inspect +import re +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent, indent +from typing import Any, Callable, Optional, Sequence, Set, TypeVar, Union + +from autogen_core.code_executor import Alias, CodeResult, FunctionWithRequirements, FunctionWithRequirementsStr, Import +from typing_extensions import ParamSpec + + +@dataclass +class CommandLineCodeResult(CodeResult): + """A code result class for command line code executor.""" + + code_file: Optional[str] + + +T = TypeVar("T") +P = ParamSpec("P") + + +def _to_code(func: Union[FunctionWithRequirements[T, P], Callable[P, T], FunctionWithRequirementsStr]) -> str: + if isinstance(func, FunctionWithRequirementsStr): + return func.func + + code = inspect.getsource(func) + # Strip the decorator + if code.startswith("@"): + code = code[code.index("\n") + 1 :] + return code + + +def _import_to_str(im: Import) -> str: + if isinstance(im, str): + return f"import {im}" + elif isinstance(im, Alias): + return f"import {im.name} as {im.alias}" + else: + + def to_str(i: Union[str, Alias]) -> str: + if isinstance(i, str): + return i + else: + return f"{i.name} as {i.alias}" + + imports = ", ".join(map(to_str, im.imports)) + return f"from {im.module} import {imports}" + + +def build_python_functions_file( + funcs: Sequence[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]], +) -> str: + """:meta private:""" + # First collect all global imports + global_imports: Set[Import] = set() + for func in funcs: + if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)): + global_imports.update(func.global_imports) + + content = "\n".join(map(_import_to_str, global_imports)) + "\n\n" + + for func in funcs: + content += _to_code(func) + "\n\n" + + return content + + +def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str: + """Generate a stub for a function as a string + + Args: + func (Callable[..., Any]): The function to generate a stub for + + Returns: + str: The stub for the function + """ + if isinstance(func, FunctionWithRequirementsStr): + return to_stub(func.compiled_func) + + content = f"def {func.__name__}{inspect.signature(func)}:\n" + docstring = func.__doc__ + + if docstring: + docstring = dedent(docstring) + docstring = '"""' + docstring + '"""' + docstring = indent(docstring, " ") + content += docstring + "\n" + + content += " ..." + return content + + +# Raises ValueError if the file is not in the workspace +def get_file_name_from_content(code: str, workspace_path: Path) -> Optional[str]: + first_line = code.split("\n")[0] + # TODO - support other languages + if first_line.startswith("# filename:"): + filename = first_line.split(":")[1].strip() + + # Handle relative paths in the filename + path = Path(filename) + if not path.is_absolute(): + path = workspace_path / path + path = path.resolve() + # Throws an error if the file is not in the workspace + relative = path.relative_to(workspace_path.resolve()) + return str(relative) + + return None + + +def silence_pip(code: str, lang: str) -> str: + """Apply -qqq flag to pip install commands.""" + if lang == "python": + regex = r"^! ?pip install" + elif lang in ["bash", "shell", "sh", "pwsh", "powershell", "ps1"]: + regex = r"^pip install" + else: + return code + + # Find lines that start with pip install and make sure "-qqq" flag is added. + lines = code.split("\n") + for i, line in enumerate(lines): + # use regex to find lines that start with pip install. + match = re.search(regex, line) + if match is not None: + if "-qqq" not in line: + lines[i] = line.replace(match.group(0), match.group(0) + " -qqq") + return "\n".join(lines) + + +def get_required_packages(code: str, lang: str) -> set[str]: + ret: set[str] = set() + if lang == "python": + regex = r"^! ?pip install(.*)$" + else: + return ret + + # Find lines that start with pip install and make sure "-qqq" flag is added. + lines = code.split("\n") + for _, line in enumerate(lines): + # use regex to find lines that start with pip install. + match = re.search(regex, line) + if match is not None: + reqs = match.group(1).split(",") + ret = {req.strip(" ") for req in reqs} + return ret + + +PYTHON_VARIANTS = ["python", "Python", "py"] + + +def lang_to_cmd(lang: str) -> str: + if lang in PYTHON_VARIANTS: + return "python" + if lang.startswith("python") or lang in ["bash", "sh"]: + return lang + if lang in ["shell"]: + return "sh" + else: + raise ValueError(f"Unsupported language: {lang}") + + +# Regular expression for finding a code block +# ```[ \t]*(\w+)?[ \t]*\r?\n(.*?)[ \t]*\r?\n``` Matches multi-line code blocks. +# The [ \t]* matches the potential spaces before language name. +# The (\w+)? matches the language, where the ? indicates it is optional. +# The [ \t]* matches the potential spaces (not newlines) after language name. +# The \r?\n makes sure there is a linebreak after ```. +# The (.*?) matches the code itself (non-greedy). +# The \r?\n makes sure there is a linebreak before ```. +# The [ \t]* matches the potential spaces before closing ``` (the spec allows indentation). +CODE_BLOCK_PATTERN = r"```[ \t]*(\w+)?[ \t]*\r?\n(.*?)\r?\n[ \t]*```" + + +def infer_lang(code: str) -> str: + """infer the language for the code. + TODO: make it robust. + """ + if code.startswith("python ") or code.startswith("pip") or code.startswith("python3 "): + return "sh" + + # check if code is a valid python code + try: + compile(code, "test", "exec") + return "python" + except SyntaxError: + # not a valid python code + return "unknown" diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py index 442486a83..170c4f468 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_docker_code_executor.py @@ -15,12 +15,15 @@ from types import TracebackType from typing import Any, Callable, ClassVar, List, Optional, ParamSpec, Type, Union from autogen_core import CancellationToken -from autogen_core.components.code_executor import ( +from autogen_core.code_executor import ( CodeBlock, CodeExecutor, - CommandLineCodeResult, FunctionWithRequirements, FunctionWithRequirementsStr, +) + +from ._common import ( + CommandLineCodeResult, build_python_functions_file, get_file_name_from_content, lang_to_cmd, diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py new file mode 100644 index 000000000..9055804d1 --- /dev/null +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py @@ -0,0 +1,358 @@ +# File based from: https://github.com/microsoft/autogen/blob/main/autogen/coding/local_commandline_code_executor.py +# Credit to original authors + +import asyncio +import logging +import os +import sys +import warnings +from hashlib import sha256 +from pathlib import Path +from string import Template +from types import SimpleNamespace +from typing import Any, Callable, ClassVar, List, Optional, Sequence, Union + +from autogen_core import CancellationToken +from autogen_core.code_executor import CodeBlock, CodeExecutor, FunctionWithRequirements, FunctionWithRequirementsStr +from typing_extensions import ParamSpec + +from .._common import ( + PYTHON_VARIANTS, + CommandLineCodeResult, + build_python_functions_file, + get_file_name_from_content, + lang_to_cmd, + silence_pip, + to_stub, +) + +__all__ = ("LocalCommandLineCodeExecutor",) + +A = ParamSpec("A") + + +class LocalCommandLineCodeExecutor(CodeExecutor): + """A code executor class that executes code through a local command line + environment. + + .. danger:: + + This will execute code on the local machine. If being used with LLM generated code, caution should be used. + + Each code block is saved as a file and executed in a separate process in + the working directory, and a unique file is generated and saved in the + working directory for each code block. + The code blocks are executed in the order they are received. + Command line code is sanitized using regular expression match against a list of dangerous commands in order to prevent self-destructive + commands from being executed which may potentially affect the users environment. + Currently the only supported languages is Python and shell scripts. + For Python code, use the language "python" for the code block. + For shell scripts, use the language "bash", "shell", or "sh" for the code + block. + + Args: + timeout (int): The timeout for the execution of any single code block. Default is 60. + work_dir (str): The working directory for the code execution. If None, + a default working directory will be used. The default working + directory is the current directory ".". + functions (List[Union[FunctionWithRequirements[Any, A], Callable[..., Any]]]): A list of functions that are available to the code executor. Default is an empty list. + functions_module (str, optional): The name of the module that will be created to store the functions. Defaults to "functions". + virtual_env_context (Optional[SimpleNamespace], optional): The virtual environment context. Defaults to None. + + Example: + + How to use `LocalCommandLineCodeExecutor` with a virtual environment different from the one used to run the autogen application: + Set up a virtual environment using the `venv` module, and pass its context to the initializer of `LocalCommandLineCodeExecutor`. This way, the executor will run code within the new environment. + + .. code-block:: python + + import venv + from pathlib import Path + import asyncio + + from autogen_core import CancellationToken + from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor + + + async def example(): + work_dir = Path("coding") + work_dir.mkdir(exist_ok=True) + + venv_dir = work_dir / ".venv" + venv_builder = venv.EnvBuilder(with_pip=True) + venv_builder.create(venv_dir) + venv_context = venv_builder.ensure_directories(venv_dir) + + local_executor = LocalCommandLineCodeExecutor(work_dir=work_dir, virtual_env_context=venv_context) + await local_executor.execute_code_blocks( + code_blocks=[ + CodeBlock(language="bash", code="pip install matplotlib"), + ], + cancellation_token=CancellationToken(), + ) + + + asyncio.run(example()) + + """ + + SUPPORTED_LANGUAGES: ClassVar[List[str]] = [ + "bash", + "shell", + "sh", + "pwsh", + "powershell", + "ps1", + "python", + ] + FUNCTION_PROMPT_TEMPLATE: ClassVar[ + str + ] = """You have access to the following user defined functions. They can be accessed from the module called `$module_name` by their function names. + +For example, if there was a function called `foo` you could import it by writing `from $module_name import foo` + +$functions""" + + def __init__( + self, + timeout: int = 60, + work_dir: Union[Path, str] = Path("."), + functions: Sequence[ + Union[ + FunctionWithRequirements[Any, A], + Callable[..., Any], + FunctionWithRequirementsStr, + ] + ] = [], + functions_module: str = "functions", + virtual_env_context: Optional[SimpleNamespace] = None, + ): + if timeout < 1: + raise ValueError("Timeout must be greater than or equal to 1.") + + if isinstance(work_dir, str): + work_dir = Path(work_dir) + + if not functions_module.isidentifier(): + raise ValueError("Module name must be a valid Python identifier") + + self._functions_module = functions_module + + work_dir.mkdir(exist_ok=True) + + self._timeout = timeout + self._work_dir: Path = work_dir + + self._functions = functions + # Setup could take some time so we intentionally wait for the first code block to do it. + if len(functions) > 0: + self._setup_functions_complete = False + else: + self._setup_functions_complete = True + + self._virtual_env_context: Optional[SimpleNamespace] = virtual_env_context + + def format_functions_for_prompt(self, prompt_template: str = FUNCTION_PROMPT_TEMPLATE) -> str: + """(Experimental) Format the functions for a prompt. + + The template includes two variables: + - `$module_name`: The module name. + - `$functions`: The functions formatted as stubs with two newlines between each function. + + Args: + prompt_template (str): The prompt template. Default is the class default. + + Returns: + str: The formatted prompt. + """ + + template = Template(prompt_template) + return template.substitute( + module_name=self._functions_module, + functions="\n\n".join([to_stub(func) for func in self._functions]), + ) + + @property + def functions_module(self) -> str: + """(Experimental) The module name for the functions.""" + return self._functions_module + + @property + def functions(self) -> List[str]: + raise NotImplementedError + + @property + def timeout(self) -> int: + """(Experimental) The timeout for code execution.""" + return self._timeout + + @property + def work_dir(self) -> Path: + """(Experimental) The working directory for the code execution.""" + return self._work_dir + + async def _setup_functions(self, cancellation_token: CancellationToken) -> None: + func_file_content = build_python_functions_file(self._functions) + func_file = self._work_dir / f"{self._functions_module}.py" + func_file.write_text(func_file_content) + + # Collect requirements + lists_of_packages = [x.python_packages for x in self._functions if isinstance(x, FunctionWithRequirements)] + flattened_packages = [item for sublist in lists_of_packages for item in sublist] + required_packages = list(set(flattened_packages)) + if len(required_packages) > 0: + logging.info("Ensuring packages are installed in executor.") + + cmd_args = ["-m", "pip", "install"] + cmd_args.extend(required_packages) + + if self._virtual_env_context: + py_executable = self._virtual_env_context.env_exe + else: + py_executable = sys.executable + + task = asyncio.create_task( + asyncio.create_subprocess_exec( + py_executable, + *cmd_args, + cwd=self._work_dir, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + ) + cancellation_token.link_future(task) + try: + proc = await task + stdout, stderr = await asyncio.wait_for(proc.communicate(), self._timeout) + except asyncio.TimeoutError as e: + raise ValueError("Pip install timed out") from e + except asyncio.CancelledError as e: + raise ValueError("Pip install was cancelled") from e + + if proc.returncode is not None and proc.returncode != 0: + raise ValueError(f"Pip install failed. {stdout.decode()}, {stderr.decode()}") + + # Attempt to load the function file to check for syntax errors, imports etc. + exec_result = await self._execute_code_dont_check_setup( + [CodeBlock(code=func_file_content, language="python")], cancellation_token + ) + + if exec_result.exit_code != 0: + raise ValueError(f"Functions failed to load: {exec_result.output}") + + self._setup_functions_complete = True + + async def execute_code_blocks( + self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken + ) -> CommandLineCodeResult: + """(Experimental) Execute the code blocks and return the result. + + Args: + code_blocks (List[CodeBlock]): The code blocks to execute. + cancellation_token (CancellationToken): a token to cancel the operation + + Returns: + CommandLineCodeResult: The result of the code execution.""" + + if not self._setup_functions_complete: + await self._setup_functions(cancellation_token) + + return await self._execute_code_dont_check_setup(code_blocks, cancellation_token) + + async def _execute_code_dont_check_setup( + self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken + ) -> CommandLineCodeResult: + logs_all: str = "" + file_names: List[Path] = [] + exitcode = 0 + for code_block in code_blocks: + lang, code = code_block.language, code_block.code + lang = lang.lower() + + code = silence_pip(code, lang) + + if lang in PYTHON_VARIANTS: + lang = "python" + + if lang not in self.SUPPORTED_LANGUAGES: + # In case the language is not supported, we return an error message. + exitcode = 1 + logs_all += "\n" + f"unknown language {lang}" + break + + try: + # Check if there is a filename comment + filename = get_file_name_from_content(code, self._work_dir) + except ValueError: + return CommandLineCodeResult( + exit_code=1, + output="Filename is not in the workspace", + code_file=None, + ) + + if filename is None: + # create a file with an automatically generated name + code_hash = sha256(code.encode()).hexdigest() + filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}" + + written_file = (self._work_dir / filename).resolve() + with written_file.open("w", encoding="utf-8") as f: + f.write(code) + file_names.append(written_file) + + env = os.environ.copy() + + if self._virtual_env_context: + virtual_env_exe_abs_path = os.path.abspath(self._virtual_env_context.env_exe) + virtual_env_bin_abs_path = os.path.abspath(self._virtual_env_context.bin_path) + env["PATH"] = f"{virtual_env_bin_abs_path}{os.pathsep}{env['PATH']}" + + program = virtual_env_exe_abs_path if lang.startswith("python") else lang_to_cmd(lang) + else: + program = sys.executable if lang.startswith("python") else lang_to_cmd(lang) + + # Wrap in a task to make it cancellable + task = asyncio.create_task( + asyncio.create_subprocess_exec( + program, + str(written_file.absolute()), + cwd=self._work_dir, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + ) + cancellation_token.link_future(task) + try: + proc = await task + stdout, stderr = await asyncio.wait_for(proc.communicate(), self._timeout) + exitcode = proc.returncode or 0 + + except asyncio.TimeoutError: + logs_all += "\n Timeout" + # Same exit code as the timeout command on linux. + exitcode = 124 + break + except asyncio.CancelledError: + logs_all += "\n Cancelled" + # TODO: which exit code? 125 is Operation Canceled + exitcode = 125 + break + + self._running_cmd_task = None + + logs_all += stderr.decode() + logs_all += stdout.decode() + + if exitcode != 0: + break + + code_file = str(file_names[0]) if len(file_names) > 0 else None + return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file) + + async def restart(self) -> None: + """(Experimental) Restart the code executor.""" + warnings.warn( + "Restarting local command line code executor is not supported. No action is taken.", + stacklevel=2, + ) diff --git a/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py b/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py index df5caf5ce..e46912568 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py +++ b/python/packages/autogen-ext/tests/code_executors/test_aca_dynamic_sessions.py @@ -9,7 +9,7 @@ import tempfile import pytest from anyio import open_file from autogen_core import CancellationToken -from autogen_core.components.code_executor import CodeBlock +from autogen_core.code_executor import CodeBlock from autogen_ext.code_executors import ACADynamicSessionsCodeExecutor from azure.identity import DefaultAzureCredential diff --git a/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py b/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py index 7b8f2ef2a..e826abec0 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py +++ b/python/packages/autogen-ext/tests/code_executors/test_aca_user_defined_functions.py @@ -6,7 +6,7 @@ import os import polars import pytest from autogen_core import CancellationToken -from autogen_core.components.code_executor import ( +from autogen_core.code_executor import ( CodeBlock, FunctionWithRequirements, with_requirements, diff --git a/python/packages/autogen-core/tests/execution/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py similarity index 98% rename from python/packages/autogen-core/tests/execution/test_commandline_code_executor.py rename to python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py index 482330714..e5e4407f6 100644 --- a/python/packages/autogen-core/tests/execution/test_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py @@ -14,7 +14,8 @@ import pytest import pytest_asyncio from aiofiles import open from autogen_core import CancellationToken -from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor +from autogen_core.code_executor import CodeBlock +from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor @pytest_asyncio.fixture(scope="function") # type: ignore diff --git a/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py index bee4957f3..c9e57aee1 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_docker_commandline_code_executor.py @@ -9,7 +9,7 @@ import pytest import pytest_asyncio from aiofiles import open from autogen_core import CancellationToken -from autogen_core.components.code_executor import CodeBlock +from autogen_core.code_executor import CodeBlock from autogen_ext.code_executors import DockerCommandLineCodeExecutor diff --git a/python/packages/autogen-core/tests/execution/test_user_defined_functions.py b/python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py similarity index 98% rename from python/packages/autogen-core/tests/execution/test_user_defined_functions.py rename to python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py index 6aa3a9e45..0815e173b 100644 --- a/python/packages/autogen-core/tests/execution/test_user_defined_functions.py +++ b/python/packages/autogen-ext/tests/code_executors/test_user_defined_functions.py @@ -7,12 +7,12 @@ import tempfile import polars import pytest from autogen_core import CancellationToken -from autogen_core.components.code_executor import ( +from autogen_core.code_executor import ( CodeBlock, FunctionWithRequirements, - LocalCommandLineCodeExecutor, with_requirements, ) +from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor ENVIRON_KEY_AZURE_POOL_ENDPOINT = "AZURE_POOL_ENDPOINT" diff --git a/python/packages/autogen-magentic-one/examples/example.py b/python/packages/autogen-magentic-one/examples/example.py index 2b96db531..fb0a7d9b4 100644 --- a/python/packages/autogen-magentic-one/examples/example.py +++ b/python/packages/autogen-magentic-one/examples/example.py @@ -7,7 +7,7 @@ import os from autogen_core import AgentId, AgentProxy, SingleThreadedAgentRuntime from autogen_core.application.logging import EVENT_LOGGER_NAME -from autogen_core.components.code_executor import CodeBlock +from autogen_core.code_executor import CodeBlock from autogen_ext.code_executors import DockerCommandLineCodeExecutor from autogen_magentic_one.agents.coder import Coder, Executor from autogen_magentic_one.agents.file_surfer import FileSurfer diff --git a/python/packages/autogen-magentic-one/examples/example_coder.py b/python/packages/autogen-magentic-one/examples/example_coder.py index b861c5855..6c7bfc6e2 100644 --- a/python/packages/autogen-magentic-one/examples/example_coder.py +++ b/python/packages/autogen-magentic-one/examples/example_coder.py @@ -9,7 +9,7 @@ import logging from autogen_core import AgentId, AgentProxy, SingleThreadedAgentRuntime from autogen_core.application.logging import EVENT_LOGGER_NAME -from autogen_core.components.code_executor import CodeBlock +from autogen_core.code_executor import CodeBlock from autogen_ext.code_executors import DockerCommandLineCodeExecutor from autogen_magentic_one.agents.coder import Coder, Executor from autogen_magentic_one.agents.orchestrator import RoundRobinOrchestrator diff --git a/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py b/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py index f30a8da41..427adde86 100644 --- a/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py +++ b/python/packages/autogen-magentic-one/src/autogen_magentic_one/agents/coder.py @@ -2,7 +2,7 @@ import re from typing import Awaitable, Callable, List, Literal, Tuple, Union from autogen_core import CancellationToken, default_subscription -from autogen_core.components.code_executor import CodeBlock, CodeExecutor +from autogen_core.code_executor import CodeBlock, CodeExecutor from autogen_core.components.models import ( ChatCompletionClient, SystemMessage,