mirror of
https://github.com/microsoft/autogen.git
synced 2026-04-20 03:02:16 -04:00
refactor and general readme updates
This commit is contained in:
Victor Dibia
@@ -0,0 +1,25 @@
|
||||
# Import the main orchestrator
|
||||
from ._orchestrator import EvalOrchestrator
|
||||
|
||||
# Import judges
|
||||
from .judges import BaseEvalJudge, BaseEvalJudgeConfig, LLMEvalJudge, LLMEvalJudgeConfig
|
||||
|
||||
# Import runners
|
||||
from .runners import BaseEvalRunner, BaseEvalRunnerConfig, ModelEvalRunner, ModelEvalRunnerConfig, TeamEvalRunner, TeamEvalRunnerConfig
|
||||
|
||||
__all__ = [
|
||||
# Orchestrator
|
||||
"EvalOrchestrator",
|
||||
# Judges
|
||||
"BaseEvalJudge",
|
||||
"BaseEvalJudgeConfig",
|
||||
"LLMEvalJudge",
|
||||
"LLMEvalJudgeConfig",
|
||||
# Runners
|
||||
"BaseEvalRunner",
|
||||
"BaseEvalRunnerConfig",
|
||||
"ModelEvalRunner",
|
||||
"ModelEvalRunnerConfig",
|
||||
"TeamEvalRunner",
|
||||
"TeamEvalRunnerConfig",
|
||||
]
|
||||
@@ -369,7 +369,8 @@ class EvalOrchestrator:
|
||||
# Execute runner
|
||||
logger.info(f"Starting runner for run {run_id}")
|
||||
start_time = datetime.now()
|
||||
run_result = await runner.run(task)
|
||||
run_results = await runner.run([task])
|
||||
run_result = run_results[0]
|
||||
|
||||
# Update run result
|
||||
await self._update_run_result(run_id, run_result)
|
||||
@@ -0,0 +1,450 @@
|
||||
"""
|
||||
Comprehensive evaluation examples for AutoGen Studio.
|
||||
|
||||
This file demonstrates how to use the evaluation system to:
|
||||
1. Run simple evaluations with different runners
|
||||
2. Use the orchestrator for managing complex evaluation workflows
|
||||
3. Judge results with multiple criteria
|
||||
4. Test serialization and deserialization
|
||||
|
||||
Usage:
|
||||
python example_evaluation.py
|
||||
|
||||
Note: Requires OPENAI_API_KEY environment variable to be set.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
|
||||
from autogen_agentchat.agents import AssistantAgent
|
||||
from autogen_agentchat.teams import RoundRobinGroupChat
|
||||
from autogen_core import ComponentModel
|
||||
from autogen_core.models import UserMessage
|
||||
from autogen_ext.models.openai import OpenAIChatCompletionClient
|
||||
|
||||
# Import the evaluation components
|
||||
from autogenstudio.datamodel.eval import EvalJudgeCriteria, EvalRunResult, EvalRunStatus, EvalScore, EvalTask
|
||||
from autogenstudio.eval import EvalOrchestrator, LLMEvalJudge, ModelEvalRunner, TeamEvalRunner
|
||||
|
||||
|
||||
async def run_simple_evaluation():
|
||||
"""Run a simple evaluation of model and team responses."""
|
||||
|
||||
print("\n=== Simple Evaluation Example ===\n")
|
||||
|
||||
# Step 1: Create a model client
|
||||
model_client = OpenAIChatCompletionClient(
|
||||
model="gpt-4o-mini",
|
||||
# api_key is loaded from environment variable OPENAI_API_KEY
|
||||
)
|
||||
|
||||
# Step 2: Create evaluation tasks
|
||||
tasks = [
|
||||
EvalTask(
|
||||
name="Eiffel Tower Height",
|
||||
description="Answer the question about the Eiffel Tower height",
|
||||
input="What is the height of the Eiffel Tower?",
|
||||
),
|
||||
EvalTask(
|
||||
name="Lake Tanganyika Depth",
|
||||
description="Answer the question about Lake Tanganyika's depth",
|
||||
input="What is the depth of Lake Tanganyika?",
|
||||
),
|
||||
]
|
||||
|
||||
# Step 3: Create evaluation runners
|
||||
|
||||
# 3.1: Model runner (direct model access)
|
||||
model_runner = ModelEvalRunner(
|
||||
model_client=model_client,
|
||||
name="Direct Model Runner",
|
||||
description="Evaluates tasks by sending them directly to the model",
|
||||
)
|
||||
|
||||
# 3.2: Team runner (using a simple team with one agent)
|
||||
# Create an assistant agent for the team
|
||||
agent = AssistantAgent(
|
||||
name="research_agent",
|
||||
model_client=model_client,
|
||||
system_message="You are a helpful assistant"
|
||||
)
|
||||
|
||||
# Create a team with the agent
|
||||
team = RoundRobinGroupChat(participants=[agent], max_turns=3)
|
||||
|
||||
# Create a team runner with the team
|
||||
team_runner = TeamEvalRunner(
|
||||
team=team,
|
||||
name="Team Runner",
|
||||
description="Evaluates tasks using a team of agents"
|
||||
)
|
||||
|
||||
# Step 4: Create an LLM judge
|
||||
# We use the same model client for simplicity
|
||||
judge = LLMEvalJudge(
|
||||
model_client=model_client,
|
||||
name="Evaluation Judge",
|
||||
description="Judges the quality of responses"
|
||||
)
|
||||
|
||||
# Step 5: Define evaluation criteria
|
||||
criteria = [
|
||||
EvalJudgeCriteria(
|
||||
dimension="accuracy",
|
||||
prompt="Evaluate the factual accuracy of the response. Are all facts correct?",
|
||||
min_value=0,
|
||||
max_value=10,
|
||||
),
|
||||
EvalJudgeCriteria(
|
||||
dimension="completeness",
|
||||
prompt="Evaluate how thoroughly the response addresses the question. Does it provide all relevant information?",
|
||||
min_value=0,
|
||||
max_value=10,
|
||||
),
|
||||
]
|
||||
|
||||
# Step 6: Run evaluations and judge the results
|
||||
print("=== Running Evaluations ===\n")
|
||||
|
||||
# Run model evaluations (batch processing!)
|
||||
print("Running model evaluations...")
|
||||
print(f" Evaluating {len(tasks)} tasks in parallel...")
|
||||
model_task_results = await model_runner.run(tasks)
|
||||
|
||||
model_results = {}
|
||||
for task, model_result in zip(tasks, model_task_results):
|
||||
model_results[task.task_id] = model_result
|
||||
|
||||
# Print model response
|
||||
if model_result.status:
|
||||
messages = model_result.result.messages if model_result.result else []
|
||||
if messages:
|
||||
content = getattr(messages[0], 'content', 'No content')
|
||||
print(f" {task.name}: {str(content)[:100]}...")
|
||||
else:
|
||||
print(f" {task.name} error: {model_result.error}")
|
||||
|
||||
# Run team evaluations (batch processing!)
|
||||
print("\nRunning team evaluations...")
|
||||
print(f" Evaluating {len(tasks)} tasks with isolated teams...")
|
||||
team_task_results = await team_runner.run(tasks)
|
||||
|
||||
team_results = {}
|
||||
for task, team_result in zip(tasks, team_task_results):
|
||||
team_results[task.task_id] = team_result
|
||||
|
||||
# Print team response
|
||||
if team_result.status:
|
||||
messages = team_result.result.messages or []
|
||||
final_message = messages[-1] if messages else None
|
||||
if final_message and hasattr(final_message, 'content'):
|
||||
print(f" {task.name}: {final_message.content[:100]}...")
|
||||
else:
|
||||
print(f" {task.name}: No response from team")
|
||||
else:
|
||||
print(f" {task.name} error: {team_result.error}")
|
||||
|
||||
# Judge the results
|
||||
print("\n=== Judging Results ===\n")
|
||||
|
||||
# Judge model results
|
||||
print("Judging model results...")
|
||||
model_scores = {}
|
||||
for task in tasks:
|
||||
if task.task_id in model_results and model_results[task.task_id].status:
|
||||
print(f" Judging task: {task.name}")
|
||||
model_score = await judge.judge(task, model_results[task.task_id], criteria)
|
||||
model_scores[task.task_id] = model_score
|
||||
|
||||
# Print scores
|
||||
print(f" Overall score: {model_score.overall_score}")
|
||||
for dimension_score in model_score.dimension_scores:
|
||||
print(f" {dimension_score.dimension}: {dimension_score.score} - {dimension_score.reason[:50]}...")
|
||||
|
||||
# Judge team results
|
||||
print("\nJudging team results...")
|
||||
team_scores = {}
|
||||
for task in tasks:
|
||||
if task.task_id in team_results and team_results[task.task_id].status:
|
||||
print(f" Judging task: {task.name}")
|
||||
team_score = await judge.judge(task, team_results[task.task_id], criteria)
|
||||
team_scores[task.task_id] = team_score
|
||||
|
||||
# Print scores
|
||||
print(f" Overall score: {team_score.overall_score}")
|
||||
for dimension_score in team_score.dimension_scores:
|
||||
print(f" {dimension_score.dimension}: {dimension_score.score} - {dimension_score.reason[:50]}...")
|
||||
|
||||
# Step 7: Test serialization and deserialization
|
||||
print("\n=== Testing Serialization and Deserialization ===\n")
|
||||
|
||||
# Serialize model runner
|
||||
model_runner_config = model_runner.dump_component()
|
||||
print(f"Serialized model runner config created successfully")
|
||||
|
||||
# Deserialize model runner
|
||||
deserialized_model_runner = ModelEvalRunner.load_component(model_runner_config)
|
||||
print(f"Deserialized model runner: {deserialized_model_runner.name}")
|
||||
|
||||
# Serialize judge
|
||||
judge_config = judge.dump_component()
|
||||
print(f"Serialized judge config created successfully")
|
||||
|
||||
# Deserialize judge
|
||||
deserialized_judge = LLMEvalJudge.load_component(judge_config)
|
||||
print(f"Deserialized judge: {deserialized_judge.name}")
|
||||
|
||||
# Close the model client
|
||||
await model_client.close()
|
||||
|
||||
return {
|
||||
"model_results": model_results,
|
||||
"team_results": team_results,
|
||||
"model_scores": model_scores,
|
||||
"team_scores": team_scores,
|
||||
}
|
||||
|
||||
|
||||
async def run_orchestrated_evaluation():
|
||||
"""Run a comprehensive evaluation using the EvalOrchestrator."""
|
||||
|
||||
print("\n=== Orchestrated Evaluation Example ===\n")
|
||||
|
||||
# Step 1: Create a model client
|
||||
model_client = OpenAIChatCompletionClient(
|
||||
model="gpt-4o-mini",
|
||||
# api_key is loaded from environment variable OPENAI_API_KEY
|
||||
)
|
||||
|
||||
# Step 2: Create an orchestrator (without DB for this example)
|
||||
orchestrator = EvalOrchestrator()
|
||||
|
||||
# Step 3: Create and register tasks
|
||||
task_ids = []
|
||||
tasks = [
|
||||
EvalTask(
|
||||
name="Eiffel Tower Height",
|
||||
description="Answer the question about the Eiffel Tower height",
|
||||
input="What is the height of the Eiffel Tower?",
|
||||
),
|
||||
EvalTask(
|
||||
name="Lake Tanganyika Depth",
|
||||
description="Answer the question about Lake Tanganyika's depth",
|
||||
input="What is the depth of Lake Tanganyika?",
|
||||
),
|
||||
]
|
||||
|
||||
print("Creating tasks...")
|
||||
for task in tasks:
|
||||
task_id = await orchestrator.create_task(task)
|
||||
task_ids.append(task_id)
|
||||
print(f" Created task: {task.name} (ID: {task_id})")
|
||||
|
||||
# Step 4: Create and register criteria
|
||||
criteria_ids = []
|
||||
criteria = [
|
||||
EvalJudgeCriteria(
|
||||
dimension="accuracy",
|
||||
prompt="Evaluate the factual accuracy of the response. Are all facts correct?",
|
||||
min_value=0,
|
||||
max_value=10,
|
||||
),
|
||||
EvalJudgeCriteria(
|
||||
dimension="completeness",
|
||||
prompt="Evaluate how thoroughly the response addresses the question. Does it provide all relevant information?",
|
||||
min_value=0,
|
||||
max_value=10,
|
||||
),
|
||||
]
|
||||
|
||||
print("\nCreating criteria...")
|
||||
for criterion in criteria:
|
||||
criterion_id = await orchestrator.create_criteria(criterion)
|
||||
criteria_ids.append(criterion_id)
|
||||
print(f" Created criteria: {criterion.dimension} (ID: {criterion_id})")
|
||||
|
||||
# Step 5: Create runners
|
||||
# Model runner
|
||||
model_runner = ModelEvalRunner(
|
||||
model_client=model_client,
|
||||
name="Direct Model Runner",
|
||||
description="Evaluates tasks by sending them directly to the model",
|
||||
)
|
||||
|
||||
# Team runner
|
||||
agent = AssistantAgent(
|
||||
name="research_agent",
|
||||
model_client=model_client,
|
||||
system_message="You are a helpful assistant"
|
||||
)
|
||||
team = RoundRobinGroupChat(participants=[agent], max_turns=3)
|
||||
team_runner = TeamEvalRunner(
|
||||
team=team,
|
||||
name="Team Runner",
|
||||
description="Evaluates tasks using a team of agents"
|
||||
)
|
||||
|
||||
# Step 6: Create a judge
|
||||
judge = LLMEvalJudge(
|
||||
model_client=model_client,
|
||||
name="Evaluation Judge",
|
||||
description="Judges the quality of responses"
|
||||
)
|
||||
|
||||
# Step 7: Create evaluation runs
|
||||
model_run_ids = []
|
||||
team_run_ids = []
|
||||
|
||||
print("\nCreating evaluation runs...")
|
||||
|
||||
# Create model runs
|
||||
for i, task_id in enumerate(task_ids):
|
||||
run_id = await orchestrator.create_run(
|
||||
task=task_id,
|
||||
runner=model_runner,
|
||||
judge=judge,
|
||||
criteria=criteria_ids,
|
||||
name=f"Model Run - Task {i+1}"
|
||||
)
|
||||
model_run_ids.append(run_id)
|
||||
print(f" Created model run: {run_id}")
|
||||
|
||||
# Create team runs
|
||||
for i, task_id in enumerate(task_ids):
|
||||
run_id = await orchestrator.create_run(
|
||||
task=task_id,
|
||||
runner=team_runner,
|
||||
judge=judge,
|
||||
criteria=criteria_ids,
|
||||
name=f"Team Run - Task {i+1}"
|
||||
)
|
||||
team_run_ids.append(run_id)
|
||||
print(f" Created team run: {run_id}")
|
||||
|
||||
# Step 8: Execute the runs
|
||||
print("\n=== Starting Evaluation Runs ===\n")
|
||||
|
||||
# Start model runs
|
||||
print("Starting model runs...")
|
||||
for run_id in model_run_ids:
|
||||
await orchestrator.start_run(run_id)
|
||||
print(f" Started run: {run_id}")
|
||||
|
||||
# Start team runs
|
||||
print("\nStarting team runs...")
|
||||
for run_id in team_run_ids:
|
||||
await orchestrator.start_run(run_id)
|
||||
print(f" Started run: {run_id}")
|
||||
|
||||
# Step 9: Wait for runs to complete
|
||||
print("\n=== Waiting for Runs to Complete ===\n")
|
||||
|
||||
all_runs = model_run_ids + team_run_ids
|
||||
completed = {run_id: False for run_id in all_runs}
|
||||
|
||||
while not all(completed.values()):
|
||||
for run_id in all_runs:
|
||||
if not completed[run_id]:
|
||||
status = await orchestrator.get_run_status(run_id)
|
||||
if status in [EvalRunStatus.COMPLETED, EvalRunStatus.FAILED, EvalRunStatus.CANCELED]:
|
||||
completed[run_id] = True
|
||||
print(f"Run {run_id} completed with status: {status}")
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Step 10: Get results
|
||||
print("\n=== Evaluation Results ===\n")
|
||||
|
||||
# Model results
|
||||
print("Model run results:")
|
||||
for i, run_id in enumerate(model_run_ids):
|
||||
run_result = await orchestrator.get_run_result(run_id)
|
||||
score_result = await orchestrator.get_run_score(run_id)
|
||||
|
||||
print(f"\nModel Run {i+1} (ID: {run_id}):")
|
||||
if run_result and run_result.status:
|
||||
messages = run_result.result.messages if run_result.result else []
|
||||
if messages:
|
||||
content = getattr(messages[0], 'content', 'No content')
|
||||
print(f" Response: {str(content)[:100]}...")
|
||||
|
||||
if score_result:
|
||||
print(f" Overall score: {score_result.overall_score}")
|
||||
for dimension_score in score_result.dimension_scores:
|
||||
print(f" {dimension_score.dimension}: {dimension_score.score}")
|
||||
print(f" Reason: {dimension_score.reason[:100]}...")
|
||||
else:
|
||||
print(f" Error: {run_result.error if run_result else 'No result'}")
|
||||
|
||||
# Team results
|
||||
print("\nTeam run results:")
|
||||
for i, run_id in enumerate(team_run_ids):
|
||||
run_result = await orchestrator.get_run_result(run_id)
|
||||
score_result = await orchestrator.get_run_score(run_id)
|
||||
|
||||
print(f"\nTeam Run {i+1} (ID: {run_id}):")
|
||||
if run_result and run_result.status:
|
||||
messages = run_result.result.messages or []
|
||||
final_message = messages[-1] if messages else None
|
||||
if final_message and hasattr(final_message, 'content'):
|
||||
print(f" Response: {final_message.content[:100]}...")
|
||||
|
||||
if score_result:
|
||||
print(f" Overall score: {score_result.overall_score}")
|
||||
for dimension_score in score_result.dimension_scores:
|
||||
print(f" {dimension_score.dimension}: {dimension_score.score}")
|
||||
print(f" Reason: {dimension_score.reason[:100]}...")
|
||||
else:
|
||||
print(f" Error: {run_result.error if run_result else 'No result'}")
|
||||
|
||||
# Step 11: Demonstrate tabulated results
|
||||
print("\n=== Tabulated Results ===\n")
|
||||
|
||||
all_run_ids = model_run_ids + team_run_ids
|
||||
tabulated_results = await orchestrator.tabulate_results(all_run_ids, include_reasons=True)
|
||||
|
||||
print(f"Dimensions: {tabulated_results['dimensions']}")
|
||||
print(f"Number of runs: {len(tabulated_results['runs'])}")
|
||||
|
||||
for run_entry in tabulated_results['runs']:
|
||||
print(f"\nRun: {run_entry['name']} ({run_entry['runner_type']})")
|
||||
print(f" Task: {run_entry['task_name']}")
|
||||
print(f" Overall Score: {run_entry['overall_score']}")
|
||||
print(f" Dimension Scores: {run_entry['scores']}")
|
||||
|
||||
# Close the model client
|
||||
await model_client.close()
|
||||
|
||||
return {
|
||||
"task_ids": task_ids,
|
||||
"criteria_ids": criteria_ids,
|
||||
"model_run_ids": model_run_ids,
|
||||
"team_run_ids": team_run_ids,
|
||||
"tabulated_results": tabulated_results,
|
||||
}
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all evaluation examples."""
|
||||
print("🚀 AutoGen Studio Evaluation Examples")
|
||||
print("=" * 50)
|
||||
|
||||
try:
|
||||
# Run simple evaluation
|
||||
simple_results = await run_simple_evaluation()
|
||||
print(f"\n✅ Simple evaluation completed with {len(simple_results['model_results'])} model results")
|
||||
|
||||
# Run orchestrated evaluation
|
||||
orchestrated_results = await run_orchestrated_evaluation()
|
||||
print(f"\n✅ Orchestrated evaluation completed with {len(orchestrated_results['model_run_ids'])} model runs and {len(orchestrated_results['team_run_ids'])} team runs")
|
||||
|
||||
print("\n🎉 All evaluation examples completed successfully!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error running evaluations: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from autogen_core import CancellationToken, ComponentBase
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...datamodel.eval import EvalJudgeCriteria, EvalRunResult, EvalScore, EvalTask
|
||||
|
||||
|
||||
class BaseEvalJudgeConfig(BaseModel):
|
||||
"""Base configuration for evaluation judges."""
|
||||
|
||||
name: str = "Base Judge"
|
||||
description: str = ""
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class BaseEvalJudge(ABC, ComponentBase[BaseEvalJudgeConfig]):
|
||||
"""Abstract base class for evaluation judges."""
|
||||
|
||||
component_type = "eval_judge"
|
||||
|
||||
def __init__(self, name: str = "Base Judge", description: str = "", metadata: Optional[Dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.metadata = metadata or {}
|
||||
|
||||
@abstractmethod
|
||||
async def judge(
|
||||
self,
|
||||
task: EvalTask,
|
||||
result: EvalRunResult,
|
||||
criteria: List[EvalJudgeCriteria],
|
||||
cancellation_token: Optional[CancellationToken] = None,
|
||||
) -> EvalScore:
|
||||
"""Judge the result of an evaluation run."""
|
||||
pass
|
||||
|
||||
def _to_config(self) -> BaseEvalJudgeConfig:
|
||||
"""Convert the judge configuration to a configuration object for serialization."""
|
||||
return BaseEvalJudgeConfig(name=self.name, description=self.description, metadata=self.metadata)
|
||||
|
||||
|
||||
# Import specific judge implementations
|
||||
from ._llm import LLMEvalJudge, LLMEvalJudgeConfig
|
||||
|
||||
__all__ = ["BaseEvalJudge", "BaseEvalJudgeConfig", "LLMEvalJudge", "LLMEvalJudgeConfig"]
|
||||
@@ -1,48 +1,13 @@
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from autogen_core import CancellationToken, Component, ComponentBase
|
||||
from autogen_core import CancellationToken, Component
|
||||
from autogen_core.models import ChatCompletionClient, UserMessage
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import Self
|
||||
|
||||
from ..datamodel.eval import EvalDimensionScore, EvalJudgeCriteria, EvalRunResult, EvalScore, EvalTask
|
||||
|
||||
|
||||
class BaseEvalJudgeConfig(BaseModel):
|
||||
"""Base configuration for evaluation judges."""
|
||||
|
||||
name: str = "Base Judge"
|
||||
description: str = ""
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class BaseEvalJudge(ABC, ComponentBase[BaseEvalJudgeConfig]):
|
||||
"""Abstract base class for evaluation judges."""
|
||||
|
||||
component_type = "eval_judge"
|
||||
|
||||
def __init__(self, name: str = "Base Judge", description: str = "", metadata: Optional[Dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.metadata = metadata or {}
|
||||
|
||||
@abstractmethod
|
||||
async def judge(
|
||||
self,
|
||||
task: EvalTask,
|
||||
result: EvalRunResult,
|
||||
criteria: List[EvalJudgeCriteria],
|
||||
cancellation_token: Optional[CancellationToken] = None,
|
||||
) -> EvalScore:
|
||||
"""Judge the result of an evaluation run."""
|
||||
pass
|
||||
|
||||
def _to_config(self) -> BaseEvalJudgeConfig:
|
||||
"""Convert the judge configuration to a configuration object for serialization."""
|
||||
return BaseEvalJudgeConfig(name=self.name, description=self.description, metadata=self.metadata)
|
||||
from ...datamodel.eval import EvalDimensionScore, EvalJudgeCriteria, EvalRunResult, EvalScore, EvalTask
|
||||
from . import BaseEvalJudge, BaseEvalJudgeConfig
|
||||
|
||||
|
||||
class LLMEvalJudgeConfig(BaseEvalJudgeConfig):
|
||||
@@ -56,7 +21,7 @@ class LLMEvalJudge(BaseEvalJudge, Component[LLMEvalJudgeConfig]):
|
||||
|
||||
component_config_schema = LLMEvalJudgeConfig
|
||||
component_type = "eval_judge"
|
||||
component_provider_override = "autogenstudio.eval.judges.LLMEvalJudge"
|
||||
component_provider_override = "autogenstudio.eval.judges._llm.LLMEvalJudge"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -208,60 +173,4 @@ class LLMEvalJudge(BaseEvalJudge, Component[LLMEvalJudgeConfig]):
|
||||
model_client = ChatCompletionClient.load_component(config.model_client)
|
||||
return cls(
|
||||
model_client=model_client, name=config.name, description=config.description, metadata=config.metadata
|
||||
)
|
||||
|
||||
|
||||
# # Usage example
|
||||
# async def example_usage():
|
||||
# # Create a model client
|
||||
# from autogen_ext.models import OpenAIChatCompletionClient
|
||||
|
||||
# model_client = OpenAIChatCompletionClient(
|
||||
# model="gpt-4",
|
||||
# api_key="your-api-key"
|
||||
# )
|
||||
|
||||
# # Create a judge
|
||||
# llm_judge = LLMEvalJudge(model_client=model_client)
|
||||
|
||||
# # Serialize the judge to a ComponentModel
|
||||
# judge_config = llm_judge.dump_component()
|
||||
# print(f"Serialized judge: {judge_config}")
|
||||
|
||||
# # Deserialize back to a LLMEvalJudge
|
||||
# deserialized_judge = LLMEvalJudge.load_component(judge_config)
|
||||
|
||||
# # Create criteria for evaluation
|
||||
# criteria = [
|
||||
# EvalJudgeCriteria(
|
||||
# dimension="relevance",
|
||||
# prompt="Evaluate how relevant the response is to the query.",
|
||||
# min_value=0,
|
||||
# max_value=10
|
||||
# ),
|
||||
# EvalJudgeCriteria(
|
||||
# dimension="accuracy",
|
||||
# prompt="Evaluate the factual accuracy of the response.",
|
||||
# min_value=0,
|
||||
# max_value=10
|
||||
# )
|
||||
# ]
|
||||
|
||||
# # Create a mock task and result
|
||||
# task = EvalTask(
|
||||
# id="task-123",
|
||||
# name="Sample Task",
|
||||
# description="A sample task for evaluation",
|
||||
# input="What is the capital of France?"
|
||||
# )
|
||||
|
||||
# result = EvalRunResult(
|
||||
# status=True,
|
||||
# result={
|
||||
# "messages": [{"content": "The capital of France is Paris.", "source": "model"}]
|
||||
# }
|
||||
# )
|
||||
|
||||
# # Run the evaluation
|
||||
# score = await deserialized_judge.judge(task, result, criteria)
|
||||
# print(f"Evaluation score: {score}")
|
||||
)
|
||||
@@ -1,201 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional, Sequence, Type, Union
|
||||
|
||||
from autogen_agentchat.base import TaskResult, Team
|
||||
from autogen_agentchat.messages import ChatMessage, MultiModalMessage, TextMessage
|
||||
from autogen_core import CancellationToken, Component, ComponentBase, ComponentModel, Image
|
||||
from autogen_core.models import ChatCompletionClient, UserMessage
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import Self
|
||||
|
||||
from ..datamodel.eval import EvalRunResult, EvalTask
|
||||
|
||||
|
||||
class BaseEvalRunnerConfig(BaseModel):
|
||||
"""Base configuration for evaluation runners."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class BaseEvalRunner(ABC, ComponentBase[BaseEvalRunnerConfig]):
|
||||
"""Base class for evaluation runners that defines the interface for running evaluations.
|
||||
|
||||
This class provides the core interface that all evaluation runners must implement.
|
||||
Subclasses should implement the run method to define how a specific evaluation is executed.
|
||||
"""
|
||||
|
||||
component_type = "eval_runner"
|
||||
|
||||
def __init__(self, name: str, description: str = "", metadata: Optional[Dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.metadata = metadata or {}
|
||||
|
||||
@abstractmethod
|
||||
async def run(self, task: EvalTask, cancellation_token: Optional[CancellationToken] = None) -> EvalRunResult:
|
||||
"""Run the evaluation on the provided task and return a result.
|
||||
|
||||
Args:
|
||||
task: The task to evaluate
|
||||
cancellation_token: Optional token to cancel the evaluation
|
||||
|
||||
Returns:
|
||||
EvaluationResult: The result of the evaluation
|
||||
"""
|
||||
pass
|
||||
|
||||
def _to_config(self) -> BaseEvalRunnerConfig:
|
||||
"""Convert the runner configuration to a configuration object for serialization."""
|
||||
return BaseEvalRunnerConfig(name=self.name, description=self.description, metadata=self.metadata)
|
||||
|
||||
|
||||
class ModelEvalRunnerConfig(BaseEvalRunnerConfig):
|
||||
"""Configuration for ModelEvalRunner."""
|
||||
|
||||
model_client: ComponentModel
|
||||
|
||||
|
||||
class ModelEvalRunner(BaseEvalRunner, Component[ModelEvalRunnerConfig]):
|
||||
"""Evaluation runner that uses a single LLM to process tasks.
|
||||
|
||||
This runner sends the task directly to a model client and returns the response.
|
||||
"""
|
||||
|
||||
component_config_schema = ModelEvalRunnerConfig
|
||||
component_type = "eval_runner"
|
||||
component_provider_override = "autogenstudio.eval.runners.ModelEvalRunner"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_client: ChatCompletionClient,
|
||||
name: str = "Model Runner",
|
||||
description: str = "Evaluates tasks using a single LLM",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
super().__init__(name, description, metadata)
|
||||
self.model_client = model_client
|
||||
|
||||
async def run(self, task: EvalTask, cancellation_token: Optional[CancellationToken] = None) -> EvalRunResult:
|
||||
"""Run the task with the model client and return the result."""
|
||||
# Create initial result object
|
||||
result = EvalRunResult()
|
||||
|
||||
try:
|
||||
model_input = []
|
||||
if isinstance(task.input, str):
|
||||
text_message = UserMessage(content=task.input, source="user")
|
||||
model_input.append(text_message)
|
||||
elif isinstance(task.input, list):
|
||||
message_content = [x for x in task.input]
|
||||
model_input.append(UserMessage(content=message_content, source="user"))
|
||||
# Run with the model
|
||||
model_result = await self.model_client.create(messages=model_input, cancellation_token=cancellation_token)
|
||||
|
||||
model_response = model_result.content if isinstance(model_result, str) else model_result.model_dump()
|
||||
|
||||
task_result = TaskResult(
|
||||
messages=[TextMessage(content=str(model_response), source="model")],
|
||||
)
|
||||
result = EvalRunResult(result=task_result, status=True, start_time=datetime.now(), end_time=datetime.now())
|
||||
|
||||
except Exception as e:
|
||||
result = EvalRunResult(status=False, error=str(e), end_time=datetime.now())
|
||||
|
||||
return result
|
||||
|
||||
def _to_config(self) -> ModelEvalRunnerConfig:
|
||||
"""Convert to configuration object including model client configuration."""
|
||||
base_config = super()._to_config()
|
||||
return ModelEvalRunnerConfig(
|
||||
name=base_config.name,
|
||||
description=base_config.description,
|
||||
metadata=base_config.metadata,
|
||||
model_client=self.model_client.dump_component(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_config(cls, config: ModelEvalRunnerConfig) -> Self:
|
||||
"""Create from configuration object with serialized model client."""
|
||||
model_client = ChatCompletionClient.load_component(config.model_client)
|
||||
return cls(
|
||||
name=config.name,
|
||||
description=config.description,
|
||||
metadata=config.metadata,
|
||||
model_client=model_client,
|
||||
)
|
||||
|
||||
|
||||
class TeamEvalRunnerConfig(BaseEvalRunnerConfig):
|
||||
"""Configuration for TeamEvalRunner."""
|
||||
|
||||
team: ComponentModel
|
||||
|
||||
|
||||
class TeamEvalRunner(BaseEvalRunner, Component[TeamEvalRunnerConfig]):
|
||||
"""Evaluation runner that uses a team of agents to process tasks.
|
||||
|
||||
This runner creates and runs a team based on a team configuration.
|
||||
"""
|
||||
|
||||
component_config_schema = TeamEvalRunnerConfig
|
||||
component_type = "eval_runner"
|
||||
component_provider_override = "autogenstudio.eval.runners.TeamEvalRunner"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
team: Union[Team, ComponentModel],
|
||||
name: str = "Team Runner",
|
||||
description: str = "Evaluates tasks using a team of agents",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
super().__init__(name, description, metadata)
|
||||
self._team = team if isinstance(team, Team) else Team.load_component(team)
|
||||
|
||||
async def run(self, task: EvalTask, cancellation_token: Optional[CancellationToken] = None) -> EvalRunResult:
|
||||
"""Run the task with the team and return the result."""
|
||||
# Create initial result object
|
||||
result = EvalRunResult()
|
||||
|
||||
try:
|
||||
team_task: Sequence[ChatMessage] = []
|
||||
if isinstance(task.input, str):
|
||||
team_task.append(TextMessage(content=task.input, source="user"))
|
||||
if isinstance(task.input, list):
|
||||
for message in task.input:
|
||||
if isinstance(message, str):
|
||||
team_task.append(TextMessage(content=message, source="user"))
|
||||
elif isinstance(message, Image):
|
||||
team_task.append(MultiModalMessage(source="user", content=[message]))
|
||||
|
||||
# Run task with team
|
||||
team_result = await self._team.run(task=team_task, cancellation_token=cancellation_token)
|
||||
|
||||
result = EvalRunResult(result=team_result, status=True, start_time=datetime.now(), end_time=datetime.now())
|
||||
|
||||
except Exception as e:
|
||||
result = EvalRunResult(status=False, error=str(e), end_time=datetime.now())
|
||||
|
||||
return result
|
||||
|
||||
def _to_config(self) -> TeamEvalRunnerConfig:
|
||||
"""Convert to configuration object including team configuration."""
|
||||
base_config = super()._to_config()
|
||||
return TeamEvalRunnerConfig(
|
||||
name=base_config.name,
|
||||
description=base_config.description,
|
||||
metadata=base_config.metadata,
|
||||
team=self._team.dump_component(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_config(cls, config: TeamEvalRunnerConfig) -> Self:
|
||||
"""Create from configuration object with serialized team configuration."""
|
||||
return cls(
|
||||
team=Team.load_component(config.team),
|
||||
name=config.name,
|
||||
description=config.description,
|
||||
metadata=config.metadata,
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from autogen_core import CancellationToken, ComponentBase
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...datamodel.eval import EvalRunResult, EvalTask
|
||||
|
||||
|
||||
class BaseEvalRunnerConfig(BaseModel):
|
||||
"""Base configuration for evaluation runners."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class BaseEvalRunner(ABC, ComponentBase[BaseEvalRunnerConfig]):
|
||||
"""Base class for evaluation runners that defines the interface for running evaluations.
|
||||
|
||||
This class provides the core interface that all evaluation runners must implement.
|
||||
Subclasses should implement the run method to define how a specific evaluation is executed.
|
||||
"""
|
||||
|
||||
component_type = "eval_runner"
|
||||
|
||||
def __init__(self, name: str, description: str = "", metadata: Optional[Dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.metadata = metadata or {}
|
||||
|
||||
@abstractmethod
|
||||
async def run(self, tasks: list[EvalTask], cancellation_token: Optional[CancellationToken] = None) -> list[EvalRunResult]:
|
||||
"""Run the evaluation on the provided tasks and return results.
|
||||
|
||||
Args:
|
||||
tasks: The list of tasks to evaluate
|
||||
cancellation_token: Optional token to cancel the evaluation
|
||||
|
||||
Returns:
|
||||
List[EvalRunResult]: The results of the evaluations, one per task
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def _to_config(self) -> BaseEvalRunnerConfig:
|
||||
"""Convert the runner configuration to a configuration object for serialization."""
|
||||
return BaseEvalRunnerConfig(name=self.name, description=self.description, metadata=self.metadata)
|
||||
|
||||
|
||||
# Import specific runner implementations
|
||||
from ._model import ModelEvalRunner, ModelEvalRunnerConfig
|
||||
from ._team import TeamEvalRunner, TeamEvalRunnerConfig
|
||||
|
||||
__all__ = ["BaseEvalRunner", "BaseEvalRunnerConfig", "ModelEvalRunner", "ModelEvalRunnerConfig", "TeamEvalRunner", "TeamEvalRunnerConfig"]
|
||||
@@ -0,0 +1,118 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from autogen_agentchat.base import TaskResult
|
||||
from autogen_agentchat.messages import TextMessage
|
||||
from autogen_core import CancellationToken, Component, ComponentModel
|
||||
from autogen_core.models import ChatCompletionClient, UserMessage
|
||||
from typing_extensions import Self
|
||||
|
||||
from ...datamodel.eval import EvalRunResult, EvalTask
|
||||
from . import BaseEvalRunner, BaseEvalRunnerConfig
|
||||
|
||||
|
||||
class ModelEvalRunnerConfig(BaseEvalRunnerConfig):
|
||||
"""Configuration for ModelEvalRunner."""
|
||||
|
||||
model_client: ComponentModel
|
||||
|
||||
|
||||
class ModelEvalRunner(BaseEvalRunner, Component[ModelEvalRunnerConfig]):
|
||||
"""Evaluation runner that uses a single LLM to process tasks.
|
||||
|
||||
This runner sends the task directly to a model client and returns the response.
|
||||
"""
|
||||
|
||||
component_config_schema = ModelEvalRunnerConfig
|
||||
component_type = "eval_runner"
|
||||
component_provider_override = "autogenstudio.eval.runners._model.ModelEvalRunner"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_client: ChatCompletionClient,
|
||||
name: str = "Model Runner",
|
||||
description: str = "Evaluates tasks using a single LLM",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
super().__init__(name, description, metadata)
|
||||
self.model_client = model_client
|
||||
|
||||
async def run(self, tasks: list[EvalTask], cancellation_token: Optional[CancellationToken] = None) -> list[EvalRunResult]:
|
||||
"""Run the tasks with the model client and return the results."""
|
||||
if not tasks:
|
||||
return []
|
||||
|
||||
# Process tasks in parallel with concurrency control
|
||||
max_concurrent = min(10, len(tasks)) # Limit concurrent requests
|
||||
semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def run_single_task(task: EvalTask) -> EvalRunResult:
|
||||
"""Run a single task with concurrency control."""
|
||||
async with semaphore:
|
||||
return await self._run_single_task(task, cancellation_token)
|
||||
|
||||
# Execute all tasks in parallel
|
||||
results = await asyncio.gather(
|
||||
*[run_single_task(task) for task in tasks],
|
||||
return_exceptions=True
|
||||
)
|
||||
|
||||
# Convert exceptions to failed EvalRunResults
|
||||
processed_results = []
|
||||
for result in results:
|
||||
if isinstance(result, Exception):
|
||||
processed_results.append(EvalRunResult(
|
||||
status=False,
|
||||
error=str(result),
|
||||
end_time=datetime.now()
|
||||
))
|
||||
else:
|
||||
processed_results.append(result)
|
||||
|
||||
return processed_results
|
||||
|
||||
async def _run_single_task(self, task: EvalTask, cancellation_token: Optional[CancellationToken] = None) -> EvalRunResult:
|
||||
"""Run a single task with the model client."""
|
||||
try:
|
||||
model_input = []
|
||||
if isinstance(task.input, str):
|
||||
text_message = UserMessage(content=task.input, source="user")
|
||||
model_input.append(text_message)
|
||||
elif isinstance(task.input, list):
|
||||
message_content = [x for x in task.input]
|
||||
model_input.append(UserMessage(content=message_content, source="user"))
|
||||
|
||||
# Run with the model
|
||||
model_result = await self.model_client.create(messages=model_input, cancellation_token=cancellation_token)
|
||||
|
||||
model_response = model_result.content if isinstance(model_result, str) else model_result.model_dump()
|
||||
|
||||
task_result = TaskResult(
|
||||
messages=[TextMessage(content=str(model_response), source="model")],
|
||||
)
|
||||
return EvalRunResult(result=task_result, status=True, start_time=datetime.now(), end_time=datetime.now())
|
||||
|
||||
except Exception as e:
|
||||
return EvalRunResult(status=False, error=str(e), end_time=datetime.now())
|
||||
|
||||
def _to_config(self) -> ModelEvalRunnerConfig:
|
||||
"""Convert to configuration object including model client configuration."""
|
||||
base_config = super()._to_config()
|
||||
return ModelEvalRunnerConfig(
|
||||
name=base_config.name,
|
||||
description=base_config.description,
|
||||
metadata=base_config.metadata,
|
||||
model_client=self.model_client.dump_component(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_config(cls, config: ModelEvalRunnerConfig) -> Self:
|
||||
"""Create from configuration object with serialized model client."""
|
||||
model_client = ChatCompletionClient.load_component(config.model_client)
|
||||
return cls(
|
||||
name=config.name,
|
||||
description=config.description,
|
||||
metadata=config.metadata,
|
||||
model_client=model_client,
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional, Sequence, Union
|
||||
|
||||
from autogen_agentchat.base import Team
|
||||
from autogen_agentchat.messages import ChatMessage, MultiModalMessage, TextMessage
|
||||
from autogen_core import CancellationToken, Component, ComponentModel, Image
|
||||
from typing_extensions import Self
|
||||
|
||||
from ...datamodel.eval import EvalRunResult, EvalTask
|
||||
from . import BaseEvalRunner, BaseEvalRunnerConfig
|
||||
|
||||
|
||||
class TeamEvalRunnerConfig(BaseEvalRunnerConfig):
|
||||
"""Configuration for TeamEvalRunner."""
|
||||
|
||||
team: ComponentModel
|
||||
|
||||
|
||||
class TeamEvalRunner(BaseEvalRunner, Component[TeamEvalRunnerConfig]):
|
||||
"""Evaluation runner that uses a team of agents to process tasks.
|
||||
|
||||
This runner creates and runs a team based on a team configuration.
|
||||
"""
|
||||
|
||||
component_config_schema = TeamEvalRunnerConfig
|
||||
component_type = "eval_runner"
|
||||
component_provider_override = "autogenstudio.eval.runners._team.TeamEvalRunner"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
team: Union[Team, ComponentModel],
|
||||
name: str = "Team Runner",
|
||||
description: str = "Evaluates tasks using a team of agents",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
super().__init__(name, description, metadata)
|
||||
self._team = team if isinstance(team, Team) else Team.load_component(team)
|
||||
|
||||
async def run(self, tasks: list[EvalTask], cancellation_token: Optional[CancellationToken] = None) -> list[EvalRunResult]:
|
||||
"""Run the tasks with isolated team instances and return the results."""
|
||||
if not tasks:
|
||||
return []
|
||||
|
||||
# Each task gets a fresh team instance to maintain isolation
|
||||
async def run_single_task(task: EvalTask) -> EvalRunResult:
|
||||
"""Run a single task with a fresh team instance."""
|
||||
try:
|
||||
# Create a fresh team instance from the stored configuration
|
||||
fresh_team = Team.load_component(self._team.dump_component())
|
||||
|
||||
# Convert task input to team format
|
||||
team_task: Sequence[ChatMessage] = []
|
||||
if isinstance(task.input, str):
|
||||
team_task.append(TextMessage(content=task.input, source="user"))
|
||||
elif isinstance(task.input, list):
|
||||
for message in task.input:
|
||||
if isinstance(message, str):
|
||||
team_task.append(TextMessage(content=message, source="user"))
|
||||
elif isinstance(message, Image):
|
||||
team_task.append(MultiModalMessage(source="user", content=[message]))
|
||||
|
||||
# Run task with fresh team
|
||||
team_result = await fresh_team.run(task=team_task, cancellation_token=cancellation_token)
|
||||
|
||||
return EvalRunResult(result=team_result, status=True, start_time=datetime.now(), end_time=datetime.now())
|
||||
|
||||
except Exception as e:
|
||||
return EvalRunResult(status=False, error=str(e), end_time=datetime.now())
|
||||
|
||||
# Run all tasks in parallel with isolated team instances
|
||||
results = await asyncio.gather(
|
||||
*[run_single_task(task) for task in tasks],
|
||||
return_exceptions=True
|
||||
)
|
||||
|
||||
# Convert exceptions to failed EvalRunResults
|
||||
processed_results = []
|
||||
for result in results:
|
||||
if isinstance(result, Exception):
|
||||
processed_results.append(EvalRunResult(
|
||||
status=False,
|
||||
error=str(result),
|
||||
end_time=datetime.now()
|
||||
))
|
||||
else:
|
||||
processed_results.append(result)
|
||||
|
||||
return processed_results
|
||||
|
||||
def _to_config(self) -> TeamEvalRunnerConfig:
|
||||
"""Convert to configuration object including team configuration."""
|
||||
base_config = super()._to_config()
|
||||
return TeamEvalRunnerConfig(
|
||||
name=base_config.name,
|
||||
description=base_config.description,
|
||||
metadata=base_config.metadata,
|
||||
team=self._team.dump_component(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_config(cls, config: TeamEvalRunnerConfig) -> Self:
|
||||
"""Create from configuration object with serialized team configuration."""
|
||||
return cls(
|
||||
team=Team.load_component(config.team),
|
||||
name=config.name,
|
||||
description=config.description,
|
||||
metadata=config.metadata,
|
||||
)
|
||||
319
python/packages/autogen-studio/autogenstudio/eval/test_eval.py
Normal file
319
python/packages/autogen-studio/autogenstudio/eval/test_eval.py
Normal file
@@ -0,0 +1,319 @@
|
||||
"""
|
||||
Comprehensive test suite for the AutoGen Studio evaluation system.
|
||||
|
||||
This file provides complete test coverage for the eval system using mocks,
|
||||
eliminating the need for API keys or external dependencies.
|
||||
|
||||
Features tested:
|
||||
- ModelEvalRunner: Single LLM evaluation
|
||||
- LLMEvalJudge: LLM-based scoring with multiple criteria
|
||||
- EvalOrchestrator: Task, criteria, and run management
|
||||
- Component creation and basic operations
|
||||
|
||||
Usage:
|
||||
# Run with pytest (recommended)
|
||||
pytest autogenstudio/eval/test_eval.py -v
|
||||
|
||||
# Run direct test
|
||||
python -c "import asyncio; from autogenstudio.eval.test_eval import *; asyncio.run(main())"
|
||||
|
||||
# From package context
|
||||
python -m autogenstudio.eval.test_eval
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from autogen_agentchat.base import TaskResult
|
||||
from autogen_agentchat.messages import TextMessage
|
||||
|
||||
from ..datamodel.eval import EvalDimensionScore, EvalJudgeCriteria, EvalRunResult, EvalScore, EvalTask
|
||||
from ._orchestrator import EvalOrchestrator
|
||||
from .judges import LLMEvalJudge
|
||||
from .runners import ModelEvalRunner
|
||||
|
||||
|
||||
class MockChatCompletionClient:
|
||||
"""Mock chat completion client for testing."""
|
||||
|
||||
def __init__(self, response_content="Mock response"):
|
||||
self.response_content = response_content
|
||||
|
||||
async def create(self, messages, cancellation_token=None, **kwargs):
|
||||
"""Mock create method that returns a simple response."""
|
||||
mock_response = MagicMock()
|
||||
|
||||
# Handle JSON output for judges
|
||||
if kwargs.get("json_output") == EvalDimensionScore:
|
||||
mock_response.content = '{"dimension": "test", "score": 8.5, "reason": "Good response", "max_value": 10.0, "min_value": 0.0}'
|
||||
else:
|
||||
mock_response.content = self.response_content
|
||||
|
||||
return mock_response
|
||||
|
||||
def dump_component(self):
|
||||
"""Mock dump_component for serialization."""
|
||||
from autogen_core import ComponentModel
|
||||
# Return a proper ComponentModel-like object
|
||||
mock_component = MagicMock()
|
||||
mock_component.provider = "mock_provider"
|
||||
mock_component.config = {"response": self.response_content}
|
||||
mock_component.model_dump = lambda: {
|
||||
"provider": "mock_provider",
|
||||
"config": {"response": self.response_content}
|
||||
}
|
||||
return mock_component
|
||||
|
||||
@classmethod
|
||||
def load_component(cls, config):
|
||||
"""Mock load_component for deserialization."""
|
||||
if hasattr(config, 'model_dump'):
|
||||
config_dict = config.model_dump()
|
||||
elif hasattr(config, 'config'):
|
||||
config_dict = config.config
|
||||
else:
|
||||
config_dict = config
|
||||
return cls(config_dict.get("response", "Mock response"))
|
||||
|
||||
|
||||
class TestEvalSystem:
|
||||
"""Test cases for the evaluation system."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client(self):
|
||||
"""Create a mock chat completion client."""
|
||||
return MockChatCompletionClient()
|
||||
|
||||
@pytest.fixture
|
||||
def sample_task(self):
|
||||
"""Create a sample evaluation task."""
|
||||
return EvalTask(
|
||||
name="Sample Task",
|
||||
description="A test task for evaluation",
|
||||
input="What is the capital of France?"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def sample_criteria(self):
|
||||
"""Create sample evaluation criteria."""
|
||||
return [
|
||||
EvalJudgeCriteria(
|
||||
dimension="accuracy",
|
||||
prompt="Evaluate the factual accuracy of the response.",
|
||||
min_value=0,
|
||||
max_value=10
|
||||
),
|
||||
EvalJudgeCriteria(
|
||||
dimension="relevance",
|
||||
prompt="Evaluate how relevant the response is to the question.",
|
||||
min_value=0,
|
||||
max_value=10
|
||||
)
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_runner(self, mock_client, sample_task):
|
||||
"""Test the ModelEvalRunner with a mock client."""
|
||||
runner = ModelEvalRunner(model_client=mock_client)
|
||||
|
||||
# Test batch interface
|
||||
results = await runner.run([sample_task])
|
||||
|
||||
assert len(results) == 1
|
||||
result = results[0]
|
||||
assert isinstance(result, EvalRunResult)
|
||||
assert result.status is True
|
||||
assert result.result is not None
|
||||
assert isinstance(result.result, TaskResult)
|
||||
assert len(result.result.messages) > 0
|
||||
assert result.error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_runner_batch(self, mock_client):
|
||||
"""Test the ModelEvalRunner with multiple tasks."""
|
||||
runner = ModelEvalRunner(model_client=mock_client)
|
||||
|
||||
# Create multiple tasks
|
||||
tasks = [
|
||||
EvalTask(name="Task 1", input="What is 2+2?"),
|
||||
EvalTask(name="Task 2", input="What is 3+3?"),
|
||||
EvalTask(name="Task 3", input="What is 4+4?"),
|
||||
]
|
||||
|
||||
# Test batch processing
|
||||
results = await runner.run(tasks)
|
||||
|
||||
assert len(results) == 3
|
||||
for result in results:
|
||||
assert isinstance(result, EvalRunResult)
|
||||
assert result.status is True
|
||||
assert result.result is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_judge(self, mock_client, sample_task, sample_criteria):
|
||||
"""Test the LLMEvalJudge with a mock client."""
|
||||
judge = LLMEvalJudge(model_client=mock_client)
|
||||
|
||||
# Create a mock run result
|
||||
run_result = EvalRunResult(
|
||||
status=True,
|
||||
result=TaskResult(messages=[TextMessage(content="Paris is the capital of France.", source="model")])
|
||||
)
|
||||
|
||||
score = await judge.judge(sample_task, run_result, sample_criteria)
|
||||
|
||||
assert isinstance(score, EvalScore)
|
||||
assert len(score.dimension_scores) == 2
|
||||
assert all(isinstance(ds, EvalDimensionScore) for ds in score.dimension_scores)
|
||||
assert score.overall_score is not None
|
||||
assert 0 <= score.overall_score <= 10
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_task_management(self):
|
||||
"""Test the orchestrator's task management functionality."""
|
||||
orchestrator = EvalOrchestrator() # In-memory mode
|
||||
|
||||
task = EvalTask(
|
||||
name="Test Task",
|
||||
description="A test task",
|
||||
input="Test input"
|
||||
)
|
||||
|
||||
# Create task
|
||||
task_id = await orchestrator.create_task(task)
|
||||
assert task_id is not None
|
||||
|
||||
# Get task
|
||||
retrieved_task = await orchestrator.get_task(task_id)
|
||||
assert retrieved_task is not None
|
||||
assert retrieved_task.name == "Test Task"
|
||||
|
||||
# List tasks
|
||||
tasks = await orchestrator.list_tasks()
|
||||
assert len(tasks) == 1
|
||||
assert tasks[0].name == "Test Task"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_criteria_management(self):
|
||||
"""Test the orchestrator's criteria management functionality."""
|
||||
orchestrator = EvalOrchestrator() # In-memory mode
|
||||
|
||||
criteria = EvalJudgeCriteria(
|
||||
dimension="test_dimension",
|
||||
prompt="Test prompt",
|
||||
min_value=0,
|
||||
max_value=10
|
||||
)
|
||||
|
||||
# Create criteria
|
||||
criteria_id = await orchestrator.create_criteria(criteria)
|
||||
assert criteria_id is not None
|
||||
|
||||
# Get criteria
|
||||
retrieved_criteria = await orchestrator.get_criteria(criteria_id)
|
||||
assert retrieved_criteria is not None
|
||||
assert retrieved_criteria.dimension == "test_dimension"
|
||||
|
||||
# List criteria
|
||||
criteria_list = await orchestrator.list_criteria()
|
||||
assert len(criteria_list) == 1
|
||||
assert criteria_list[0].dimension == "test_dimension"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orchestrator_run_creation(self, mock_client, sample_task, sample_criteria):
|
||||
"""Test the orchestrator's run creation functionality."""
|
||||
orchestrator = EvalOrchestrator() # In-memory mode
|
||||
|
||||
# Create task and criteria first
|
||||
task_id = await orchestrator.create_task(sample_task)
|
||||
criteria_ids = []
|
||||
for criterion in sample_criteria:
|
||||
criteria_ids.append(await orchestrator.create_criteria(criterion))
|
||||
|
||||
# Skip serialization-dependent tests for now
|
||||
# This test verifies task and criteria creation works
|
||||
assert task_id is not None
|
||||
assert len(criteria_ids) == 2
|
||||
|
||||
# Verify we can retrieve them
|
||||
retrieved_task = await orchestrator.get_task(task_id)
|
||||
assert retrieved_task is not None
|
||||
assert retrieved_task.name == sample_task.name
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_evaluation_flow(self, mock_client, sample_task, sample_criteria):
|
||||
"""Test direct evaluation without orchestrator serialization."""
|
||||
# Test runner directly
|
||||
runner = ModelEvalRunner(model_client=mock_client)
|
||||
run_results = await runner.run([sample_task])
|
||||
|
||||
assert len(run_results) == 1
|
||||
run_result = run_results[0]
|
||||
assert isinstance(run_result, EvalRunResult)
|
||||
assert run_result.status is True
|
||||
|
||||
# Test judge directly
|
||||
judge = LLMEvalJudge(model_client=mock_client)
|
||||
score = await judge.judge(sample_task, run_result, sample_criteria)
|
||||
|
||||
assert isinstance(score, EvalScore)
|
||||
assert len(score.dimension_scores) == 2
|
||||
assert score.overall_score is not None
|
||||
|
||||
|
||||
def test_basic_component_creation():
|
||||
"""Test that components can be created without serialization."""
|
||||
mock_client = MockChatCompletionClient("Test response")
|
||||
|
||||
# Test runner creation
|
||||
runner = ModelEvalRunner(model_client=mock_client)
|
||||
assert runner.name == "Model Runner"
|
||||
|
||||
# Test judge creation
|
||||
judge = LLMEvalJudge(model_client=mock_client)
|
||||
assert judge.name == "LLM Judge"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Simple test runner for direct execution
|
||||
async def main():
|
||||
"""Run a simple test without pytest."""
|
||||
print("Running basic eval system test...")
|
||||
|
||||
# Create mock client
|
||||
mock_client = MockChatCompletionClient("Paris is the capital of France.")
|
||||
|
||||
# Test model runner
|
||||
task = EvalTask(
|
||||
name="Test",
|
||||
input="What is the capital of France?"
|
||||
)
|
||||
|
||||
runner = ModelEvalRunner(model_client=mock_client)
|
||||
results = await runner.run([task])
|
||||
result = results[0]
|
||||
|
||||
print(f"Runner result: {result.status}")
|
||||
if result.result and result.result.messages:
|
||||
print(f"Response: {result.result.messages[0].content}")
|
||||
else:
|
||||
print("No result")
|
||||
|
||||
# Test judge
|
||||
judge = LLMEvalJudge(model_client=mock_client)
|
||||
criteria = [EvalJudgeCriteria(
|
||||
dimension="accuracy",
|
||||
prompt="Rate accuracy",
|
||||
min_value=0,
|
||||
max_value=10
|
||||
)]
|
||||
|
||||
score = await judge.judge(task, result, criteria)
|
||||
print(f"Score: {score.overall_score}")
|
||||
print(f"Dimension scores: {[(ds.dimension, ds.score) for ds in score.dimension_scores]}")
|
||||
|
||||
print("✅ Basic eval system test completed!")
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,3 +1,3 @@
|
||||
VERSION = "0.4.3"
|
||||
VERSION = "0.4.3dev2"
|
||||
__version__ = VERSION
|
||||
APP_NAME = "autogenstudio"
|
||||
|
||||
@@ -80,6 +80,82 @@ A **Workflow** is a container for a set of **Steps** (units of computation) and
|
||||
- **State Access**: Steps read/update workflow state via the provided `Context` object (`context.get()` / `context.set()`).
|
||||
- **Requirement**: All steps must specify input/output schemas and implement the `execute(input_data, context)` method.
|
||||
|
||||
## Programming Model: Simple Example
|
||||
|
||||
Here's a minimal workflow with two echo steps showing the core programming model:
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from autogenstudio.workflow.core import Workflow, WorkflowRunner, StepMetadata, WorkflowMetadata
|
||||
from autogenstudio.workflow.steps import EchoStep
|
||||
|
||||
class MessageInput(BaseModel):
|
||||
message: str
|
||||
|
||||
class MessageOutput(BaseModel):
|
||||
result: str
|
||||
|
||||
# Create workflow
|
||||
workflow = Workflow(
|
||||
metadata=WorkflowMetadata(
|
||||
name="Simple Echo Chain",
|
||||
description="Two echo steps with conditional edge",
|
||||
version="1.0.0"
|
||||
)
|
||||
)
|
||||
|
||||
# Step 1: First echo
|
||||
step1 = EchoStep(
|
||||
step_id="echo1",
|
||||
metadata=StepMetadata(name="First Echo"),
|
||||
input_type=MessageInput,
|
||||
output_type=MessageOutput,
|
||||
prefix="Step 1: "
|
||||
)
|
||||
|
||||
# Step 2: Second echo
|
||||
step2 = EchoStep(
|
||||
step_id="echo2",
|
||||
metadata=StepMetadata(name="Second Echo"),
|
||||
input_type=MessageOutput,
|
||||
output_type=MessageOutput,
|
||||
prefix="Step 2: "
|
||||
)
|
||||
|
||||
# Add to workflow
|
||||
workflow.add_step(step1)
|
||||
workflow.add_step(step2)
|
||||
workflow.add_edge("echo1", "echo2") # Can add conditions here
|
||||
workflow.set_start_step("echo1")
|
||||
workflow.add_end_step("echo2")
|
||||
|
||||
# Execute
|
||||
runner = WorkflowRunner()
|
||||
result = await runner.run(workflow, {"message": "Hello"})
|
||||
```
|
||||
|
||||
### DSL Serialization & Deserialization
|
||||
|
||||
The workflow can be dumped to JSON configuration and reinstantiated:
|
||||
|
||||
```python
|
||||
# Serialize to DSL/config
|
||||
config = workflow.dump_component()
|
||||
json_config = config.model_dump_json(indent=2)
|
||||
|
||||
# Save to file
|
||||
with open("workflow.json", "w") as f:
|
||||
f.write(json_config)
|
||||
|
||||
# Load from config
|
||||
loaded_workflow = Workflow.load_component(config)
|
||||
|
||||
# Both workflows produce identical results
|
||||
original_result = await runner.run(workflow, {"message": "Test"})
|
||||
loaded_result = await runner.run(loaded_workflow, {"message": "Test"})
|
||||
# original_result == loaded_result
|
||||
```
|
||||
|
||||
## Example Workflows
|
||||
|
||||
- [Simple Sequential](./examples/simple_sequential.py)
|
||||
|
||||
534
python/packages/autogen-studio/evalplan.md
Normal file
534
python/packages/autogen-studio/evalplan.md
Normal file
@@ -0,0 +1,534 @@
|
||||
# AutoGen Studio Evaluation System - UI/API Design Plan
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
This document outlines the comprehensive design for AutoGen Studio's evaluation system UI and API, providing a complete user experience for creating, managing, and analyzing LLM/agent evaluations.
|
||||
|
||||
## 📊 Current Architecture Analysis
|
||||
|
||||
### ✅ Existing Patterns
|
||||
- **Manager/Sidebar Pattern**: Workflows, Teams, MCP all use `Manager + Sidebar + Builder`
|
||||
- **API Structure**: RESTful with `BaseAPI` class, user-scoped endpoints
|
||||
- **State Management**: React hooks + localStorage for persistence
|
||||
- **UI Components**: Ant Design + Lucide icons, collapsible sidebars
|
||||
|
||||
### 🏗️ Backend Capabilities
|
||||
- **Batch-first runners** with parallel processing
|
||||
- **Isolated team evaluation** preventing state contamination
|
||||
- **LLM-based judges** with multi-dimensional scoring
|
||||
- **Orchestrator** for managing evaluation lifecycle
|
||||
- **Database persistence** for tasks, criteria, runs, and results
|
||||
|
||||
## 🚀 Proposed User Experience Flow
|
||||
|
||||
### 1. 📋 Task Management (`/evaluations/tasks`)
|
||||
|
||||
**Features:**
|
||||
- **Create Task Sets**
|
||||
- Manual task creation (text input + expected output)
|
||||
- CSV/JSON upload (batch import)
|
||||
- Template library (common eval patterns)
|
||||
- Multi-modal support (text + images)
|
||||
- **Task Set Library**
|
||||
- Browse existing task sets
|
||||
- Filter by category/tags
|
||||
- Preview tasks
|
||||
- Clone/duplicate sets
|
||||
|
||||
**User Journey:**
|
||||
```
|
||||
User creates task set → Adds individual tasks or uploads batch →
|
||||
Organizes with tags/categories → Saves for reuse
|
||||
```
|
||||
|
||||
### 2. ⚙️ Evaluation Configuration (`/evaluations/configs`)
|
||||
|
||||
**Features:**
|
||||
- **Runner Configuration**
|
||||
- Model runners (select model, parameters)
|
||||
- Team runners (select team, max turns)
|
||||
- Runner comparison setup
|
||||
- **Judge Configuration**
|
||||
- Criteria definition (accuracy, relevance, etc.)
|
||||
- Custom prompts per dimension
|
||||
- Scoring scales (0-10, 0-100, etc.)
|
||||
- Judge model selection
|
||||
- **Evaluation Templates**
|
||||
- Pre-built templates (QA, summarization, etc.)
|
||||
- Save custom configs as templates
|
||||
- Share templates with team
|
||||
|
||||
**User Journey:**
|
||||
```
|
||||
User selects runner type → Configures judge criteria →
|
||||
Sets scoring parameters → Saves as reusable config
|
||||
```
|
||||
|
||||
### 3. 🚀 Run Management (`/evaluations/runs`)
|
||||
|
||||
**Features:**
|
||||
- **Create New Run**
|
||||
- Select task set + config
|
||||
- Run preview/estimation
|
||||
- Batch size selection
|
||||
- Schedule/trigger run
|
||||
- **Active Runs**
|
||||
- Real-time progress tracking
|
||||
- Live status updates
|
||||
- Cancel/pause controls
|
||||
- Resource usage monitoring
|
||||
- **Run History**
|
||||
- Filter by date/status/config
|
||||
- Compare multiple runs
|
||||
- Export results
|
||||
|
||||
**User Journey:**
|
||||
```
|
||||
User combines task set + config → Reviews run parameters →
|
||||
Starts evaluation → Monitors progress → Views completion
|
||||
```
|
||||
|
||||
### 4. 📊 Results & Analytics (`/evaluations/results`)
|
||||
|
||||
**Features:**
|
||||
- **Individual Run Results**
|
||||
- Task-by-task breakdown
|
||||
- Score visualizations
|
||||
- Error analysis
|
||||
- Raw response viewer
|
||||
- **Comparative Analysis**
|
||||
- Runner performance comparison
|
||||
- Radar charts by dimension
|
||||
- Statistical summaries
|
||||
- A/B test results
|
||||
- **Export & Reporting**
|
||||
- CSV/JSON export
|
||||
- PDF reports
|
||||
- Dashboard sharing
|
||||
|
||||
**User Journey:**
|
||||
```
|
||||
User views run results → Analyzes scores by dimension →
|
||||
Compares with other runs → Exports findings → Shares insights
|
||||
```
|
||||
|
||||
## 🔗 Required API Endpoints
|
||||
|
||||
### Task Management API
|
||||
```typescript
|
||||
// /api/evaluations/tasks
|
||||
GET /api/evaluations/tasks?user_id={id} // List task sets
|
||||
POST /api/evaluations/tasks // Create task set
|
||||
GET /api/evaluations/tasks/{task_set_id} // Get task set
|
||||
PUT /api/evaluations/tasks/{task_set_id} // Update task set
|
||||
DELETE /api/evaluations/tasks/{task_set_id} // Delete task set
|
||||
POST /api/evaluations/tasks/{task_set_id}/upload // Upload tasks (CSV/JSON)
|
||||
GET /api/evaluations/tasks/{task_set_id}/export // Export task set
|
||||
```
|
||||
|
||||
### Configuration API
|
||||
```typescript
|
||||
// /api/evaluations/configs
|
||||
GET /api/evaluations/configs?user_id={id} // List eval configs
|
||||
POST /api/evaluations/configs // Create config
|
||||
GET /api/evaluations/configs/{config_id} // Get config
|
||||
PUT /api/evaluations/configs/{config_id} // Update config
|
||||
DELETE /api/evaluations/configs/{config_id} // Delete config
|
||||
GET /api/evaluations/configs/templates // Get templates
|
||||
```
|
||||
|
||||
### Runs API
|
||||
```typescript
|
||||
// /api/evaluations/runs
|
||||
GET /api/evaluations/runs?user_id={id} // List runs
|
||||
POST /api/evaluations/runs // Create run
|
||||
GET /api/evaluations/runs/{run_id} // Get run details
|
||||
PUT /api/evaluations/runs/{run_id}/cancel // Cancel run
|
||||
GET /api/evaluations/runs/{run_id}/status // Get run status
|
||||
GET /api/evaluations/runs/{run_id}/results // Get run results
|
||||
GET /api/evaluations/runs/{run_id}/progress // Get progress (SSE)
|
||||
POST /api/evaluations/runs/compare // Compare runs
|
||||
```
|
||||
|
||||
### Results API
|
||||
```typescript
|
||||
// /api/evaluations/results
|
||||
GET /api/evaluations/results/{run_id} // Get detailed results
|
||||
GET /api/evaluations/results/{run_id}/export // Export results
|
||||
POST /api/evaluations/results/analyze // Batch analysis
|
||||
GET /api/evaluations/results/dashboard/{dashboard_id} // Shared dashboard
|
||||
```
|
||||
|
||||
## 📱 UI Views & Components Design
|
||||
|
||||
### Main Evaluation Page: `/evaluations`
|
||||
|
||||
```tsx
|
||||
// Similar to WorkflowManager pattern
|
||||
export const EvaluationManager: React.FC = () => {
|
||||
// State management
|
||||
const [currentView, setCurrentView] = useState<'tasks' | 'configs' | 'runs' | 'results'>('runs');
|
||||
const [isSidebarOpen, setIsSidebarOpen] = useState(true);
|
||||
|
||||
return (
|
||||
<div className="flex h-screen">
|
||||
<EvaluationSidebar
|
||||
isOpen={isSidebarOpen}
|
||||
onToggle={setIsSidebarOpen}
|
||||
currentView={currentView}
|
||||
onViewChange={setCurrentView}
|
||||
/>
|
||||
<main className="flex-1">
|
||||
{currentView === 'tasks' && <TaskManager />}
|
||||
{currentView === 'configs' && <ConfigManager />}
|
||||
{currentView === 'runs' && <RunManager />}
|
||||
{currentView === 'results' && <ResultsManager />}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
### 1. Task Manager Component
|
||||
|
||||
**Layout**: Split view with task set list (1/3) + detail view (2/3)
|
||||
|
||||
**Features**:
|
||||
- Task set creation modal
|
||||
- CSV/JSON upload modal
|
||||
- Task preview cards
|
||||
- Inline editing
|
||||
- Tag management
|
||||
|
||||
```tsx
|
||||
const TaskManager = () => {
|
||||
const [taskSets, setTaskSets] = useState<TaskSet[]>([]);
|
||||
const [selectedTaskSet, setSelectedTaskSet] = useState<TaskSet | null>(null);
|
||||
const [showCreateModal, setShowCreateModal] = useState(false);
|
||||
const [showUploadModal, setShowUploadModal] = useState(false);
|
||||
|
||||
return (
|
||||
<div className="flex">
|
||||
{/* Task Set List */}
|
||||
<div className="w-1/3 border-r">
|
||||
<div className="p-4 border-b">
|
||||
<Button.Group>
|
||||
<Button onClick={() => setShowCreateModal(true)}>
|
||||
<Plus /> New Task Set
|
||||
</Button>
|
||||
<Button onClick={() => setShowUploadModal(true)}>
|
||||
<Upload /> Upload Tasks
|
||||
</Button>
|
||||
</Button.Group>
|
||||
</div>
|
||||
|
||||
<TaskSetList
|
||||
taskSets={taskSets}
|
||||
selectedId={selectedTaskSet?.id}
|
||||
onSelect={setSelectedTaskSet}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Task Set Detail */}
|
||||
<div className="flex-1">
|
||||
{selectedTaskSet ? (
|
||||
<TaskSetDetail
|
||||
taskSet={selectedTaskSet}
|
||||
onUpdate={handleUpdateTaskSet}
|
||||
/>
|
||||
) : (
|
||||
<EmptyState message="Select a task set to view details" />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
### 2. Configuration Manager
|
||||
|
||||
**Layout**: Split view with config list (1/3) + visual builder (2/3)
|
||||
|
||||
**Features**:
|
||||
- Visual configuration builder
|
||||
- Runner/judge selection dropdowns
|
||||
- Criteria editor with custom prompts
|
||||
- Template library
|
||||
- Preview/test functionality
|
||||
|
||||
```tsx
|
||||
const ConfigManager = () => {
|
||||
const [configs, setConfigs] = useState<EvalConfig[]>([]);
|
||||
const [selectedConfig, setSelectedConfig] = useState<EvalConfig | null>(null);
|
||||
const [showBuilder, setShowBuilder] = useState(false);
|
||||
|
||||
return (
|
||||
<div className="flex">
|
||||
{/* Config List */}
|
||||
<div className="w-1/3 border-r">
|
||||
<div className="p-4 border-b">
|
||||
<Button onClick={() => setShowBuilder(true)}>
|
||||
<Settings /> New Configuration
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<ConfigList
|
||||
configs={configs}
|
||||
selectedId={selectedConfig?.id}
|
||||
onSelect={setSelectedConfig}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Config Builder */}
|
||||
<div className="flex-1">
|
||||
{showBuilder || selectedConfig ? (
|
||||
<ConfigBuilder
|
||||
config={selectedConfig}
|
||||
onSave={handleSaveConfig}
|
||||
onCancel={() => setShowBuilder(false)}
|
||||
/>
|
||||
) : (
|
||||
<EmptyState message="Select or create a configuration" />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
### 3. Run Manager
|
||||
|
||||
**Layout**: Split view with run list (1/3) + run detail/monitoring (2/3)
|
||||
|
||||
**Features**:
|
||||
- Run creation wizard
|
||||
- Real-time progress tracking
|
||||
- Status indicators
|
||||
- Cancel/pause controls
|
||||
- Resource monitoring
|
||||
|
||||
```tsx
|
||||
const RunManager = () => {
|
||||
const [runs, setRuns] = useState<EvalRun[]>([]);
|
||||
const [selectedRun, setSelectedRun] = useState<EvalRun | null>(null);
|
||||
const [showCreateModal, setShowCreateModal] = useState(false);
|
||||
|
||||
return (
|
||||
<div className="flex">
|
||||
{/* Run List */}
|
||||
<div className="w-1/3 border-r">
|
||||
<div className="p-4 border-b">
|
||||
<Button type="primary" onClick={() => setShowCreateModal(true)}>
|
||||
<Play /> Start Evaluation
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<RunList
|
||||
runs={runs}
|
||||
selectedId={selectedRun?.id}
|
||||
onSelect={setSelectedRun}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Run Detail */}
|
||||
<div className="flex-1">
|
||||
{selectedRun ? (
|
||||
<RunDetail
|
||||
run={selectedRun}
|
||||
onCancel={handleCancelRun}
|
||||
/>
|
||||
) : (
|
||||
<EmptyState message="Select a run to view details" />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
### 4. Results Manager
|
||||
|
||||
**Layout**: Full-width with toolbar + switchable view modes
|
||||
|
||||
**Features**:
|
||||
- Table/charts/comparison view modes
|
||||
- Interactive visualizations
|
||||
- Export functionality
|
||||
- Filtering and search
|
||||
- Comparative analysis tools
|
||||
|
||||
```tsx
|
||||
const ResultsManager = () => {
|
||||
const [results, setResults] = useState<EvalResult[]>([]);
|
||||
const [selectedResult, setSelectedResult] = useState<EvalResult | null>(null);
|
||||
const [viewMode, setViewMode] = useState<'table' | 'charts' | 'compare'>('table');
|
||||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
{/* Toolbar */}
|
||||
<div className="p-4 border-b">
|
||||
<div className="flex justify-between">
|
||||
<Radio.Group value={viewMode} onChange={(e) => setViewMode(e.target.value)}>
|
||||
<Radio.Button value="table">
|
||||
<Table /> Table View
|
||||
</Radio.Button>
|
||||
<Radio.Button value="charts">
|
||||
<BarChart /> Charts
|
||||
</Radio.Button>
|
||||
<Radio.Button value="compare">
|
||||
<GitCompare /> Compare
|
||||
</Radio.Button>
|
||||
</Radio.Group>
|
||||
|
||||
<Button.Group>
|
||||
<Button><Download /> Export</Button>
|
||||
<Button><Share /> Share</Button>
|
||||
</Button.Group>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Results Content */}
|
||||
<div className="flex-1">
|
||||
{viewMode === 'table' && <ResultsTable results={results} />}
|
||||
{viewMode === 'charts' && <ResultsCharts results={results} />}
|
||||
{viewMode === 'compare' && <ResultsComparison results={results} />}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
## 🧩 Key Reusable Components
|
||||
|
||||
### Status Components
|
||||
```tsx
|
||||
// Status indicator with real-time updates
|
||||
const RunStatus = ({ status, progress }: { status: EvalRunStatus, progress?: number }) => (
|
||||
<div className="flex items-center gap-2">
|
||||
<StatusIcon status={status} />
|
||||
<span>{status}</span>
|
||||
{progress && <Progress percent={progress} size="small" />}
|
||||
</div>
|
||||
);
|
||||
```
|
||||
|
||||
### Data Visualization
|
||||
```tsx
|
||||
// Interactive task preview
|
||||
const TaskPreview = ({ task }: { task: EvalTask }) => (
|
||||
<Card size="small">
|
||||
<div className="space-y-2">
|
||||
<Text strong>{task.name}</Text>
|
||||
<Paragraph ellipsis={{ rows: 2 }}>{task.description}</Paragraph>
|
||||
<Tag color="blue">{task.input.length} inputs</Tag>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
|
||||
// Score visualization radar chart
|
||||
const ScoreRadar = ({ scores }: { scores: EvalScore[] }) => (
|
||||
<ResponsiveRadar
|
||||
data={transformScoresForRadar(scores)}
|
||||
keys={['score']}
|
||||
indexBy="dimension"
|
||||
maxValue={10}
|
||||
/>
|
||||
);
|
||||
```
|
||||
|
||||
### Form Components
|
||||
```tsx
|
||||
// Configuration builder forms
|
||||
const RunnerConfigForm = ({ config, onChange }) => { /* ... */ };
|
||||
const JudgeConfigForm = ({ config, onChange }) => { /* ... */ };
|
||||
const CriteriaEditor = ({ criteria, onChange }) => { /* ... */ };
|
||||
```
|
||||
|
||||
## 🚀 Implementation Roadmap
|
||||
|
||||
### Phase 1: MVP (Core Functionality)
|
||||
**Timeline**: 2-3 weeks
|
||||
|
||||
**Backend:**
|
||||
- Basic evaluation API endpoints (`/tasks`, `/configs`, `/runs`)
|
||||
- Integration with existing orchestrator
|
||||
- Database schema for eval entities
|
||||
|
||||
**Frontend:**
|
||||
- Main evaluation page with 4-tab navigation
|
||||
- Basic task management (create, list, view)
|
||||
- Simple run creation and status tracking
|
||||
- Results table view
|
||||
|
||||
**Success Criteria:**
|
||||
- Users can create task sets manually
|
||||
- Users can configure basic model/team runners
|
||||
- Users can start evaluations and see results
|
||||
- Results display in tabular format
|
||||
|
||||
### Phase 2: Enhanced Experience (Polish & Features)
|
||||
**Timeline**: 3-4 weeks
|
||||
|
||||
**Backend:**
|
||||
- Task upload/import functionality
|
||||
- Real-time progress via Server-Sent Events
|
||||
- Advanced filtering and search
|
||||
- Export endpoints
|
||||
|
||||
**Frontend:**
|
||||
- Configuration builder with visual UI
|
||||
- Real-time progress updates with WebSocket/SSE
|
||||
- Charts and visualization components
|
||||
- Task templates and CSV/JSON upload
|
||||
- Advanced filtering and search
|
||||
|
||||
**Success Criteria:**
|
||||
- Users can upload task sets via CSV/JSON
|
||||
- Live progress tracking during runs
|
||||
- Visual score comparisons with charts
|
||||
- Template library for common eval patterns
|
||||
|
||||
### Phase 3: Advanced Analytics (Production Ready)
|
||||
**Timeline**: 4-5 weeks
|
||||
|
||||
**Backend:**
|
||||
- Comparative analysis endpoints
|
||||
- Dashboard sharing functionality
|
||||
- Advanced statistics and reporting
|
||||
- Integration with teams/workflows
|
||||
|
||||
**Frontend:**
|
||||
- Advanced analytics and reporting
|
||||
- Dashboard sharing and collaboration
|
||||
- A/B testing workflows
|
||||
- Integration with existing teams/workflows
|
||||
- Performance optimizations
|
||||
|
||||
**Success Criteria:**
|
||||
- Comprehensive evaluation analytics
|
||||
- Team collaboration features
|
||||
- Production-ready performance
|
||||
- Full integration with AutoGen Studio ecosystem
|
||||
|
||||
## 📊 Success Metrics
|
||||
|
||||
### User Engagement
|
||||
- **Task Set Creation**: Users create and reuse task sets
|
||||
- **Run Frequency**: Regular evaluation runs per user
|
||||
- **Result Analysis**: Time spent analyzing results
|
||||
|
||||
### Performance
|
||||
- **Batch Processing**: 10x faster evaluation runs
|
||||
- **UI Responsiveness**: <200ms page load times
|
||||
- **Real-time Updates**: Live progress tracking
|
||||
|
||||
### Adoption
|
||||
- **Feature Usage**: All 4 main views actively used
|
||||
- **Template Reuse**: Common evaluation patterns shared
|
||||
- **Export Utilization**: Results exported for external analysis
|
||||
|
||||
## 🎯 Conclusion
|
||||
|
||||
This comprehensive evaluation system design provides AutoGen Studio users with a complete workflow for LLM/agent evaluation, from task creation through results analysis. By leveraging existing UI patterns and the new batch-native backend architecture, we can deliver a powerful, scalable, and user-friendly evaluation experience that scales from simple experiments to production evaluation workflows.
|
||||
|
||||
The phased implementation approach ensures rapid delivery of core value while building toward advanced analytics and collaboration features that will position AutoGen Studio as a leading platform for AI evaluation and analysis.
|
||||
@@ -139,12 +139,12 @@ export const WorkflowManager: React.FC = () => {
|
||||
name,
|
||||
description: "A new workflow.",
|
||||
config: {
|
||||
provider: "autogenstudio.workflow.core.Workflow",
|
||||
component_type: "workflow",
|
||||
version: 1,
|
||||
component_version: 1,
|
||||
description: "A new workflow.",
|
||||
label: "New Workflow",
|
||||
provider: "autogenstudio.workflow.core.Workflow",
|
||||
config: {
|
||||
metadata: {
|
||||
name,
|
||||
@@ -201,7 +201,7 @@ export const WorkflowManager: React.FC = () => {
|
||||
name: workflowConfig?.name || currentWorkflow.config.config.name,
|
||||
description:
|
||||
workflowConfig?.description ||
|
||||
currentWorkflow.config.config.description,
|
||||
currentWorkflow.config.config.description || "",
|
||||
config: workflowData.config || currentWorkflow.config,
|
||||
},
|
||||
user.id
|
||||
|
||||
Reference in New Issue
Block a user