github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-04-08 03:00:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend): Add correctness score to execution activity generation (#11325)

Browse Source 
## Summary
Add AI-generated correctness score field to execution activity status
generation to provide quantitative assessment of how well executions
achieved their intended purpose.

New page:
<img width="1000" height="229" alt="image"
src="https://github.com/user-attachments/assets/5cb907cf-5bc7-4b96-8128-8eecccde9960"
/>

Old page:
<img width="1000" alt="image"
src="https://github.com/user-attachments/assets/ece0dfab-1e50-4121-9985-d585f7fcd4d2"
/>


## What Changed
- Added `correctness_score` field (float 0.0-1.0) to
`GraphExecutionStats` model
- **REFACTORED**: Removed duplicate `llm_utils.py` and reused existing
`AIStructuredResponseGeneratorBlock` logic
- Updated activity status generator to use structured responses instead
of plain text
- Modified prompts to include correctness assessment with 5-tier scoring
system:
  - 0.0-0.2: Failure
  - 0.2-0.4: Poor 
  - 0.4-0.6: Partial Success
  - 0.6-0.8: Mostly Successful
  - 0.8-1.0: Success
- Updated manager.py to extract and set both activity_status and
correctness_score
- Fixed tests to work with existing structured response interface

## Technical Details
- **Code Reuse**: Eliminated duplication by using existing
`AIStructuredResponseGeneratorBlock` instead of creating new LLM
utilities
- Added JSON validation with retry logic for malformed responses
- Maintained backward compatibility for existing activity status
functionality
- Score is clamped to valid 0.0-1.0 range and validated
- All type errors resolved and linting passes

## Test Plan
- [x] All existing tests pass with refactored structure
- [x] Structured LLM call functionality tested with success and error
cases
- [x] Activity status generation tested with various execution scenarios
- [x] Integration tests verify both fields are properly set in execution
stats
- [x] No code duplication - reuses existing block logic

🤖 Generated with [Claude Code](https://claude.ai/code)

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: Zamil Majdy <majdyz@users.noreply.github.com>
This commit is contained in:
Zamil Majdy
2025-11-06 06:42:13 +02:00
committed by GitHub
parent 37b3e4e82e
commit 6037f80502
6 changed files with 259 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
autogpt_platform/backend/backend/data/execution.py
View File

@@ -175,6 +175,10 @@ class GraphExecutionMeta(BaseDbModel):
default=None,
description="AI-generated summary of what the agent did",
)
correctness_score: float | None = Field(
default=None,
description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",
)
def to_db(self) -> GraphExecutionStats:
return GraphExecutionStats(
@@ -187,6 +191,7 @@ class GraphExecutionMeta(BaseDbModel):
node_error_count=self.node_error_count,
error=self.error,
activity_status=self.activity_status,
correctness_score=self.correctness_score,
)
stats: Stats | None
@@ -244,6 +249,7 @@ class GraphExecutionMeta(BaseDbModel):
else stats.error
),
activity_status=stats.activity_status,
correctness_score=stats.correctness_score,
)
if stats
else None

4
autogpt_platform/backend/backend/data/model.py
View File

@@ -833,6 +833,10 @@ class GraphExecutionStats(BaseModel):
activity_status: Optional[str] = Field(
default=None, description="AI-generated summary of what the agent did"
)
correctness_score: Optional[float] = Field(
default=None,
description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",
)
class UserExecutionSummaryStats(BaseModel):

127
autogpt_platform/backend/backend/executor/activity_status_generator.py
View File

@@ -13,12 +13,11 @@ except ImportError:
from pydantic import SecretStr
from backend.blocks.llm import LlmModel, llm_call
from backend.blocks.llm import AIStructuredResponseGeneratorBlock, LlmModel
from backend.data.block import get_block
from backend.data.execution import ExecutionStatus, NodeExecutionResult
from backend.data.model import APIKeyCredentials, GraphExecutionStats
from backend.util.feature_flag import Flag, is_feature_enabled
from backend.util.retry import func_retry
from backend.util.settings import Settings
from backend.util.truncate import truncate
@@ -70,6 +69,13 @@ class NodeRelation(TypedDict):
sink_block_name: NotRequired[str] # Optional, only set if block exists
class ActivityStatusResponse(TypedDict):
"""Type definition for structured activity status response."""
activity_status: str
correctness_score: float
def _truncate_uuid(uuid_str: str) -> str:
"""Truncate UUID to first segment to reduce payload size."""
if not uuid_str:
@@ -85,9 +91,9 @@ async def generate_activity_status_for_execution(
db_client: "DatabaseManagerAsyncClient",
user_id: str,
execution_status: ExecutionStatus | None = None,
) -> str | None:
) -> ActivityStatusResponse | None:
"""
Generate an AI-based activity status summary for a graph execution.
Generate an AI-based activity status summary and correctness assessment for a graph execution.
This function handles all the data collection and AI generation logic,
keeping the manager integration simple.
@@ -102,7 +108,8 @@ async def generate_activity_status_for_execution(
execution_status: The overall execution status (COMPLETED, FAILED, TERMINATED)
Returns:
AI-generated activity status string, or None if feature is disabled
AI-generated activity status response with activity_status and correctness_status,
or None if feature is disabled
"""
# Check LaunchDarkly feature flag for AI activity status generation with full context support
if not await is_feature_enabled(Flag.AI_ACTIVITY_STATUS, user_id):
@@ -141,16 +148,27 @@ async def generate_activity_status_for_execution(
execution_status,
)
# Prepare prompt for AI
# Prepare prompt for AI with structured output requirements
prompt = [
{
"role": "system",
"content": (
"You are an AI assistant summarizing what you just did for a user in simple, friendly language. "
"Write from the user's perspective about what they accomplished, NOT about technical execution details. "
"Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps. "
"Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc. "
"Keep it to 3 sentences maximum. Be conversational and human-friendly.\n\n"
"You are an AI assistant analyzing what an agent execution accomplished and whether it worked correctly. "
"You need to provide both a user-friendly summary AND a correctness assessment.\n\n"
"FOR THE ACTIVITY STATUS:\n"
"- Write from the user's perspective about what they accomplished, NOT about technical execution details\n"
"- Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps\n"
"- Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc.\n"
"- Keep it to 3 sentences maximum. Be conversational and human-friendly\n\n"
"FOR THE CORRECTNESS SCORE:\n"
"- Provide a score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose\n"
"- Use this scoring guide:\n"
" 0.0-0.2: Failure - The result clearly did not meet the task requirements\n"
" 0.2-0.4: Poor - Major issues; only small parts of the goal were achieved\n"
" 0.4-0.6: Partial Success - Some objectives met, but with noticeable gaps or inaccuracies\n"
" 0.6-0.8: Mostly Successful - Largely achieved the intended outcome, with minor flaws\n"
" 0.8-1.0: Success - Fully met or exceeded the task requirements\n"
"- Base the score on actual outputs produced, not just technical completion\n\n"
"UNDERSTAND THE INTENDED PURPOSE:\n"
"- FIRST: Read the graph description carefully to understand what the user wanted to accomplish\n"
"- The graph name and description tell you the main goal/intention of this automation\n"
@@ -186,7 +204,7 @@ async def generate_activity_status_for_execution(
"role": "user",
"content": (
f"A user ran '{graph_name}' to accomplish something. Based on this execution data, "
f"write what they achieved in simple, user-friendly terms:\n\n"
f"provide both an activity summary and correctness assessment:\n\n"
f"{json.dumps(execution_data, indent=2)}\n\n"
"ANALYSIS CHECKLIST:\n"
"1. READ graph_info.description FIRST - this tells you what the user intended to accomplish\n"
@@ -203,13 +221,20 @@ async def generate_activity_status_for_execution(
"- If description mentions 'content generation' → was content actually generated?\n"
"- If description mentions 'social media posting' → were posts actually made?\n"
"- Match the outputs to the stated intention, not just technical completion\n\n"
"Write 1-3 sentences about what the user accomplished, such as:\n"
"PROVIDE:\n"
"activity_status: 1-3 sentences about what the user accomplished, such as:\n"
"- 'I analyzed your resume and provided detailed feedback for the IT industry.'\n"
"- 'I couldn't complete the task because critical steps failed to produce any results.'\n"
"- 'I failed to generate the content you requested due to missing API access.'\n"
"- 'I extracted key information from your documents and organized it into a summary.'\n"
"- 'The task failed because the blog post creation step didn't produce any output.'\n\n"
"BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, report this as a failure even if status is 'completed'."
"correctness_score: A float score from 0.0 to 1.0 based on how well the intended purpose was achieved:\n"
"- 0.0-0.2: Failure (didn't meet requirements)\n"
"- 0.2-0.4: Poor (major issues, minimal achievement)\n"
"- 0.4-0.6: Partial Success (some objectives met with gaps)\n"
"- 0.6-0.8: Mostly Successful (largely achieved with minor flaws)\n"
"- 0.8-1.0: Success (fully met or exceeded requirements)\n\n"
"BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, use a low score (0.0-0.4) even if status is 'completed'."
),
},
]
@@ -227,14 +252,58 @@ async def generate_activity_status_for_execution(
title="System OpenAI",
)
# Make LLM call using current event loop
activity_status = await _call_llm_direct(credentials, prompt)
# Define expected response format
expected_format = {
"activity_status": "A user-friendly 1-3 sentence summary of what was accomplished",
"correctness_score": "Float score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose",
}
logger.debug(
f"Generated activity status for {graph_exec_id}: {activity_status}"
# Use existing AIStructuredResponseGeneratorBlock for structured LLM call
structured_block = AIStructuredResponseGeneratorBlock()
# Convert credentials to the format expected by AIStructuredResponseGeneratorBlock
credentials_input = {
"provider": credentials.provider,
"id": credentials.id,
"type": credentials.type,
"title": credentials.title,
}
structured_input = AIStructuredResponseGeneratorBlock.Input(
prompt=prompt[1]["content"], # User prompt content
sys_prompt=prompt[0]["content"], # System prompt content
expected_format=expected_format,
model=LlmModel.GPT4O_MINI,
credentials=credentials_input, # type: ignore
max_tokens=150,
retry=3,
)
return activity_status
# Execute the structured LLM call
async for output_name, output_data in structured_block.run(
structured_input, credentials=credentials
):
if output_name == "response":
response = output_data
break
else:
raise RuntimeError("Failed to get response from structured LLM call")
# Create typed response with validation
correctness_score = float(response["correctness_score"])
# Clamp score to valid range
correctness_score = max(0.0, min(1.0, correctness_score))
activity_response: ActivityStatusResponse = {
"activity_status": response["activity_status"],
"correctness_score": correctness_score,
}
logger.debug(
f"Generated activity status for {graph_exec_id}: {activity_response}"
)
return activity_response
except Exception as e:
logger.error(
@@ -448,23 +517,3 @@ def _build_execution_summary(
),
},
}
@func_retry
async def _call_llm_direct(
credentials: APIKeyCredentials, prompt: list[dict[str, str]]
) -> str:
"""Make direct LLM call."""
response = await llm_call(
credentials=credentials,
llm_model=LlmModel.GPT4O_MINI,
prompt=prompt,
max_tokens=150,
compress_prompt_to_fit=True,
)
if response and response.response:
return response.response.strip()
else:
return "Unable to generate activity summary"

218
autogpt_platform/backend/backend/executor/activity_status_generator_test.py
View File

@@ -7,12 +7,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from backend.blocks.llm import LLMResponse
from backend.blocks.llm import LlmModel, LLMResponse
from backend.data.execution import ExecutionStatus, NodeExecutionResult
from backend.data.model import GraphExecutionStats
from backend.executor.activity_status_generator import (
_build_execution_summary,
_call_llm_direct,
generate_activity_status_for_execution,
)
@@ -373,25 +372,24 @@ class TestLLMCall:
"""Tests for LLM calling functionality."""
@pytest.mark.asyncio
async def test_call_llm_direct_success(self):
"""Test successful LLM call."""
async def test_structured_llm_call_success(self):
"""Test successful structured LLM call."""
from pydantic import SecretStr
from backend.blocks.llm import AIStructuredResponseGeneratorBlock
from backend.data.model import APIKeyCredentials
mock_response = LLMResponse(
raw_response={},
prompt=[],
response="Agent successfully processed user input and generated response.",
tool_calls=None,
prompt_tokens=50,
completion_tokens=20,
)
with patch(
"backend.executor.activity_status_generator.llm_call"
) as mock_llm_call:
mock_llm_call.return_value = mock_response
with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(
"backend.blocks.llm.secrets.token_hex", return_value="test123"
):
mock_llm_call.return_value = LLMResponse(
raw_response={},
prompt=[],
response='<json_output id="test123">{"activity_status": "Test completed successfully", "correctness_score": 0.9}</json_output>',
tool_calls=None,
prompt_tokens=50,
completion_tokens=20,
)
credentials = APIKeyCredentials(
id="test",
@@ -401,26 +399,61 @@ class TestLLMCall:
)
prompt = [{"role": "user", "content": "Test prompt"}]
expected_format = {
"activity_status": "User-friendly summary",
"correctness_score": "Float score from 0.0 to 1.0",
}
result = await _call_llm_direct(credentials, prompt)
# Create structured block and input
structured_block = AIStructuredResponseGeneratorBlock()
credentials_input = {
"provider": credentials.provider,
"id": credentials.id,
"type": credentials.type,
"title": credentials.title,
}
assert (
result
== "Agent successfully processed user input and generated response."
structured_input = AIStructuredResponseGeneratorBlock.Input(
prompt=prompt[0]["content"],
expected_format=expected_format,
model=LlmModel.GPT4O_MINI,
credentials=credentials_input, # type: ignore
)
mock_llm_call.assert_called_once()
# Execute the structured LLM call
result = None
async for output_name, output_data in structured_block.run(
structured_input, credentials=credentials
):
if output_name == "response":
result = output_data
break
assert result is not None
assert result["activity_status"] == "Test completed successfully"
assert result["correctness_score"] == 0.9
mock_llm_call.assert_called()
@pytest.mark.asyncio
async def test_call_llm_direct_no_response(self):
"""Test LLM call with no response."""
async def test_structured_llm_call_validation_error(self):
"""Test structured LLM call with validation error."""
from pydantic import SecretStr
from backend.blocks.llm import AIStructuredResponseGeneratorBlock
from backend.data.model import APIKeyCredentials
with patch(
"backend.executor.activity_status_generator.llm_call"
) as mock_llm_call:
mock_llm_call.return_value = None
with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(
"backend.blocks.llm.secrets.token_hex", return_value="test123"
):
# Return invalid JSON that will fail validation (missing required field)
mock_llm_call.return_value = LLMResponse(
raw_response={},
prompt=[],
response='<json_output id="test123">{"activity_status": "Test completed successfully"}</json_output>',
tool_calls=None,
prompt_tokens=50,
completion_tokens=20,
)
credentials = APIKeyCredentials(
id="test",
@@ -430,10 +463,36 @@ class TestLLMCall:
)
prompt = [{"role": "user", "content": "Test prompt"}]
expected_format = {
"activity_status": "User-friendly summary",
"correctness_score": "Float score from 0.0 to 1.0",
}
result = await _call_llm_direct(credentials, prompt)
# Create structured block and input
structured_block = AIStructuredResponseGeneratorBlock()
credentials_input = {
"provider": credentials.provider,
"id": credentials.id,
"type": credentials.type,
"title": credentials.title,
}
assert result == "Unable to generate activity summary"
structured_input = AIStructuredResponseGeneratorBlock.Input(
prompt=prompt[0]["content"],
expected_format=expected_format,
model=LlmModel.GPT4O_MINI,
credentials=credentials_input, # type: ignore
retry=1, # Use fewer retries for faster test
)
with pytest.raises(
Exception
): # AIStructuredResponseGeneratorBlock may raise different exceptions
async for output_name, output_data in structured_block.run(
structured_input, credentials=credentials
):
if output_name == "response":
break
class TestGenerateActivityStatusForExecution:
@@ -461,17 +520,25 @@ class TestGenerateActivityStatusForExecution:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
"backend.executor.activity_status_generator._call_llm_direct"
) as mock_llm, patch(
"backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
mock_llm.return_value = (
"I analyzed your data and provided the requested insights."
)
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
async def mock_run(*args, **kwargs):
yield "response", {
"activity_status": "I analyzed your data and provided the requested insights.",
"correctness_score": 0.85,
}
mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -482,11 +549,16 @@ class TestGenerateActivityStatusForExecution:
user_id="test_user",
)
assert result == "I analyzed your data and provided the requested insights."
assert result is not None
assert (
result["activity_status"]
== "I analyzed your data and provided the requested insights."
)
assert result["correctness_score"] == 0.85
mock_db_client.get_node_executions.assert_called_once()
mock_db_client.get_graph_metadata.assert_called_once()
mock_db_client.get_graph.assert_called_once()
mock_llm.assert_called_once()
mock_structured_block.assert_called_once()
@pytest.mark.asyncio
async def test_generate_status_feature_disabled(self, mock_execution_stats):
@@ -574,15 +646,25 @@ class TestGenerateActivityStatusForExecution:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
"backend.executor.activity_status_generator._call_llm_direct"
) as mock_llm, patch(
"backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
mock_llm.return_value = "Agent completed execution."
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
async def mock_run(*args, **kwargs):
yield "response", {
"activity_status": "Agent completed execution.",
"correctness_score": 0.8,
}
mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -593,10 +675,11 @@ class TestGenerateActivityStatusForExecution:
user_id="test_user",
)
assert result == "Agent completed execution."
# Should use fallback graph name in prompt
call_args = mock_llm.call_args[0][1] # prompt argument
assert "Graph test_graph" in call_args[1]["content"]
assert result is not None
assert result["activity_status"] == "Agent completed execution."
assert result["correctness_score"] == 0.8
# The structured block should have been instantiated
assert mock_structured_block.called
class TestIntegration:
@@ -626,8 +709,8 @@ class TestIntegration:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
"backend.executor.activity_status_generator.llm_call"
) as mock_llm_call, patch(
"backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
@@ -635,15 +718,16 @@ class TestIntegration:
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
mock_response = LLMResponse(
raw_response={},
prompt=[],
response=expected_activity,
tool_calls=None,
prompt_tokens=100,
completion_tokens=30,
)
mock_llm_call.return_value = mock_response
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
async def mock_run(*args, **kwargs):
yield "response", {
"activity_status": expected_activity,
"correctness_score": 0.3, # Low score since there was a failure
}
mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -654,24 +738,14 @@ class TestIntegration:
user_id="test_user",
)
assert result == expected_activity
assert result is not None
assert result["activity_status"] == expected_activity
assert result["correctness_score"] == 0.3
# Verify the correct data was passed to LLM
llm_call_args = mock_llm_call.call_args
prompt = llm_call_args[1]["prompt"]
# Check system prompt
assert prompt[0]["role"] == "system"
assert "user's perspective" in prompt[0]["content"]
# Check user prompt contains expected data
user_content = prompt[1]["content"]
assert "Test Integration Agent" in user_content
assert "user-friendly terms" in user_content.lower()
# Verify that execution data is present in the prompt
assert "{" in user_content # Should contain JSON data
assert "overall_status" in user_content
# Verify the structured block was called
assert mock_structured_block.called
# The structured block should have been instantiated
mock_structured_block.assert_called_once()
@pytest.mark.asyncio
async def test_manager_integration_with_disabled_feature(

17
autogpt_platform/backend/backend/executor/manager.py
View File

@@ -696,7 +696,7 @@ class ExecutionProcessor:
exec_meta.status = status
# Activity status handling
activity_status = asyncio.run_coroutine_threadsafe(
activity_response = asyncio.run_coroutine_threadsafe(
generate_activity_status_for_execution(
graph_exec_id=graph_exec.graph_exec_id,
graph_id=graph_exec.graph_id,
@@ -708,18 +708,21 @@ class ExecutionProcessor:
),
self.node_execution_loop,
).result(timeout=60.0)
if activity_status is not None:
exec_stats.activity_status = activity_status
log_metadata.info(f"Generated activity status: {activity_status}")
if activity_response is not None:
exec_stats.activity_status = activity_response["activity_status"]
exec_stats.correctness_score = activity_response["correctness_score"]
log_metadata.info(
f"Generated activity status: {activity_response['activity_status']} "
f"(correctness: {activity_response['correctness_score']:.2f})"
)
else:
log_metadata.debug(
"Activity status generation disabled, not setting field"
"Activity status generation disabled, not setting fields"
)
finally:
# Communication handling
self._handle_agent_run_notif(db_client, graph_exec, exec_stats)
finally:
update_graph_execution_state(
db_client=db_client,
graph_exec_id=graph_exec.graph_exec_id,

5
autogpt_platform/frontend/src/app/api/openapi.json
View File

@@ -7821,6 +7821,11 @@
"anyOf": [{ "type": "string" }, { "type": "null" }],
"title": "Activity Status",
"description": "AI-generated summary of what the agent did"
},
"correctness_score": {
"anyOf": [{ "type": "number" }, { "type": "null" }],
"title": "Correctness Score",
"description": "AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose"
}
},
"additionalProperties": true,