Compare commits

...

5 Commits

Author SHA1 Message Date
openhands
0329c337ce Fix Python lint issues 2025-04-30 16:18:20 +00:00
openhands
7e038a3ced Fix error message handling in agent controller 2025-04-29 22:54:57 +00:00
openhands
5bddd0f0c2 Merge main into rb/err-details and fix merge conflicts 2025-04-29 12:38:02 +00:00
openhands
e4b81b8a87 Fix unit tests to match refactored error handling 2025-04-16 19:42:52 +00:00
Robert Brennan
4b0299be3a refactor err details 2025-04-07 17:24:37 -04:00
15 changed files with 148 additions and 124 deletions

View File

@@ -93,7 +93,7 @@ describe("HomeScreen", () => {
it("should have responsive layout for mobile and desktop screens", async () => {
renderHomeScreen();
const mainContainer = screen.getByTestId("home-screen").querySelector("main");
expect(mainContainer).toHaveClass("flex", "flex-col", "md:flex-row");
});

View File

@@ -260,39 +260,43 @@ class AgentController:
# Store the error reason before setting the agent state
self.state.last_error = f'{type(e).__name__}: {str(e)}'
if self.status_callback is not None:
err_id = ''
if isinstance(e, AuthenticationError):
err_id = 'STATUS$ERROR_LLM_AUTHENTICATION'
self.state.last_error = err_id
elif isinstance(
e,
(
ServiceUnavailableError,
APIConnectionError,
APIError,
),
):
err_id = 'STATUS$ERROR_LLM_SERVICE_UNAVAILABLE'
self.state.last_error = err_id
elif isinstance(e, InternalServerError):
err_id = 'STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR'
self.state.last_error = err_id
elif isinstance(e, BadRequestError) and 'ExceededBudget' in str(e):
err_id = 'STATUS$ERROR_LLM_OUT_OF_CREDITS'
self.state.last_error = err_id
elif isinstance(e, ContentPolicyViolationError) or (
isinstance(e, BadRequestError)
and 'ContentPolicyViolationError' in str(e)
):
err_id = 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
self.state.last_error = err_id
elif isinstance(e, RateLimitError):
await self.set_agent_state_to(AgentState.RATE_LIMITED)
return
self.status_callback('error', err_id, self.state.last_error)
if isinstance(e, RateLimitError):
await self.set_agent_state_to(AgentState.RATE_LIMITED)
return
err_id = ''
err_details = type(e).__name__
if isinstance(e, AuthenticationError):
err_id = 'STATUS$ERROR_LLM_AUTHENTICATION'
elif isinstance(
e,
(
ServiceUnavailableError,
APIConnectionError,
APIError,
),
):
err_id = 'STATUS$ERROR_LLM_SERVICE_UNAVAILABLE'
elif isinstance(e, InternalServerError):
err_id = 'STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR'
elif isinstance(e, BadRequestError) and 'ExceededBudget' in str(e):
err_id = 'STATUS$ERROR_LLM_OUT_OF_CREDITS'
elif isinstance(e, ContentPolicyViolationError) or (
isinstance(e, BadRequestError) and 'ContentPolicyViolationError' in str(e)
):
err_id = 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
if err_id:
# These err_details will end up on the frontend. We only plumb through known errors
# listed above to avoid exposing sensitive information
err_details = type(e).__name__ + ': ' + str(e)
self.state.last_error = err_details
else:
self.state.last_error = f'{type(e).__name__}: {str(e)}'
if self.status_callback is not None:
self.status_callback('error', err_id, err_details)
# Set the agent state to ERROR after storing the reason
await self.set_agent_state_to(AgentState.ERROR)
def step(self) -> None:

View File

@@ -3,4 +3,4 @@ Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retriev
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then use the {{ apiName }} to look at the {{ ciSystem }} that are failing on the most recent commit. Try and reproduce the failure locally.
Get things working locally, then push your changes. Sleep for 30 seconds at a time until the {{ ciProvider }} {{ ciSystem.lower() }} have run again.
If they are still failing, repeat the process.
If they are still failing, repeat the process.

View File

@@ -1,4 +1,4 @@
You are working on {{ requestType }} #{{ issue_number }} in repository {{ repo }}. You need to fix the merge conflicts.
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the {{ requestTypeShort }} details.
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then resolve the merge conflicts. If you aren't sure what the right solution is, look back through the commit history at the commits that introduced the conflict and resolve them accordingly.
Then resolve the merge conflicts. If you aren't sure what the right solution is, look back through the commit history at the commits that introduced the conflict and resolve them accordingly.

View File

@@ -1,4 +1,4 @@
You are working on Issue #{{ issue_number }} in repository {{ repo }}. Your goal is to fix the issue.
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the issue details and any comments on the issue.
Then check out a new branch and investigate what changes will need to be made.
Finally, make the required changes and open up a {{ requestVerb }}. Be sure to reference the issue in the {{ requestTypeShort }} description.
Finally, make the required changes and open up a {{ requestVerb }}. Be sure to reference the issue in the {{ requestTypeShort }} description.

View File

@@ -2,4 +2,4 @@ You are working on {{ requestType }} #{{ issue_number }} in repository {{ repo }
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the {{ requestTypeShort }} details.
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then use the {{ apiName }} to retrieve all the feedback on the {{ requestTypeShort }} so far.
If anything hasn't been addressed, address it and commit your changes back to the same branch.
If anything hasn't been addressed, address it and commit your changes back to the same branch.

View File

@@ -10,8 +10,8 @@ from openhands.events.event_store import EventStore
from openhands.server.config.server_config import ServerConfig
from openhands.server.monitoring import MonitoringListener
from openhands.server.session.conversation import Conversation
from openhands.storage.data_models.settings import Settings
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore

View File

@@ -18,9 +18,9 @@ from openhands.server.monitoring import MonitoringListener
from openhands.server.session.agent_session import WAIT_TIME_BEFORE_CLOSE
from openhands.server.session.conversation import Conversation
from openhands.server.session.session import ROOM_KEY, Session
from openhands.storage.data_models.settings import Settings
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.conversation_metadata import ConversationMetadata
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
from openhands.utils.async_utils import GENERAL_TIMEOUT, call_async_from_sync, wait_all
from openhands.utils.import_utils import get_impl

View File

@@ -42,7 +42,6 @@ from openhands.storage.data_models.conversation_status import ConversationStatus
from openhands.utils.async_utils import wait_all
from openhands.utils.conversation_summary import generate_conversation_title
app = APIRouter(prefix='/api')
@@ -54,7 +53,7 @@ class InitSessionRequest(BaseModel):
image_urls: list[str] | None = None
replay_json: str | None = None
suggested_task: SuggestedTask | None = None
async def _create_new_conversation(
user_id: str | None,
@@ -67,10 +66,14 @@ async def _create_new_conversation(
conversation_trigger: ConversationTrigger = ConversationTrigger.GUI,
attach_convo_id: bool = False,
):
print("trigger", conversation_trigger)
print('trigger', conversation_trigger)
logger.info(
'Creating conversation',
extra={'signal': 'create_conversation', 'user_id': user_id, 'trigger': conversation_trigger.value},
extra={
'signal': 'create_conversation',
'user_id': user_id,
'trigger': conversation_trigger.value,
},
)
logger.info('Loading settings')
settings_store = await SettingsStoreImpl.get_instance(config, user_id)
@@ -190,7 +193,7 @@ async def new_conversation(
initial_user_msg=initial_user_msg,
image_urls=image_urls,
replay_json=replay_json,
conversation_trigger=conversation_trigger
conversation_trigger=conversation_trigger,
)
return JSONResponse(

View File

@@ -2,9 +2,8 @@ from typing import Any
from fastapi import APIRouter
from openhands.security.options import SecurityAnalyzers
from openhands.controller.agent import Agent
from openhands.security.options import SecurityAnalyzers
from openhands.server.shared import config, server_config
from openhands.utils.llm import get_supported_llm_models

View File

@@ -17,13 +17,13 @@ from openhands.server.settings import (
POSTSettingsModel,
)
from openhands.server.shared import config
from openhands.storage.data_models.settings import Settings
from openhands.server.user_auth import (
get_provider_tokens,
get_user_id,
get_user_settings,
get_user_settings_store,
)
from openhands.storage.data_models.settings import Settings
from openhands.storage.settings.settings_store import SettingsStore
app = APIRouter(prefix='/api')

View File

@@ -4,8 +4,8 @@ import json
from dataclasses import dataclass
from openhands.core.config.app_config import AppConfig
from openhands.storage.data_models.settings import Settings
from openhands.storage import get_file_store
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
from openhands.storage.settings.settings_store import SettingsStore
from openhands.utils.async_utils import call_sync_from_async

View File

@@ -26,7 +26,7 @@ from openhands.resolver.resolver_output import ResolverOutput
@pytest.fixture
def default_mock_args():
"""Fixture that provides a default mock args object with common values.
Tests can override specific attributes as needed.
"""
mock_args = MagicMock()
@@ -53,10 +53,13 @@ def default_mock_args():
@pytest.fixture
def mock_github_token():
"""Fixture that patches the identify_token function to return GitHub provider type.
This eliminates the need for repeated patching in each test function.
"""
with patch('openhands.resolver.resolve_issue.identify_token', return_value=ProviderType.GITHUB) as patched:
with patch(
'openhands.resolver.resolve_issue.identify_token',
return_value=ProviderType.GITHUB,
) as patched:
yield patched
@@ -152,7 +155,9 @@ async def test_resolve_issue_no_issues_found(default_mock_args, mock_github_toke
# Verify that the handler was correctly configured and called
resolver.issue_handler_factory.assert_called_once()
mock_handler.get_converted_issues.assert_called_once_with(issue_numbers=[5432], comment_id=None)
mock_handler.get_converted_issues.assert_called_once_with(
issue_numbers=[5432], comment_id=None
)
def test_download_issues_from_github():
@@ -348,9 +353,7 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
# Create resolver with mocked token identification
resolver = IssueResolver(default_mock_args)
result = await resolver.complete_runtime(
mock_runtime, 'base_commit_hash'
)
result = await resolver.complete_runtime(mock_runtime, 'base_commit_hash')
assert result == {'git_patch': 'git diff content'}
assert mock_runtime.run_action.call_count == 5
@@ -358,7 +361,7 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"test_case",
'test_case',
[
{
'name': 'successful_run',
@@ -410,11 +413,20 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
'expected_error': None,
'expected_explanation': 'Non-JSON explanation',
'is_pr': True,
'comment_success': [True, False], # To trigger the PR success logging code path
'comment_success': [
True,
False,
], # To trigger the PR success logging code path
},
],
)
async def test_process_issue(default_mock_args, mock_github_token, mock_output_dir, mock_prompt_template, test_case):
async def test_process_issue(
default_mock_args,
mock_github_token,
mock_output_dir,
mock_prompt_template,
test_case,
):
"""Test the process_issue method with different scenarios."""
# Set up test data
@@ -426,7 +438,7 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
body='This is a test issue',
)
base_commit = 'abcdef1234567890'
# Customize the mock args for this test
default_mock_args.output_dir = mock_output_dir
default_mock_args.issue_type = 'pr' if test_case.get('is_pr', False) else 'issue'
@@ -457,7 +469,7 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
# Mock the create_runtime function
mock_create_runtime = MagicMock(return_value=mock_runtime)
# Mock the run_controller function
mock_run_controller = AsyncMock()
if test_case['run_controller_raises']:
@@ -466,14 +478,15 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
mock_run_controller.return_value = test_case['run_controller_return']
# Patch the necessary functions and methods
with patch('openhands.resolver.resolve_issue.create_runtime', mock_create_runtime), \
patch('openhands.resolver.resolve_issue.run_controller', mock_run_controller), \
patch.object(resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}), \
patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime:
with patch(
'openhands.resolver.resolve_issue.create_runtime', mock_create_runtime
), patch(
'openhands.resolver.resolve_issue.run_controller', mock_run_controller
), patch.object(
resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}
), patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime:
# Call the process_issue method
result = await resolver.process_issue(issue, base_commit, handler_instance)
# Assert the result matches our expectations
assert isinstance(result, ResolverOutput)
@@ -490,16 +503,17 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
mock_initialize_runtime.assert_called_once()
mock_run_controller.assert_called_once()
resolver.complete_runtime.assert_awaited_once_with(mock_runtime, base_commit)
# Assert run_controller was called with the right parameters
if not test_case['run_controller_raises']:
# Check that the first positional argument is a config
assert 'config' in mock_run_controller.call_args[1]
# Check that initial_user_action is a MessageAction with the right content
assert isinstance(mock_run_controller.call_args[1]['initial_user_action'], MessageAction)
assert isinstance(
mock_run_controller.call_args[1]['initial_user_action'], MessageAction
)
assert mock_run_controller.call_args[1]['runtime'] == mock_runtime
# Assert that guess_success was called only for successful runs
if test_case['expected_success']:
handler_instance.guess_success.assert_called_once()

View File

@@ -19,14 +19,16 @@ from openhands.resolver.interfaces.issue_definitions import (
ServiceContextIssue,
ServiceContextPR,
)
from openhands.resolver.resolve_issue import IssueResolver, SandboxConfig, AppConfig, AgentConfig
from openhands.resolver.resolve_issue import (
IssueResolver,
)
from openhands.resolver.resolver_output import ResolverOutput
@pytest.fixture
def default_mock_args():
"""Fixture that provides a default mock args object with common values.
Tests can override specific attributes as needed.
"""
mock_args = MagicMock()
@@ -52,10 +54,13 @@ def default_mock_args():
@pytest.fixture
def mock_gitlab_token():
"""Fixture that patches the identify_token function to return GitLab provider type.
This eliminates the need for repeated patching in each test function.
"""
with patch('openhands.resolver.resolve_issue.identify_token', return_value=ProviderType.GITLAB) as patched:
with patch(
'openhands.resolver.resolve_issue.identify_token',
return_value=ProviderType.GITLAB,
) as patched:
yield patched
@@ -124,10 +129,10 @@ def test_initialize_runtime(default_mock_args, mock_gitlab_token):
exit_code=0, content='', command='git config --global core.pager ""'
),
]
# Create resolver with mocked token identification
resolver = IssueResolver(default_mock_args)
resolver.initialize_runtime(mock_runtime)
if os.getenv('GITLAB_CI') == 'true':
@@ -154,24 +159,26 @@ async def test_resolve_issue_no_issues_found(default_mock_args, mock_gitlab_toke
# Customize the mock args for this test
default_mock_args.issue_number = 5432
# Create a resolver instance with mocked token identification
resolver = IssueResolver(default_mock_args)
# Mock the issue_handler_factory method
resolver.issue_handler_factory = MagicMock(return_value=mock_handler)
# Test that the correct exception is raised
with pytest.raises(ValueError) as exc_info:
await resolver.resolve_issue()
# Verify the error message
assert 'No issues found for issue number 5432' in str(exc_info.value)
assert 'test-owner/test-repo' in str(exc_info.value)
# Verify that the handler was correctly configured and called
resolver.issue_handler_factory.assert_called_once()
mock_handler.get_converted_issues.assert_called_once_with(issue_numbers=[5432], comment_id=None)
mock_handler.get_converted_issues.assert_called_once_with(
issue_numbers=[5432], comment_id=None
)
def test_download_issues_from_gitlab():
@@ -377,12 +384,14 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
content='',
command='git config --global --add safe.directory /workspace',
),
create_cmd_output(exit_code=0, content='', command='git add -A'),
create_cmd_output(
exit_code=0, content='', command='git add -A'
exit_code=0,
content='git diff content',
command='git diff --no-color --cached base_commit_hash',
),
create_cmd_output(exit_code=0, content='git diff content', command='git diff --no-color --cached base_commit_hash'),
]
# Create a resolver instance with mocked token identification
resolver = IssueResolver(default_mock_args)
@@ -394,7 +403,7 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"test_case",
'test_case',
[
{
'name': 'successful_run',
@@ -448,7 +457,13 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
},
],
)
async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_dir, mock_prompt_template, test_case):
async def test_process_issue(
default_mock_args,
mock_gitlab_token,
mock_output_dir,
mock_prompt_template,
test_case,
):
"""Test the process_issue method with different scenarios."""
# Set up test data
issue = Issue(
@@ -482,7 +497,7 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
mock_runtime = MagicMock()
mock_runtime.connect = AsyncMock()
mock_create_runtime = MagicMock(return_value=mock_runtime)
# Configure run_controller mock based on test case
mock_run_controller = AsyncMock()
if test_case.get('run_controller_raises'):
@@ -491,16 +506,18 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
mock_run_controller.return_value = test_case['run_controller_return']
# Patch the necessary functions and methods
with patch('openhands.resolver.resolve_issue.create_runtime', mock_create_runtime), \
patch('openhands.resolver.resolve_issue.run_controller', mock_run_controller), \
patch.object(resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}), \
patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime, \
patch('openhands.resolver.resolve_issue.SandboxConfig', return_value=MagicMock()), \
patch('openhands.resolver.resolve_issue.AppConfig', return_value=MagicMock()):
with patch(
'openhands.resolver.resolve_issue.create_runtime', mock_create_runtime
), patch(
'openhands.resolver.resolve_issue.run_controller', mock_run_controller
), patch.object(
resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}
), patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime, patch(
'openhands.resolver.resolve_issue.SandboxConfig', return_value=MagicMock()
), patch('openhands.resolver.resolve_issue.AppConfig', return_value=MagicMock()):
# Call the process_issue method
result = await resolver.process_issue(issue, base_commit, handler_instance)
mock_create_runtime.assert_called_once()
mock_runtime.connect.assert_called_once()
mock_initialize_runtime.assert_called_once()
@@ -521,6 +538,7 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
else:
handler_instance.guess_success.assert_not_called()
def test_get_instruction(mock_prompt_template, mock_followup_prompt_template):
issue = Issue(
owner='test_owner',
@@ -923,4 +941,4 @@ def test_download_issue_with_specific_comment():
if __name__ == '__main__':
pytest.main()
pytest.main()

View File

@@ -204,11 +204,14 @@ async def test_react_to_content_policy_violation(
mock_status_callback.assert_called_once_with(
'error',
'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION',
'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION',
'ContentPolicyViolationError: litellm.BadRequestError: litellm.ContentPolicyViolationError: Output blocked by content filtering policy',
)
# Verify the state was updated correctly
assert controller.state.last_error == 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
assert (
controller.state.last_error
== 'ContentPolicyViolationError: litellm.BadRequestError: litellm.ContentPolicyViolationError: Output blocked by content filtering policy'
)
assert controller.state.agent_state == AgentState.ERROR
await controller.close()
@@ -272,10 +275,8 @@ async def test_run_controller_with_fatal_error(
error_observation = error_observations[0]
assert state.iteration == 3
assert state.agent_state == AgentState.ERROR
assert state.last_error == 'AgentStuckInLoopError: Agent got stuck in a loop'
assert (
error_observation.reason == 'AgentStuckInLoopError: Agent got stuck in a loop'
)
assert state.last_error == 'AgentStuckInLoopError'
assert error_observation.reason == 'AgentStuckInLoopError'
assert len(events) == 12
@@ -355,7 +356,7 @@ async def test_run_controller_stop_with_stuck(
assert last_event['observation'] == 'agent_state_changed'
assert state.agent_state == AgentState.ERROR
assert state.last_error == 'AgentStuckInLoopError: Agent got stuck in a loop'
assert state.last_error == 'AgentStuckInLoopError'
@pytest.mark.asyncio
@@ -688,20 +689,14 @@ async def test_run_controller_max_iterations_has_metrics(
)
assert state.iteration == 3
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 3, max iteration: 3'
)
assert 'RuntimeError' in state.last_error
error_observations = test_event_stream.get_matching_events(
reverse=True, limit=1, event_types=(AgentStateChangedObservation)
)
assert len(error_observations) == 1
error_observation = error_observations[0]
assert (
error_observation.reason
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 3, max iteration: 3'
)
assert 'RuntimeError' in error_observation.reason
assert (
state.metrics.accumulated_cost == 10.0 * 3
@@ -945,10 +940,7 @@ async def test_run_controller_with_context_window_exceeded_with_truncation(
# expected reason
assert state.iteration == 5
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 5, max iteration: 5'
)
assert state.last_error == 'RuntimeError'
# Check that the context window exceeded error was raised during the run
assert step_state.has_errored
@@ -1022,20 +1014,14 @@ async def test_run_controller_with_context_window_exceeded_without_truncation(
# With the refactored system message handling, the iteration count is different
assert state.iteration == 1
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'LLMContextWindowExceedError: Conversation history longer than LLM context window limit. Consider turning on enable_history_truncation config to avoid this error'
)
assert state.last_error == 'LLMContextWindowExceedError'
error_observations = test_event_stream.get_matching_events(
reverse=True, limit=1, event_types=(AgentStateChangedObservation)
)
assert len(error_observations) == 1
error_observation = error_observations[0]
assert (
error_observation.reason
== 'LLMContextWindowExceedError: Conversation history longer than LLM context window limit. Consider turning on enable_history_truncation config to avoid this error'
)
assert 'LLMContextWindowExceedError' in error_observation.reason
# Check that the context window exceeded error was raised during the run
assert step_state.has_errored