Compare commits

..

5 Commits

Author SHA1 Message Date
openhands 0329c337ce Fix Python lint issues 2025-04-30 16:18:20 +00:00
openhands 7e038a3ced Fix error message handling in agent controller 2025-04-29 22:54:57 +00:00
openhands 5bddd0f0c2 Merge main into rb/err-details and fix merge conflicts 2025-04-29 12:38:02 +00:00
openhands e4b81b8a87 Fix unit tests to match refactored error handling 2025-04-16 19:42:52 +00:00
Robert Brennan 4b0299be3a refactor err details 2025-04-07 17:24:37 -04:00
17 changed files with 264 additions and 191 deletions
@@ -1 +0,0 @@
# Using GitLab CI Runners
@@ -93,7 +93,7 @@ describe("HomeScreen", () => {
it("should have responsive layout for mobile and desktop screens", async () => {
renderHomeScreen();
const mainContainer = screen.getByTestId("home-screen").querySelector("main");
expect(mainContainer).toHaveClass("flex", "flex-col", "md:flex-row");
});
+36 -32
View File
@@ -260,39 +260,43 @@ class AgentController:
# Store the error reason before setting the agent state
self.state.last_error = f'{type(e).__name__}: {str(e)}'
if self.status_callback is not None:
err_id = ''
if isinstance(e, AuthenticationError):
err_id = 'STATUS$ERROR_LLM_AUTHENTICATION'
self.state.last_error = err_id
elif isinstance(
e,
(
ServiceUnavailableError,
APIConnectionError,
APIError,
),
):
err_id = 'STATUS$ERROR_LLM_SERVICE_UNAVAILABLE'
self.state.last_error = err_id
elif isinstance(e, InternalServerError):
err_id = 'STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR'
self.state.last_error = err_id
elif isinstance(e, BadRequestError) and 'ExceededBudget' in str(e):
err_id = 'STATUS$ERROR_LLM_OUT_OF_CREDITS'
self.state.last_error = err_id
elif isinstance(e, ContentPolicyViolationError) or (
isinstance(e, BadRequestError)
and 'ContentPolicyViolationError' in str(e)
):
err_id = 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
self.state.last_error = err_id
elif isinstance(e, RateLimitError):
await self.set_agent_state_to(AgentState.RATE_LIMITED)
return
self.status_callback('error', err_id, self.state.last_error)
if isinstance(e, RateLimitError):
await self.set_agent_state_to(AgentState.RATE_LIMITED)
return
err_id = ''
err_details = type(e).__name__
if isinstance(e, AuthenticationError):
err_id = 'STATUS$ERROR_LLM_AUTHENTICATION'
elif isinstance(
e,
(
ServiceUnavailableError,
APIConnectionError,
APIError,
),
):
err_id = 'STATUS$ERROR_LLM_SERVICE_UNAVAILABLE'
elif isinstance(e, InternalServerError):
err_id = 'STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR'
elif isinstance(e, BadRequestError) and 'ExceededBudget' in str(e):
err_id = 'STATUS$ERROR_LLM_OUT_OF_CREDITS'
elif isinstance(e, ContentPolicyViolationError) or (
isinstance(e, BadRequestError) and 'ContentPolicyViolationError' in str(e)
):
err_id = 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
if err_id:
# These err_details will end up on the frontend. We only plumb through known errors
# listed above to avoid exposing sensitive information
err_details = type(e).__name__ + ': ' + str(e)
self.state.last_error = err_details
else:
self.state.last_error = f'{type(e).__name__}: {str(e)}'
if self.status_callback is not None:
self.status_callback('error', err_id, err_details)
# Set the agent state to ERROR after storing the reason
await self.set_agent_state_to(AgentState.ERROR)
def step(self) -> None:
@@ -3,4 +3,4 @@ Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retriev
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then use the {{ apiName }} to look at the {{ ciSystem }} that are failing on the most recent commit. Try and reproduce the failure locally.
Get things working locally, then push your changes. Sleep for 30 seconds at a time until the {{ ciProvider }} {{ ciSystem.lower() }} have run again.
If they are still failing, repeat the process.
If they are still failing, repeat the process.
@@ -1,4 +1,4 @@
You are working on {{ requestType }} #{{ issue_number }} in repository {{ repo }}. You need to fix the merge conflicts.
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the {{ requestTypeShort }} details.
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then resolve the merge conflicts. If you aren't sure what the right solution is, look back through the commit history at the commits that introduced the conflict and resolve them accordingly.
Then resolve the merge conflicts. If you aren't sure what the right solution is, look back through the commit history at the commits that introduced the conflict and resolve them accordingly.
@@ -1,4 +1,4 @@
You are working on Issue #{{ issue_number }} in repository {{ repo }}. Your goal is to fix the issue.
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the issue details and any comments on the issue.
Then check out a new branch and investigate what changes will need to be made.
Finally, make the required changes and open up a {{ requestVerb }}. Be sure to reference the issue in the {{ requestTypeShort }} description.
Finally, make the required changes and open up a {{ requestVerb }}. Be sure to reference the issue in the {{ requestTypeShort }} description.
@@ -2,4 +2,4 @@ You are working on {{ requestType }} #{{ issue_number }} in repository {{ repo }
Use the {{ apiName }} with the {{ tokenEnvVar }} environment variable to retrieve the {{ requestTypeShort }} details.
Check out the branch from that {{ requestVerb }} and look at the diff versus the base branch of the {{ requestTypeShort }} to understand the {{ requestTypeShort }}'s intention.
Then use the {{ apiName }} to retrieve all the feedback on the {{ requestTypeShort }} so far.
If anything hasn't been addressed, address it and commit your changes back to the same branch.
If anything hasn't been addressed, address it and commit your changes back to the same branch.
+116 -66
View File
@@ -1,141 +1,191 @@
# OpenHands GitHub & GitLab Issue Resolver 🙌
# OpenHands Github & Gitlab Issue Resolver 🙌
Need help resolving GitHub or GitLab issues? Let an AI agent help you out!
Need help resolving a GitHub issue but don't have the time to do it yourself? Let an AI agent help you out!
This tool uses [OpenHands](https://github.com/all-hands-ai/openhands) AI agents to automatically resolve issues in your repositories. It's designed to handle one issue at a time with high quality.
This tool allows you to use open-source AI agents based on [OpenHands](https://github.com/all-hands-ai/openhands)
to attempt to resolve GitHub issues automatically. While it can handle multiple issues, it's primarily designed
to help you resolve one issue at a time with high quality.
## 1. Setting Up for GitHub (Action Workflow)
Getting started is simple - just follow the instructions below.
### Prerequisites
## Using the GitHub Actions Workflow
- [Create a personal access token](https://github.com/settings/tokens?type=beta) with read/write scope for
This repository includes a GitHub Actions workflow that can automatically attempt to fix individual issues labeled with 'fix-me'.
Follow these steps to use this workflow in your own repository:
- "contents"
- "issues"
- "pull requests"
- "workflows"
1. [Create a personal access token](https://github.com/settings/tokens?type=beta) with read/write scope for "contents", "issues", "pull requests", and "workflows"
- Create an LLM API key (e,g [Claude API](https://www.anthropic.com/api))
Note: If you're working with an organizational repository, you may need to configure the organization's personal access token policy first. See [Setting a personal access token policy for your organization](https://docs.github.com/en/organizations/managing-programmatic-access-to-your-organization/setting-a-personal-access-token-policy-for-your-organization) for details.
### Installation
2. Create an API key for the [Claude API](https://www.anthropic.com/api) (recommended) or another supported LLM service
1. Copy `examples/openhands-resolver.yml` to your repository's `.github/workflows/` directory
3. Copy `examples/openhands-resolver.yml` to your repository's `.github/workflows/` directory
2. Configure repository permissions:
4. Configure repository permissions:
- Go to `Settings -> Actions -> General -> Workflow permissions`
- Select "Read and write permissions"
- Enable "Allow Github Actions to create and approve pull requests"
- Go to `Settings -> Actions -> General -> Workflow permissions`
- Select **Read and write permissions**
- Enable **Allow Github Actions to create and approve pull requests**
> If "Read and write permissions" is greyed out:
>
> - Check organization settings first
> - Otherwise, permissions might need to be set in [Enterprise policy settings](https://docs.github.com/en/enterprise-cloud@latest/admin/enforcing-policies/enforcing-policies-for-your-enterprise/enforcing-policies-for-github-actions-in-your-enterprise#enforcing-a-policy-for-workflow-permissions-in-your-enterprise)
3. Set up [GitHub secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions):
Note: If the "Read and write permissions" option is greyed out:
- First check if permissions need to be set at the organization level
- If still greyed out at the organization level, permissions need to be set in the [Enterprise policy settings](https://docs.github.com/en/enterprise-cloud@latest/admin/enforcing-policies/enforcing-policies-for-your-enterprise/enforcing-policies-for-github-actions-in-your-enterprise#enforcing-a-policy-for-workflow-permissions-in-your-enterprise)
5. Set up [GitHub secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions):
- Required:
- `LLM_API_KEY`: Your LLM API key
- `LLM_API_KEY`: Your LLM API key
- Optional:
- `PAT_USERNAME`: GitHub username for the personal access token
- `PAT_TOKEN`: The personal access token
- `LLM_BASE_URL`: Base URL for LLM API (only if using a proxy)
- [See how to customize more configurations](https://docs.all-hands.dev/modules/usage/how-to/github-action#custom-configurations)
## 2. Setting up GitLab (CI Runner)
Note: You can set these secrets at the organization level to use across multiple repositories.
### Prerequisites
6. Set up any [custom configurations required](https://docs.all-hands.dev/modules/usage/how-to/github-action#custom-configurations)
Create a GitLab Personal Access Token with API, read/write access
7. Usage:
There are two ways to trigger the OpenHands agent:
### Installation
a. Using the 'fix-me' label:
- Add the 'fix-me' label to any issue you want the AI to resolve
- The agent will consider all comments in the issue thread when resolving
- The workflow will:
1. Attempt to resolve the issue using OpenHands
2. Create a draft PR if successful, or push a branch if unsuccessful
3. Comment on the issue with the results
4. Remove the 'fix-me' label once processed
## 3. Triggering OpenHands Agent
b. Using `@openhands-agent` mention:
- Create a new comment containing `@openhands-agent` in any issue
- The agent will only consider the comment where it's mentioned
- The workflow will:
1. Attempt to resolve the issue based on the specific comment
2. Create a draft PR if successful, or push a branch if unsuccessful
3. Comment on the issue with the results
You can trigger OpenHands in two shared ways (works for both GitHub and GitLab):
Need help? Feel free to [open an issue](https://github.com/all-hands-ai/openhands/issues) or email us at [contact@all-hands.dev](mailto:contact@all-hands.dev).
Using the 'fix-me' label:
## Manual Installation
- Add the 'fix-me' label to any issue you want the AI to resolve
- The agent will consider all comments in the issue thread when resolving
If you prefer to run the resolver programmatically instead of using GitHub Actions, follow these steps:
Using `@openhands-agent` in an issue/pr comment:
- Create a new comment containing `@openhands-agent`
- The agent will only consider the comment + comment thread where it's mentioned
## 4. Running Locally
### Installation
1. Install the package:
```bash
pip install openhands-ai
```
### Setup
2. Create a GitHub or GitLab access token:
- Create a GitHub acces token
- Visit [GitHub's token settings](https://github.com/settings/personal-access-tokens/new)
- Create a fine-grained token with these scopes:
- "Content"
- "Pull requests"
- "Issues"
- "Workflows"
- If you don't have push access to the target repo, you can fork it first
Create a GitHub or GitLab access token with appropriate permissions
- Create a GitLab acces token
- Visit [GitLab's token settings](https://gitlab.com/-/user_settings/personal_access_tokens)
- Create a fine-grained token with these scopes:
- 'api'
- 'read_api'
- 'read_user'
- 'read_repository'
- 'write_repository'
Set up environment variables:
3. Set up environment variables:
```bash
# GitHub credentials
export GITHUB_TOKEN="your-github-token"
export GIT_USERNAME="your-github-username"
# GitLab credentials (if using GitLab)
# GitHub credentials
export GITHUB_TOKEN="your-github-token"
export GIT_USERNAME="your-github-username" # Optional, defaults to token owner
# GitLab credentials if you're using GitLab repo
export GITLAB_TOKEN="your-gitlab-token"
export GIT_USERNAME="your-gitlab-username"
export GIT_USERNAME="your-gitlab-username" # Optional, defaults to token owner
# LLM configuration
export LLM_MODEL="anthropic/claude-3-5-sonnet-20241022"
export LLM_MODEL="anthropic/claude-3-5-sonnet-20241022" # Recommended
export LLM_API_KEY="your-llm-api-key"
export LLM_BASE_URL="your-api-url" # Optional
export LLM_BASE_URL="your-api-url" # Optional, for API proxies
```
### Resolving Issues
Note: OpenHands works best with powerful models like Anthropic's Claude or OpenAI's GPT-4. While other models are supported, they may not perform as well for complex issue resolution.
Resolve a single issue:
## Resolving Issues
The resolver can automatically attempt to fix a single issue in your repository using the following command:
```bash
python -m openhands.resolver.resolve_issue --selected-repo [OWNER]/[REPO] --issue-number [NUMBER]
```
### Responding to PR Comments
For instance, if you want to resolve issue #100 in this repo, you would run:
Respond to comments on pull requests:
```bash
python -m openhands.resolver.resolve_issue --selected-repo all-hands-ai/openhands --issue-number 100
```
The output will be written to the `output/` directory.
If you've installed the package from source using poetry, you can use:
```bash
poetry run python openhands/resolver/resolve_issue.py --selected-repo all-hands-ai/openhands --issue-number 100
```
## Responding to PR Comments
The resolver can also respond to comments on pull requests using:
```bash
python -m openhands.resolver.send_pull_request --issue-number PR_NUMBER --issue-type pr
```
### Visualizing Results
This functionality is available both through the GitHub Actions workflow and when running the resolver locally.
View successful PRs:
## Visualizing successful PRs
To find successful PRs, you can run the following command:
```bash
grep '"success":true' output/output.jsonl | sed 's/.*\("number":[0-9]*\).*/\1/g'
```
Visualize specific PR:
Then you can go through and visualize the ones you'd like.
```bash
python -m openhands.resolver.visualize_resolver_output --issue-number ISSUE_NUMBER --vis-method json
```
### Uploading PRs
## Uploading PRs
Upload your changes in one of three ways:
If you find any PRs that were successful, you can upload them.
There are three ways you can upload:
1. `branch` - upload a branch without creating a PR
2. `draft` - create a draft PR
3. `ready` - create a non-draft PR that's ready for review
```bash
python -m openhands.resolver.send_pull_request --issue-number ISSUE_NUMBER --username YOUR_GITHUB_OR_GITLAB_USERNAME --pr-type [branch|draft|ready]
python -m openhands.resolver.send_pull_request --issue-number ISSUE_NUMBER --username YOUR_GITHUB_OR_GITLAB_USERNAME --pr-type draft
```
## Custom Instructions
If you want to upload to a fork, you can do so by specifying the `fork-owner`:
Add repository-specific instructions by creating a file at `.openhands/microagents/repo.md` in your repository. For more information about repository microagents, see [Repository Instructions](https://github.com/All-Hands-AI/OpenHands/tree/main/microagents#2-repository-instructions-private).
```bash
python -m openhands.resolver.send_pull_request --issue-number ISSUE_NUMBER --username YOUR_GITHUB_OR_GITLAB_USERNAME --pr-type draft --fork-owner YOUR_GITHUB_OR_GITLAB_USERNAME
```
## Providing Custom Instructions
You can customize how the AI agent approaches issue resolution by adding a repository microagent file at `.openhands/microagents/repo.md` in your repository. This file's contents will be automatically loaded in the prompt when working with your repository. For more information about repository microagents, see [Repository Instructions](https://github.com/All-Hands-AI/OpenHands/tree/main/microagents#2-repository-instructions-private).
## Troubleshooting
If you have any issues, please open an issue on this github repo, we're happy to help!
If you have any issues, please open an issue on this github or gitlab repo, we're happy to help!
Alternatively, you can [email us](mailto:contact@all-hands.dev) or join the OpenHands Slack workspace (see [the README](/README.md) for an invite link).
@@ -10,8 +10,8 @@ from openhands.events.event_store import EventStore
from openhands.server.config.server_config import ServerConfig
from openhands.server.monitoring import MonitoringListener
from openhands.server.session.conversation import Conversation
from openhands.storage.data_models.settings import Settings
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
@@ -18,9 +18,9 @@ from openhands.server.monitoring import MonitoringListener
from openhands.server.session.agent_session import WAIT_TIME_BEFORE_CLOSE
from openhands.server.session.conversation import Conversation
from openhands.server.session.session import ROOM_KEY, Session
from openhands.storage.data_models.settings import Settings
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.storage.data_models.conversation_metadata import ConversationMetadata
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
from openhands.utils.async_utils import GENERAL_TIMEOUT, call_async_from_sync, wait_all
from openhands.utils.import_utils import get_impl
@@ -42,7 +42,6 @@ from openhands.storage.data_models.conversation_status import ConversationStatus
from openhands.utils.async_utils import wait_all
from openhands.utils.conversation_summary import generate_conversation_title
app = APIRouter(prefix='/api')
@@ -54,7 +53,7 @@ class InitSessionRequest(BaseModel):
image_urls: list[str] | None = None
replay_json: str | None = None
suggested_task: SuggestedTask | None = None
async def _create_new_conversation(
user_id: str | None,
@@ -67,10 +66,14 @@ async def _create_new_conversation(
conversation_trigger: ConversationTrigger = ConversationTrigger.GUI,
attach_convo_id: bool = False,
):
print("trigger", conversation_trigger)
print('trigger', conversation_trigger)
logger.info(
'Creating conversation',
extra={'signal': 'create_conversation', 'user_id': user_id, 'trigger': conversation_trigger.value},
extra={
'signal': 'create_conversation',
'user_id': user_id,
'trigger': conversation_trigger.value,
},
)
logger.info('Loading settings')
settings_store = await SettingsStoreImpl.get_instance(config, user_id)
@@ -190,7 +193,7 @@ async def new_conversation(
initial_user_msg=initial_user_msg,
image_urls=image_urls,
replay_json=replay_json,
conversation_trigger=conversation_trigger
conversation_trigger=conversation_trigger,
)
return JSONResponse(
+1 -2
View File
@@ -2,9 +2,8 @@ from typing import Any
from fastapi import APIRouter
from openhands.security.options import SecurityAnalyzers
from openhands.controller.agent import Agent
from openhands.security.options import SecurityAnalyzers
from openhands.server.shared import config, server_config
from openhands.utils.llm import get_supported_llm_models
+1 -1
View File
@@ -17,13 +17,13 @@ from openhands.server.settings import (
POSTSettingsModel,
)
from openhands.server.shared import config
from openhands.storage.data_models.settings import Settings
from openhands.server.user_auth import (
get_provider_tokens,
get_user_id,
get_user_settings,
get_user_settings_store,
)
from openhands.storage.data_models.settings import Settings
from openhands.storage.settings.settings_store import SettingsStore
app = APIRouter(prefix='/api')
@@ -4,8 +4,8 @@ import json
from dataclasses import dataclass
from openhands.core.config.app_config import AppConfig
from openhands.storage.data_models.settings import Settings
from openhands.storage import get_file_store
from openhands.storage.data_models.settings import Settings
from openhands.storage.files import FileStore
from openhands.storage.settings.settings_store import SettingsStore
from openhands.utils.async_utils import call_sync_from_async
@@ -26,7 +26,7 @@ from openhands.resolver.resolver_output import ResolverOutput
@pytest.fixture
def default_mock_args():
"""Fixture that provides a default mock args object with common values.
Tests can override specific attributes as needed.
"""
mock_args = MagicMock()
@@ -53,10 +53,13 @@ def default_mock_args():
@pytest.fixture
def mock_github_token():
"""Fixture that patches the identify_token function to return GitHub provider type.
This eliminates the need for repeated patching in each test function.
"""
with patch('openhands.resolver.resolve_issue.identify_token', return_value=ProviderType.GITHUB) as patched:
with patch(
'openhands.resolver.resolve_issue.identify_token',
return_value=ProviderType.GITHUB,
) as patched:
yield patched
@@ -152,7 +155,9 @@ async def test_resolve_issue_no_issues_found(default_mock_args, mock_github_toke
# Verify that the handler was correctly configured and called
resolver.issue_handler_factory.assert_called_once()
mock_handler.get_converted_issues.assert_called_once_with(issue_numbers=[5432], comment_id=None)
mock_handler.get_converted_issues.assert_called_once_with(
issue_numbers=[5432], comment_id=None
)
def test_download_issues_from_github():
@@ -348,9 +353,7 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
# Create resolver with mocked token identification
resolver = IssueResolver(default_mock_args)
result = await resolver.complete_runtime(
mock_runtime, 'base_commit_hash'
)
result = await resolver.complete_runtime(mock_runtime, 'base_commit_hash')
assert result == {'git_patch': 'git diff content'}
assert mock_runtime.run_action.call_count == 5
@@ -358,7 +361,7 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"test_case",
'test_case',
[
{
'name': 'successful_run',
@@ -410,11 +413,20 @@ async def test_complete_runtime(default_mock_args, mock_github_token):
'expected_error': None,
'expected_explanation': 'Non-JSON explanation',
'is_pr': True,
'comment_success': [True, False], # To trigger the PR success logging code path
'comment_success': [
True,
False,
], # To trigger the PR success logging code path
},
],
)
async def test_process_issue(default_mock_args, mock_github_token, mock_output_dir, mock_prompt_template, test_case):
async def test_process_issue(
default_mock_args,
mock_github_token,
mock_output_dir,
mock_prompt_template,
test_case,
):
"""Test the process_issue method with different scenarios."""
# Set up test data
@@ -426,7 +438,7 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
body='This is a test issue',
)
base_commit = 'abcdef1234567890'
# Customize the mock args for this test
default_mock_args.output_dir = mock_output_dir
default_mock_args.issue_type = 'pr' if test_case.get('is_pr', False) else 'issue'
@@ -457,7 +469,7 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
# Mock the create_runtime function
mock_create_runtime = MagicMock(return_value=mock_runtime)
# Mock the run_controller function
mock_run_controller = AsyncMock()
if test_case['run_controller_raises']:
@@ -466,14 +478,15 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
mock_run_controller.return_value = test_case['run_controller_return']
# Patch the necessary functions and methods
with patch('openhands.resolver.resolve_issue.create_runtime', mock_create_runtime), \
patch('openhands.resolver.resolve_issue.run_controller', mock_run_controller), \
patch.object(resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}), \
patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime:
with patch(
'openhands.resolver.resolve_issue.create_runtime', mock_create_runtime
), patch(
'openhands.resolver.resolve_issue.run_controller', mock_run_controller
), patch.object(
resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}
), patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime:
# Call the process_issue method
result = await resolver.process_issue(issue, base_commit, handler_instance)
# Assert the result matches our expectations
assert isinstance(result, ResolverOutput)
@@ -490,16 +503,17 @@ async def test_process_issue(default_mock_args, mock_github_token, mock_output_d
mock_initialize_runtime.assert_called_once()
mock_run_controller.assert_called_once()
resolver.complete_runtime.assert_awaited_once_with(mock_runtime, base_commit)
# Assert run_controller was called with the right parameters
if not test_case['run_controller_raises']:
# Check that the first positional argument is a config
assert 'config' in mock_run_controller.call_args[1]
# Check that initial_user_action is a MessageAction with the right content
assert isinstance(mock_run_controller.call_args[1]['initial_user_action'], MessageAction)
assert isinstance(
mock_run_controller.call_args[1]['initial_user_action'], MessageAction
)
assert mock_run_controller.call_args[1]['runtime'] == mock_runtime
# Assert that guess_success was called only for successful runs
if test_case['expected_success']:
handler_instance.guess_success.assert_called_once()
@@ -19,14 +19,16 @@ from openhands.resolver.interfaces.issue_definitions import (
ServiceContextIssue,
ServiceContextPR,
)
from openhands.resolver.resolve_issue import IssueResolver, SandboxConfig, AppConfig, AgentConfig
from openhands.resolver.resolve_issue import (
IssueResolver,
)
from openhands.resolver.resolver_output import ResolverOutput
@pytest.fixture
def default_mock_args():
"""Fixture that provides a default mock args object with common values.
Tests can override specific attributes as needed.
"""
mock_args = MagicMock()
@@ -52,10 +54,13 @@ def default_mock_args():
@pytest.fixture
def mock_gitlab_token():
"""Fixture that patches the identify_token function to return GitLab provider type.
This eliminates the need for repeated patching in each test function.
"""
with patch('openhands.resolver.resolve_issue.identify_token', return_value=ProviderType.GITLAB) as patched:
with patch(
'openhands.resolver.resolve_issue.identify_token',
return_value=ProviderType.GITLAB,
) as patched:
yield patched
@@ -124,10 +129,10 @@ def test_initialize_runtime(default_mock_args, mock_gitlab_token):
exit_code=0, content='', command='git config --global core.pager ""'
),
]
# Create resolver with mocked token identification
resolver = IssueResolver(default_mock_args)
resolver.initialize_runtime(mock_runtime)
if os.getenv('GITLAB_CI') == 'true':
@@ -154,24 +159,26 @@ async def test_resolve_issue_no_issues_found(default_mock_args, mock_gitlab_toke
# Customize the mock args for this test
default_mock_args.issue_number = 5432
# Create a resolver instance with mocked token identification
resolver = IssueResolver(default_mock_args)
# Mock the issue_handler_factory method
resolver.issue_handler_factory = MagicMock(return_value=mock_handler)
# Test that the correct exception is raised
with pytest.raises(ValueError) as exc_info:
await resolver.resolve_issue()
# Verify the error message
assert 'No issues found for issue number 5432' in str(exc_info.value)
assert 'test-owner/test-repo' in str(exc_info.value)
# Verify that the handler was correctly configured and called
resolver.issue_handler_factory.assert_called_once()
mock_handler.get_converted_issues.assert_called_once_with(issue_numbers=[5432], comment_id=None)
mock_handler.get_converted_issues.assert_called_once_with(
issue_numbers=[5432], comment_id=None
)
def test_download_issues_from_gitlab():
@@ -377,12 +384,14 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
content='',
command='git config --global --add safe.directory /workspace',
),
create_cmd_output(exit_code=0, content='', command='git add -A'),
create_cmd_output(
exit_code=0, content='', command='git add -A'
exit_code=0,
content='git diff content',
command='git diff --no-color --cached base_commit_hash',
),
create_cmd_output(exit_code=0, content='git diff content', command='git diff --no-color --cached base_commit_hash'),
]
# Create a resolver instance with mocked token identification
resolver = IssueResolver(default_mock_args)
@@ -394,7 +403,7 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"test_case",
'test_case',
[
{
'name': 'successful_run',
@@ -448,7 +457,13 @@ async def test_complete_runtime(default_mock_args, mock_gitlab_token):
},
],
)
async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_dir, mock_prompt_template, test_case):
async def test_process_issue(
default_mock_args,
mock_gitlab_token,
mock_output_dir,
mock_prompt_template,
test_case,
):
"""Test the process_issue method with different scenarios."""
# Set up test data
issue = Issue(
@@ -482,7 +497,7 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
mock_runtime = MagicMock()
mock_runtime.connect = AsyncMock()
mock_create_runtime = MagicMock(return_value=mock_runtime)
# Configure run_controller mock based on test case
mock_run_controller = AsyncMock()
if test_case.get('run_controller_raises'):
@@ -491,16 +506,18 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
mock_run_controller.return_value = test_case['run_controller_return']
# Patch the necessary functions and methods
with patch('openhands.resolver.resolve_issue.create_runtime', mock_create_runtime), \
patch('openhands.resolver.resolve_issue.run_controller', mock_run_controller), \
patch.object(resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}), \
patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime, \
patch('openhands.resolver.resolve_issue.SandboxConfig', return_value=MagicMock()), \
patch('openhands.resolver.resolve_issue.AppConfig', return_value=MagicMock()):
with patch(
'openhands.resolver.resolve_issue.create_runtime', mock_create_runtime
), patch(
'openhands.resolver.resolve_issue.run_controller', mock_run_controller
), patch.object(
resolver, 'complete_runtime', return_value={'git_patch': 'test patch'}
), patch.object(resolver, 'initialize_runtime') as mock_initialize_runtime, patch(
'openhands.resolver.resolve_issue.SandboxConfig', return_value=MagicMock()
), patch('openhands.resolver.resolve_issue.AppConfig', return_value=MagicMock()):
# Call the process_issue method
result = await resolver.process_issue(issue, base_commit, handler_instance)
mock_create_runtime.assert_called_once()
mock_runtime.connect.assert_called_once()
mock_initialize_runtime.assert_called_once()
@@ -521,6 +538,7 @@ async def test_process_issue(default_mock_args, mock_gitlab_token, mock_output_d
else:
handler_instance.guess_success.assert_not_called()
def test_get_instruction(mock_prompt_template, mock_followup_prompt_template):
issue = Issue(
owner='test_owner',
@@ -923,4 +941,4 @@ def test_download_issue_with_specific_comment():
if __name__ == '__main__':
pytest.main()
pytest.main()
+13 -27
View File
@@ -204,11 +204,14 @@ async def test_react_to_content_policy_violation(
mock_status_callback.assert_called_once_with(
'error',
'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION',
'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION',
'ContentPolicyViolationError: litellm.BadRequestError: litellm.ContentPolicyViolationError: Output blocked by content filtering policy',
)
# Verify the state was updated correctly
assert controller.state.last_error == 'STATUS$ERROR_LLM_CONTENT_POLICY_VIOLATION'
assert (
controller.state.last_error
== 'ContentPolicyViolationError: litellm.BadRequestError: litellm.ContentPolicyViolationError: Output blocked by content filtering policy'
)
assert controller.state.agent_state == AgentState.ERROR
await controller.close()
@@ -272,10 +275,8 @@ async def test_run_controller_with_fatal_error(
error_observation = error_observations[0]
assert state.iteration == 3
assert state.agent_state == AgentState.ERROR
assert state.last_error == 'AgentStuckInLoopError: Agent got stuck in a loop'
assert (
error_observation.reason == 'AgentStuckInLoopError: Agent got stuck in a loop'
)
assert state.last_error == 'AgentStuckInLoopError'
assert error_observation.reason == 'AgentStuckInLoopError'
assert len(events) == 12
@@ -355,7 +356,7 @@ async def test_run_controller_stop_with_stuck(
assert last_event['observation'] == 'agent_state_changed'
assert state.agent_state == AgentState.ERROR
assert state.last_error == 'AgentStuckInLoopError: Agent got stuck in a loop'
assert state.last_error == 'AgentStuckInLoopError'
@pytest.mark.asyncio
@@ -688,20 +689,14 @@ async def test_run_controller_max_iterations_has_metrics(
)
assert state.iteration == 3
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 3, max iteration: 3'
)
assert 'RuntimeError' in state.last_error
error_observations = test_event_stream.get_matching_events(
reverse=True, limit=1, event_types=(AgentStateChangedObservation)
)
assert len(error_observations) == 1
error_observation = error_observations[0]
assert (
error_observation.reason
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 3, max iteration: 3'
)
assert 'RuntimeError' in error_observation.reason
assert (
state.metrics.accumulated_cost == 10.0 * 3
@@ -945,10 +940,7 @@ async def test_run_controller_with_context_window_exceeded_with_truncation(
# expected reason
assert state.iteration == 5
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'RuntimeError: Agent reached maximum iteration in headless mode. Current iteration: 5, max iteration: 5'
)
assert state.last_error == 'RuntimeError'
# Check that the context window exceeded error was raised during the run
assert step_state.has_errored
@@ -1022,20 +1014,14 @@ async def test_run_controller_with_context_window_exceeded_without_truncation(
# With the refactored system message handling, the iteration count is different
assert state.iteration == 1
assert state.agent_state == AgentState.ERROR
assert (
state.last_error
== 'LLMContextWindowExceedError: Conversation history longer than LLM context window limit. Consider turning on enable_history_truncation config to avoid this error'
)
assert state.last_error == 'LLMContextWindowExceedError'
error_observations = test_event_stream.get_matching_events(
reverse=True, limit=1, event_types=(AgentStateChangedObservation)
)
assert len(error_observations) == 1
error_observation = error_observations[0]
assert (
error_observation.reason
== 'LLMContextWindowExceedError: Conversation history longer than LLM context window limit. Consider turning on enable_history_truncation config to avoid this error'
)
assert 'LLMContextWindowExceedError' in error_observation.reason
# Check that the context window exceeded error was raised during the run
assert step_state.has_errored