mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2026-01-09 14:57:59 -05:00
Co-authored-by: sp.wack <83104063+amanape@users.noreply.github.com>
This commit is contained in:
@@ -29,7 +29,7 @@ const generateAgentResponse = (message: string): AssistantMessageAction => ({
|
||||
action: "message",
|
||||
args: {
|
||||
content: message,
|
||||
images_urls: [],
|
||||
image_urls: [],
|
||||
wait_for_response: false,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -2,12 +2,12 @@ import ActionType from "#/types/ActionType";
|
||||
|
||||
export function createChatMessage(
|
||||
message: string,
|
||||
images_urls: string[],
|
||||
image_urls: string[],
|
||||
timestamp: string,
|
||||
) {
|
||||
const event = {
|
||||
action: ActionType.MESSAGE,
|
||||
args: { content: message, images_urls, timestamp },
|
||||
args: { content: message, image_urls, timestamp },
|
||||
};
|
||||
return event;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ export interface UserMessageAction extends OpenHandsActionEvent<"message"> {
|
||||
source: "user";
|
||||
args: {
|
||||
content: string;
|
||||
images_urls: string[];
|
||||
image_urls: string[];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ export interface AssistantMessageAction
|
||||
source: "agent";
|
||||
args: {
|
||||
content: string;
|
||||
images_urls: string[] | null;
|
||||
image_urls: string[] | null;
|
||||
wait_for_response: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ interface LocalUserMessageAction {
|
||||
action: "message";
|
||||
args: {
|
||||
content: string;
|
||||
images_urls: string[];
|
||||
image_urls: string[];
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -196,8 +196,8 @@ class CodeActAgent(Agent):
|
||||
elif isinstance(action, MessageAction):
|
||||
role = 'user' if action.source == 'user' else 'assistant'
|
||||
content = [TextContent(text=action.content or '')]
|
||||
if self.llm.vision_is_active() and action.images_urls:
|
||||
content.append(ImageContent(image_urls=action.images_urls))
|
||||
if self.llm.vision_is_active() and action.image_urls:
|
||||
content.append(ImageContent(image_urls=action.image_urls))
|
||||
return [
|
||||
Message(
|
||||
role=role,
|
||||
|
||||
@@ -95,9 +95,9 @@ class CodeActSWEAgent(Agent):
|
||||
if (
|
||||
self.llm.vision_is_active()
|
||||
and isinstance(action, MessageAction)
|
||||
and action.images_urls
|
||||
and action.image_urls
|
||||
):
|
||||
content.append(ImageContent(image_urls=action.images_urls))
|
||||
content.append(ImageContent(image_urls=action.image_urls))
|
||||
|
||||
return Message(
|
||||
role='user' if action.source == 'user' else 'assistant', content=content
|
||||
|
||||
@@ -149,7 +149,7 @@ class State:
|
||||
for event in reversed(self.history):
|
||||
if isinstance(event, MessageAction) and event.source == 'user':
|
||||
last_user_message = event.content
|
||||
last_user_message_image_urls = event.images_urls
|
||||
last_user_message_image_urls = event.image_urls
|
||||
elif isinstance(event, AgentFinishAction):
|
||||
if last_user_message is not None:
|
||||
return last_user_message, None
|
||||
|
||||
@@ -7,7 +7,7 @@ from openhands.events.action.action import Action, ActionSecurityRisk
|
||||
@dataclass
|
||||
class MessageAction(Action):
|
||||
content: str
|
||||
images_urls: list[str] | None = None
|
||||
image_urls: list[str] | None = None
|
||||
wait_for_response: bool = False
|
||||
action: str = ActionType.MESSAGE
|
||||
security_risk: ActionSecurityRisk | None = None
|
||||
@@ -16,10 +16,18 @@ class MessageAction(Action):
|
||||
def message(self) -> str:
|
||||
return self.content
|
||||
|
||||
@property
|
||||
def images_urls(self):
|
||||
# Deprecated alias for backward compatibility
|
||||
return self.image_urls
|
||||
|
||||
@images_urls.setter
|
||||
def images_urls(self, value):
|
||||
self.image_urls = value
|
||||
def __str__(self) -> str:
|
||||
ret = f'**MessageAction** (source={self.source})\n'
|
||||
ret += f'CONTENT: {self.content}'
|
||||
if self.images_urls:
|
||||
for url in self.images_urls:
|
||||
if self.image_urls:
|
||||
for url in self.image_urls:
|
||||
ret += f'\nIMAGE_URL: {url}'
|
||||
return ret
|
||||
|
||||
@@ -66,6 +66,10 @@ def action_from_dict(action: dict) -> Action:
|
||||
if is_confirmed is not None:
|
||||
args['confirmation_state'] = is_confirmed
|
||||
|
||||
# images_urls has been renamed to image_urls
|
||||
if 'images_urls' in args:
|
||||
args['image_urls'] = args.pop('images_urls')
|
||||
|
||||
try:
|
||||
decoded_action = action_class(**args)
|
||||
if 'timeout' in action:
|
||||
|
||||
@@ -101,7 +101,7 @@ def event_to_memory(event: 'Event', max_message_chars: int) -> dict:
|
||||
d.pop('cause', None)
|
||||
d.pop('timestamp', None)
|
||||
d.pop('message', None)
|
||||
d.pop('images_urls', None)
|
||||
d.pop('image_urls', None)
|
||||
|
||||
# runnable actions have some extra fields used in the BE/FE, which should not be sent to the LLM
|
||||
if 'args' in d:
|
||||
|
||||
@@ -276,7 +276,7 @@ async def websocket_endpoint(websocket: WebSocket):
|
||||
```
|
||||
- Send a message:
|
||||
```json
|
||||
{"action": "message", "args": {"content": "Hello, how are you?", "images_urls": ["base64_url1", "base64_url2"]}}
|
||||
{"action": "message", "args": {"content": "Hello, how are you?", "image_urls": ["base64_url1", "base64_url2"]}}
|
||||
```
|
||||
- Write contents to a file:
|
||||
```json
|
||||
|
||||
@@ -163,7 +163,7 @@ class Session:
|
||||
return
|
||||
event = event_from_dict(data.copy())
|
||||
# This checks if the model supports images
|
||||
if isinstance(event, MessageAction) and event.images_urls:
|
||||
if isinstance(event, MessageAction) and event.image_urls:
|
||||
controller = self.agent_session.controller
|
||||
if controller:
|
||||
if controller.agent.llm.config.disable_vision:
|
||||
|
||||
@@ -65,7 +65,7 @@ def test_event_props_serialization_deserialization():
|
||||
'action': 'message',
|
||||
'args': {
|
||||
'content': 'This is a test.',
|
||||
'images_urls': None,
|
||||
'image_urls': None,
|
||||
'wait_for_response': False,
|
||||
},
|
||||
}
|
||||
@@ -77,7 +77,7 @@ def test_message_action_serialization_deserialization():
|
||||
'action': 'message',
|
||||
'args': {
|
||||
'content': 'This is a test.',
|
||||
'images_urls': None,
|
||||
'image_urls': None,
|
||||
'wait_for_response': False,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ def test_event_serialization_deserialization():
|
||||
'message': 'This is a test.',
|
||||
'args': {
|
||||
'content': 'This is a test.',
|
||||
'images_urls': None,
|
||||
'image_urls': None,
|
||||
'wait_for_response': False,
|
||||
},
|
||||
}
|
||||
@@ -38,7 +38,7 @@ def test_array_serialization_deserialization():
|
||||
'message': 'This is a test.',
|
||||
'args': {
|
||||
'content': 'This is a test.',
|
||||
'images_urls': None,
|
||||
'image_urls': None,
|
||||
'wait_for_response': False,
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user