allow exposing some entity attributes + work on climate type

custom_components/llama_conversation/__init__.py

@@ -9,6 +9,7 @@ import requests
 import re
 import os
 import json
+import webcolors
 
 from homeassistant.components import conversation
 from homeassistant.components.conversation.const import DOMAIN as CONVERSATION_DOMAIN
@@ -42,6 +43,7 @@ from .const import (
     CONF_REQUEST_TIMEOUT,
     CONF_BACKEND_TYPE,
     CONF_DOWNLOADED_MODEL_FILE,
+    CONF_EXTRA_ATTRIBUTES_TO_EXPOSE,
     DEFAULT_MAX_TOKENS,
     DEFAULT_PROMPT,
     DEFAULT_TEMPERATURE,
@@ -49,6 +51,7 @@ from .const import (
     DEFAULT_TOP_P,
     DEFAULT_BACKEND_TYPE,
     DEFAULT_REQUEST_TIMEOUT,
+    DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE,
     BACKEND_TYPE_REMOTE,
     DOMAIN,
     GBNF_GRAMMAR_FILE,
@@ -89,6 +92,15 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     conversation.async_unset_agent(hass, entry)
     return True
 
+async def closest_color(requested_colour):
+    min_colours = {}
+    for key, name in webcolors.CSS3_HEX_TO_NAMES.items():
+        r_c, g_c, b_c = webcolors.hex_to_rgb(key)
+        rd = (r_c - requested_colour[0]) ** 2
+        gd = (g_c - requested_colour[1]) ** 2
+        bd = (b_c - requested_colour[2]) ** 2
+        min_colours[(rd + gd + bd)] = name
+    return min_colours[min(min_colours.keys())]
 
 class LLaMAAgent(conversation.AbstractConversationAgent):
     """Local LLaMA conversation agent."""
@@ -109,6 +121,8 @@ class LLaMAAgent(conversation.AbstractConversationAgent):
 
         model_path = self.entry.data.get(CONF_DOWNLOADED_MODEL_FILE)
         self.model_name = self.entry.data.get(CONF_CHAT_MODEL, model_path)
+        self.extra_attributes_to_expose = self.entry.data \
+            .get(CONF_EXTRA_ATTRIBUTES_TO_EXPOSE, DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE).split(",")
 
         if self.use_local_backend:
             if not model_path:
@@ -339,8 +353,9 @@ class LLaMAAgent(conversation.AbstractConversationAgent):
             if not async_should_expose(self.hass, CONVERSATION_DOMAIN, state.entity_id):
                 continue
 
-            # TODO: also expose the "friendly name"
-            entity_states[state.entity_id] = { "state": state.state, "friendly_name": state.attributes["friendly_name"] }
+            attributes = dict(state.attributes)
+            attributes["state"] = state.state
+            entity_states[state.entity_id] = attributes
             domains.add(state.domain)
 
         _LOGGER.debug(f"Exposed entities: {entity_states}")
@@ -366,8 +381,29 @@ class LLaMAAgent(conversation.AbstractConversationAgent):
         """Generate a prompt for the user."""
         entities_to_expose, domains = self._async_get_exposed_entities()
 
+        def expose_attributes(attributes):
+            result = attributes["state"]
+            for attribute_name in self.extra_attributes_to_expose:
+                _LOGGER.info(f"{attribute_name} = {attributes['attribute_name']}")
+                if attribute_name not in attributes:
+                    continue
+
+                value = attributes[attribute_name]
+                if attribute_name == "current_temperature":
+                    value = int(value)
+                    if value > 50:
+                        value = value + "F"
+                    else:
+                        value = value + "C"
+                elif attribute_name == "rgb_color":
+                    value = tuple(value.split(", "))
+                    value = closest_color(value)
+                    
+                result = result + ";" + str(value)
+            return result
+
         formatted_states = "\n".join(
-            [f"{name} '{attributes['friendly_name']}' = {attributes['state']}" for name, attributes in entities_to_expose.items()]
+            [f"{name} '{attributes['friendly_name']}' = {expose_attributes(attributes)}" for name, attributes in entities_to_expose.items()]
         ) + "\n"
 
         service_dict = self.hass.services.async_services()

custom_components/llama_conversation/const.py

@@ -31,5 +31,6 @@ CONF_DOWNLOADED_MODEL_FILE = "downloaded_model_file"
 DEFAULT_DOWNLOADED_MODEL_FILE = ""
 DEFAULT_HOST = "127.0.0.1"
 DEFAULT_PORT = "5000"
-
+CONF_EXTRA_ATTRIBUTES_TO_EXPOSE = "extra_attributes_to_expose"
+DEFAULT_EXTRA_ATTRIBUTES_TO_EXPOSE = "rgb_color,current_temperature,fan_mode,media_title,volume_level"
 GBNF_GRAMMAR_FILE = "output.gbnf"

custom_components/llama_conversation/manifest.json

@@ -10,6 +10,7 @@
   "iot_class": "local_polling",
   "requirements": [
     "requests",
-    "huggingface-hub"
+    "huggingface-hub",
+    "webcolors"
   ]
 }

data/generate_home_assistant_data.py

@@ -64,13 +64,45 @@ class ClimateDeviceType(DeviceType):
         ])
 
     def get_random_state(self):
-        hvac = random.choice(["heating", "cooling", "idle"])
-        fan = random.choice(["fan on", "fan off"])
+        hvac = random.choice(["heat", "cool", "heat_cool", "off", "auto", "fan_only"])
+        fan = random.choice(["On Low", "On High", "Auto Low", "Auto High", "Off"])
         if random.random() > 0.5:
             temp = str(random.randint(60, 80)) + "F"
         else:
             temp = str(random.randint(15, 25)) + "C"
-        return f"{hvac} ({fan}) at {temp}"
+        return f"{hvac};{fan};{temp}"
+
+class MediaPlayerDeviceType(DeviceType):
+    def __init__(self):
+        super().__init__("media_player", [
+            (STATE_ON, 0.15),
+            (STATE_OFF, 0.3),
+            (STATE_IDLE, 0.1),
+            (STATE_PLAYING, 0.2),
+            (STATE_PAUSED, 0.15),
+            (STATE_STANDBY, 0.05),
+            (STATE_BUFFERING, 0.05),
+        ], [
+            "turn_on",
+            "turn_off",
+            "toggle",
+            "volume_up",
+            "volume_down",
+            "volume_mute",
+            "media_play_pause",
+            "media_play",
+            "media_pause",
+            "media_stop",
+            "media_next_track",
+            "media_previous_track"
+        ])
+
+    def get_random_state(self):
+        state = super().get_random_state()
+
+        if state != STATE_OFF:
+            pass # TODO: add volume + a random media title
+        return state
 
 SUPPORTED_DEVICES = {
     "light": DeviceType(
@@ -152,32 +184,7 @@ SUPPORTED_DEVICES = {
             "unlock",
         ],
     ),
-    "media_player": DeviceType(
-        name="media_player",
-        possible_states=[
-            (STATE_ON, 0.15),
-            (STATE_OFF, 0.3),
-            (STATE_IDLE, 0.1),
-            (STATE_PLAYING, 0.2),
-            (STATE_PAUSED, 0.15),
-            (STATE_STANDBY, 0.05),
-            (STATE_BUFFERING, 0.05),
-        ],
-        services=[
-            "turn_on",
-            "turn_off",
-            "toggle",
-            "volume_up",
-            "volume_down",
-            "volume_mute",
-            "media_play_pause",
-            "media_play",
-            "media_pause",
-            "media_stop",
-            "media_next_track",
-            "media_previous_track"
-        ],
-    ),
+    "media_player": MediaPlayerDeviceType(),
     "climate": ClimateDeviceType()
 }
 
@@ -252,6 +259,9 @@ def random_device_list(max_devices: int, avoid_device_names: list[str]):
             device_type = device_name.split(".")[0]
             friendly_name = choice["description"]
 
+            if device_type == "climate":
+                continue # don't add random thermostats. we need to be careful about how we handle multiple thermostats
+
             state = SUPPORTED_DEVICES[device_type].get_random_state()
             device_lines.append(format_device_line(
                 device_name=device_name,
@@ -276,7 +286,6 @@ def generate_static_example(action: dict, max_devices: int = 32):
     device_list, device_types = random_device_list(max_devices=max_devices, avoid_device_names=[target_device])
 
     # insert our target device somewhere random in the list
-    
     index = random.randint(0, len(device_list))
     state = SUPPORTED_DEVICES[device_type].get_random_state()
 
@@ -288,7 +297,7 @@ def generate_static_example(action: dict, max_devices: int = 32):
 
     # gather a list of all available services
     available_services = []
-    for x in device_types:
+    for x in set(device_types + [device_type]):
         available_services.extend([ f"{x}.{y}" for y in SUPPORTED_DEVICES[x].services ])
 
     return {
@@ -329,11 +338,9 @@ def generate_templated_example(template: dict, max_devices: int = 32):
 
     # gather a list of all available services
     available_services = []
-    for x in device_types:
+    for x in set(device_types + template_device_types):
         available_services.extend([ f"{x}.{y}" for y in SUPPORTED_DEVICES[x].services ])
 
-    available_services.extend(service_names)
-
     # generate the question
     if len(template_device_types) == 1:
         question = question_template.replace("<device_name>", chosen_devices[0]["description"])
@@ -345,6 +352,18 @@ def generate_templated_example(template: dict, max_devices: int = 32):
             question = question.replace(f"<device_name{(i + 1)}>", chosen_devices[i]["description"])
             answer = answer.replace(f"<device_name{(i + 1)}>", chosen_devices[i]["description"])
 
+    if any(["climate" in service for service in service_names ]):
+        temp_f = str(random.randint(60, 80))
+        temp_c = str(random.randint(15, 25))
+        humidity = str(random.randint(0, 20) * 5)
+        question = question.replace("<temp_f>", temp_f)
+        question = question.replace("<temp_c>", temp_c)
+        question = question.replace("<humidity>", humidity)
+        
+        answer = answer.replace("<temp_f>", temp_f)
+        answer = answer.replace("<temp_c>", temp_c)
+        answer = answer.replace("<humidity>", humidity)
+        
 
     # generate the list of service calls and answers
     service_calls = []
@@ -377,7 +396,7 @@ def generate_status_request(template: dict, max_devices: int = 32):
 
     # gather a list of all available services
     available_services = []
-    for x in device_types:
+    for x in set(device_types + [device_type]):
         available_services.extend([ f"{x}.{y}" for y in SUPPORTED_DEVICES[x].services ])
 
     # generate the question
@@ -457,9 +476,9 @@ def generate_example_file(filename: str, seed: int, *, static_factor: int, templ
 # TODO: add examples for rooms/groups of devices. i.e. "turn off all the lights in the kitchen"
 # TODO: expose home assistant attributes in the context
 def main():
-    # generate_example_file("sample", 42, static_factor=1, template_factor=1, status_request_factor=1)
-    generate_example_file("home_assistant_train", 42, static_factor=5, template_factor=20, status_request_factor=15)
-    generate_example_file("home_assistant_test", 12345, static_factor=0.25, template_factor=3, status_request_factor=2)
+    generate_example_file("sample", 42, static_factor=1, template_factor=1, status_request_factor=1)
+    # generate_example_file("home_assistant_train", 42, static_factor=5, template_factor=20, status_request_factor=15)
+    # generate_example_file("home_assistant_test", 12345, static_factor=0.25, template_factor=3, status_request_factor=2)
 
 if __name__ == "__main__":
     main()

data/piles/pile_of_templated_actions.csv

@@ -175,3 +175,20 @@ media_player,media_next_track,"Skip this track on <device_name>.","Skipping this
 media_player,media_previous_track,"Previous track on <device_name>, please.","Going back to previous track on <device_name>."
 media_player,media_previous_track,"Rewind to the previous song on <device_name>.","Rewinding to the previous song on <device_name>."
 media_player,media_previous_track,"Can we go back a track on <device_name>?","Going back a track on <device_name>."
+climate,set_temperature,"Set the temperature to <temp_f> degrees.","Setting temperature to <temp_f> degrees."
+climate,set_temperature,"Can you change the temperature to <temp_c> Celsius?","Changing temperature to <temp_c> Celsius."
+climate,set_temperature,"I'd like the room at <temp_f> degrees Fahrenheit, please.","Setting the room to <temp_f> degrees Fahrenheit."
+climate,set_temperature,"Please adjust the temperature to <temp_f> degrees.","Adjusting temperature to <temp_f> degrees Fahrenheit."
+climate,set_temperature,"I want the room cooler, set it to <temp_c> degrees.","Setting the room to <temp_c> degrees Celsius for cooler temperature."
+climate,set_temperature,"Make it warmer, set temperature at <temp_f> degrees.","Making it warmer, setting temperature to <temp_f> degrees."
+climate,set_temperature,"Can you lower the temperature to <temp_c>?","Lowering the temperature to <temp_c> Celsius."
+climate,set_temperature,"Raise the temperature to <temp_f> degrees, please.","Raising the temperature to <temp_f> degrees Fahrenheit."
+climate,set_humidity,"Increase the humidity to <temp_c>%.","Increasing humidity to <temp_c>%."
+climate,set_humidity,"Set the humidity level to <humidity> percent.","Setting humidity to <humidity> percent."
+climate,set_humidity,"Can you adjust the humidity to <humidity>%?","Adjusting humidity to <humidity>%."
+climate,set_fan_mode,"Set the fan to high speed.","Setting the fan to high speed."
+climate,set_fan_mode,"Please put the fan on low.","Putting the fan on low."
+climate,set_fan_mode,"Change the fan setting to medium.","Changing the fan to medium setting."
+climate,set_hvac_mode,"Switch the system to cooling mode.","Switching to cooling mode."
+climate,set_hvac_mode,"Can we set the HVAC to heat?","Setting the HVAC to heat."
+climate,set_hvac_mode,"Change the HVAC to automatic.","Changing HVAC to automatic mode."