From 411276408b3c7c8535f1230f23eeac4cc42659f2 Mon Sep 17 00:00:00 2001 From: Alex O'Connell Date: Tue, 13 Feb 2024 20:21:51 -0500 Subject: [PATCH] train new models based on stablelm + properly add new response types --- data/README.md | 7 +- data/generate_home_assistant_data.py | 48 ++- data/piles/pile_of_responses.csv | 316 +++++++++++++++++- data/piles/pile_of_system_prompts.csv | 4 +- data/test_prompts/{ => chatml}/brightness.txt | 0 data/test_prompts/chatml/christmas.txt | 28 ++ data/test_prompts/{ => chatml}/climate.txt | 0 data/test_prompts/chatml/ha_demo.txt | 24 ++ data/test_prompts/{ => chatml}/ha_demo2.txt | 0 .../{ => chatml}/logic_puzzle.txt | 0 data/test_prompts/chatml/rgb_lights.txt | 15 + data/test_prompts/{ => chatml}/sonnet.txt | 3 +- data/test_prompts/rgb_lights.txt | 15 - data/test_prompts/zephyr/brightness.txt | 16 + data/test_prompts/zephyr/christmas.txt | 28 ++ data/test_prompts/zephyr/climate.txt | 18 + data/test_prompts/zephyr/ha_demo.txt | 24 ++ data/test_prompts/zephyr/ha_demo2.txt | 22 ++ .../{ha_demo.txt => zephyr/logic_puzzle.txt} | 12 +- data/test_prompts/zephyr/rgb_lights.txt | 15 + .../{christmas.txt => zephyr/sonnet.txt} | 10 +- ...t-notes-phi.md => experiment-notes-phi.md} | 2 +- docs/experiment-notes-stablelm.md | 68 ++++ evaluate.py | 33 +- requirements.txt | 1 + train.py | 8 +- 26 files changed, 655 insertions(+), 62 deletions(-) rename data/test_prompts/{ => chatml}/brightness.txt (100%) create mode 100644 data/test_prompts/chatml/christmas.txt rename data/test_prompts/{ => chatml}/climate.txt (100%) create mode 100644 data/test_prompts/chatml/ha_demo.txt rename data/test_prompts/{ => chatml}/ha_demo2.txt (100%) rename data/test_prompts/{ => chatml}/logic_puzzle.txt (100%) create mode 100644 data/test_prompts/chatml/rgb_lights.txt rename data/test_prompts/{ => chatml}/sonnet.txt (97%) delete mode 100644 data/test_prompts/rgb_lights.txt create mode 100644 data/test_prompts/zephyr/brightness.txt create mode 100644 data/test_prompts/zephyr/christmas.txt create mode 100644 data/test_prompts/zephyr/climate.txt create mode 100644 data/test_prompts/zephyr/ha_demo.txt create mode 100644 data/test_prompts/zephyr/ha_demo2.txt rename data/test_prompts/{ha_demo.txt => zephyr/logic_puzzle.txt} (88%) create mode 100644 data/test_prompts/zephyr/rgb_lights.txt rename data/test_prompts/{christmas.txt => zephyr/sonnet.txt} (89%) rename docs/{expermement-notes-phi.md => experiment-notes-phi.md} (99%) create mode 100644 docs/experiment-notes-stablelm.md diff --git a/data/README.md b/data/README.md index 470f6ee..15b7898 100644 --- a/data/README.md +++ b/data/README.md @@ -35,4 +35,9 @@ Supported datasets right now are: - `alpaca` - `wizardlm70k` -Please note that the supported datasets all have different licenses. Be aware that the license of the resulting data mixture might be different that the license of this dataset alone. \ No newline at end of file +Please note that the supported datasets all have different licenses. Be aware that the license of the resulting data mixture might be different that the license of this dataset alone. + +## Adding new Home Assistant functionality +Adding new functionality to the model is done by providing examples of a user asking the assistant for the + +## Adding a new personality \ No newline at end of file diff --git a/data/generate_home_assistant_data.py b/data/generate_home_assistant_data.py index f4dd204..cc22ef3 100644 --- a/data/generate_home_assistant_data.py +++ b/data/generate_home_assistant_data.py @@ -274,6 +274,9 @@ def get_included_vars(response: str): pile_of_responses["contains_vars"] = pile_of_responses["response"].apply(get_included_vars) +class NoResponseAvailableException(Exception): + pass + def get_random_response(*, service: str, language: str, persona: str, question_template: str, short: bool) -> str: required_vars = list(set([var for var in var_pattern.findall(question_template) if "device_name" not in var])) @@ -286,7 +289,7 @@ def get_random_response(*, service: str, language: str, persona: str, question_t ] if len(possible_results) == 0: - raise Exception(f"No responses matched the provided filters: {service}, {language}, {persona}, {question_template}, {short}") + raise NoResponseAvailableException(f"No responses matched the provided filters: {service}, {language}, {persona}, {required_vars}, {short}") return possible_results.sample()["response"].values[0] @@ -388,8 +391,8 @@ def generate_static_example(action: dict, language: str, persona: str, max_devic response = get_random_response( service=action["service_name"], - language="en", - persona="assistant", + language=language, + persona=persona, question_template="", short=False ).lower() @@ -665,13 +668,13 @@ def format_example_sharegpt(example, persona): return { "conversations": conversation } -def generate_example_file(filename: str, seed: int, format_func: Callable, language: str, persona: str, *, static_factor: int, template_factor: int, status_request_factor: int): +def generate_example_file(filename: str, seed: int, format_func: Callable, languages: list[str], personas: list[str], *, static_factor: int, template_factor: int, status_request_factor: int): random.seed(seed) np.random.seed(seed) print("Generating...") - def run_factor_times(func, examples, data, factor): + def run_factor_times(func, examples, data, language, persona, factor): if factor >= 1: for i in range(factor): examples.append(format_func(func(data, language, persona), persona)) @@ -680,14 +683,23 @@ def generate_example_file(filename: str, seed: int, format_func: Callable, langu examples.append(format_func(func(data, language, persona), persona)) generated_examples = [] - for action in tqdm(pile_of_specific_actions): - run_factor_times(generate_static_example, generated_examples, action, static_factor) - for templated_action in tqdm(pile_of_templated_actions): - run_factor_times(generate_templated_example, generated_examples, templated_action, template_factor) + for lang in languages: + for person in personas: + for action in tqdm(pile_of_specific_actions): + try: + run_factor_times(generate_static_example, generated_examples, action, lang, person, static_factor) + except NoResponseAvailableException as ex: + print(ex) + + for templated_action in tqdm(pile_of_templated_actions): + try: + run_factor_times(generate_templated_example, generated_examples, templated_action, lang, person, template_factor) + except NoResponseAvailableException as ex: + print(ex) for status_request in tqdm(pile_of_status_requests): - run_factor_times(generate_status_request, generated_examples, status_request, status_request_factor) + run_factor_times(generate_status_request, generated_examples, status_request, "en", "assistant", status_request_factor) print(f"Generated {len(generated_examples)} examples. Saving...") @@ -766,8 +778,8 @@ def main(): args = parser.parse_args() - language = "en" - persona = "assistant" + languages = ["en"] + personas = ["assistant", "pirate", "robot"] if not args.sample and not args.train and not args.test and not args.merge: parser.print_usage() @@ -778,20 +790,20 @@ def main(): format_func = format_example_sharegpt if args.sample: - generate_example_file("sample", 42, format_func, language, persona, static_factor=1, template_factor=1, status_request_factor=1) + generate_example_file("sample", 42, format_func, languages, personas, static_factor=1, template_factor=1, status_request_factor=1) if args.train: if args.size == "small": - generate_example_file("home_assistant_train", 42, format_func, language, persona, static_factor=1, template_factor=10, status_request_factor=8) + generate_example_file("home_assistant_train", 42, format_func, languages, personas, static_factor=1, template_factor=10, status_request_factor=8) elif args.size == "medium": - generate_example_file("home_assistant_train", 42, format_func, language, persona, static_factor=5, template_factor=15, status_request_factor=12) + generate_example_file("home_assistant_train", 42, format_func, languages, personas, static_factor=5, template_factor=15, status_request_factor=12) elif args.size == "large": - generate_example_file("home_assistant_train", 42, format_func, language, persona, static_factor=5, template_factor=20, status_request_factor=15) + generate_example_file("home_assistant_train", 42, format_func, languages, personas, static_factor=5, template_factor=20, status_request_factor=15) elif args.size == "xl": - generate_example_file("home_assistant_train", 42, format_func, language, persona, static_factor=7, template_factor=25, status_request_factor=18) + generate_example_file("home_assistant_train", 42, format_func, languages, personas, static_factor=7, template_factor=25, status_request_factor=18) else: raise Exception(f"Unrecognized dataset size: {args.size}") if args.test: - generate_example_file("home_assistant_test", 12345, format_func, language, persona, static_factor=0.25, template_factor=3, status_request_factor=2) + generate_example_file("home_assistant_test", 12345, format_func, languages, personas, static_factor=0.25, template_factor=1, status_request_factor=2) if args.merge == "alpaca": merge_with_dataset("yahma/alpaca-cleaned", 42, "alpaca", format_alpaca, ["input", "output", "instruction"], format_func) diff --git a/data/piles/pile_of_responses.csv b/data/piles/pile_of_responses.csv index a7a609d..5a0b2b1 100644 --- a/data/piles/pile_of_responses.csv +++ b/data/piles/pile_of_responses.csv @@ -136,21 +136,16 @@ light.turn_on,Turning .,en,assistant,0 light.turn_on,Changing the color of to .,en,assistant,0 light.turn_on,Changing to a hue.,en,assistant,0 light.turn_on,Setting to be .,en,assistant,0 -light.turn_on,Setting to be .,en,assistant,0 light.turn_on,Making shine in .,en,assistant,0 light.turn_on,Turning to a shade.,en,assistant,0 -light.turn_on,Turning .,en,assistant,0 light.turn_on,Setting to a .,en,assistant,0 light.turn_on,Setting to a color.,en,assistant,0 -light.turn_on,Setting to a color.,en,assistant,0 light.turn_on,Making glow .,en,assistant,0 light.turn_on,Turning to .,en,assistant,0 light.turn_on,Changing to .,en,assistant,0 light.turn_on,Adjusting to color.,en,assistant,0 light.turn_on,Switching color to .,en,assistant,0 -light.turn_on,Setting to be .,en,assistant,0 light.turn_on,Setting in .,en,assistant,0 -light.turn_on,Changing to .,en,assistant,0 light.turn_on,Making display a light.,en,assistant,0 light.turn_on,Setting color to .,en,assistant,0 light.turn_off,Deactivating as requested,en,assistant,0 @@ -300,4 +295,313 @@ light.turn_on,turning on both ,en,assistant,1 lock.lock,securing ,en,assistant,1 lock.lock,locking ,en,assistant,1 lock.unlock,unlocking ,en,assistant,1 -lock.unlock,unsecuring ,en,assistant,1 \ No newline at end of file +lock.unlock,unsecuring ,en,assistant,1 +blinds.open,"Openin' the blinds for ye.",en,pirate,0 +blinds.open,"Aye, I'll be openin' the blinds.",en,pirate,0 +blinds.open,"Aye, openin' the blinds now.",en,pirate,0 +blinds.close,"Closin' the blinds as ye requested.",en,pirate,0 +blinds.close,"I'll be closin' the blinds for ye.",en,pirate,0 +blinds.close,"Aye, closin' the blinds.",en,pirate,0 +blinds.stop,"Stopin' the blinds now.",en,pirate,0 +blinds.stop,"I'll halt the blinds for ye.",en,pirate,0 +blinds.stop,"Aye, haltin' the blinds' movement.",en,pirate,0 +blinds.toggle,"Togglin' the blinds' state for ye.",en,pirate,0 +blinds.toggle,"Switchin' the blinds' state now.",en,pirate,0 +blinds.toggle,"I'll be togglin' the blinds for ye.",en,pirate,0 +blinds.open,"Liftin' blinds yarr",en,pirate,0 +blinds.open,"Openin' now",en,pirate,0 +blinds.open,"Raisin' for ye",en,pirate,0 +blinds.stop,"Freezin' position",en,pirate,0 +blinds.stop,"Haltin' now",en,pirate,0 +blinds.stop,"Stoppin' operation",en,pirate,0 +blinds.open,"Raisin' ",en,pirate,0 +blinds.close,"Closin' for ye",en,pirate,0 +blinds.close,"Lowerin' now",en,pirate,0 +blinds.close,"Shuttin' yarr",en,pirate,0 +blinds.close,"Lowerin' ",en,pirate,0 +blinds.toggle,"Flippin' state now",en,pirate,0 +blinds.toggle,"Switchin' state yarr",en,pirate,0 +blinds.toggle,"Togglin' for ye",en,pirate,0 +blinds.toggle,"Togglin' ",en,pirate,0 +climate.set_humidity,"Yarrr, rampin' up humidity to , by the sea!",en,pirate,0 +climate.set_humidity,"Arrr, settin' humidity to percent, as ye wish!",en,pirate,0 +climate.set_humidity,"By Davy Jones' locker, adjustin' humidity to %!",en,pirate,0 +climate.set_fan_mode,"Hoistin' the fan to speed, swifter than a sloop!",en,pirate,0 +climate.set_fan_mode,"Arr, puttin' the fan on , steady as she goes!",en,pirate,0 +climate.set_fan_mode,"Shiver me timbers, changin' the fan to setting!",en,pirate,0 +climate.set_hvac_mode,"Blimey, switchin' to mode, full sail ahead!",en,pirate,0 +climate.set_hvac_mode,"Arrr, settin' the HVAC to , steady as she goes!",en,pirate,0 +climate.set_hvac_mode,"By the powers, changin' HVAC to mode!",en,pirate,0 +climate.set_temperature,"Arr, settin' temperature to degrees, as warm as the Caribbean sun!",en,pirate,0 +climate.set_temperature,"Yarrr, changin' temperature to Celsius, cooler than the deep blue!",en,pirate,0 +climate.set_temperature,"Heave ho, settin' the room to degrees Fahrenheit, warm as a pirate's grog!",en,pirate,0 +climate.set_temperature,"Avast, adjustin' temperature to degrees Fahrenheit, as cozy as a ship's cabin!",en,pirate,0 +climate.set_temperature,"Arr, settin' the room to degrees Celsius for cooler breezes, as refreshing as an ocean mist!",en,pirate,0 +climate.set_temperature,"Aye, makin' it warmer, settin' temperature to degrees, like a sun-drenched cove!",en,pirate,0 +climate.set_temperature,"Blow me down, lowerin' the temperature to Celsius, as brisk as the morning sea air!",en,pirate,0 +climate.set_temperature,"Yo ho, raisin' the temperature to degrees Fahrenheit, warmer than a pirate's plunder!",en,pirate,0 +fan.turn_on,"Firin' up the fan for ye, arr!",en,pirate,0 +fan.turn_on,"Aye, I'll get the fan blowin' for ye.",en,pirate,0 +fan.turn_on,"Yar, turnin' on the fan now!",en,pirate,0 +fan.turn_off,"Killin' the fan as ye requested, matey.",en,pirate,0 +fan.turn_off,"I'll snuff out the fan for ye.",en,pirate,0 +fan.turn_off,"Aye, turnin' off the fan.",en,pirate,0 +fan.toggle,"I'll be flippin' the fan's state for ye.",en,pirate,0 +fan.toggle,"Togglin' the fan now, by the stars!",en,pirate,0 +fan.toggle,"Switchin' the fan's state for ye, savvy?",en,pirate,0 +fan.increase_speed,"Boostin' the fan speed for ye, swift as the wind!",en,pirate,0 +fan.increase_speed,"Sure as the tide, speedin' up the fan now.",en,pirate,0 +fan.increase_speed,"I'll be makin' the fan faster, hold fast!",en,pirate,0 +fan.decrease_speed,"Reducin' the fan speed as ye requested, quiet as the deep.",en,pirate,0 +fan.decrease_speed,"I'll slow down the fan for ye, easy as a calm sea.",en,pirate,0 +fan.decrease_speed,"Sure, decreasin' the fan speed, steady now.",en,pirate,0 +fan.toggle,"Flippin' state for ye, with a yo-ho-ho!",en,pirate,0 +fan.toggle,"Switchin' state as requested, on my honor!",en,pirate,0 +fan.toggle,"Togglin' now, without a fuss!",en,pirate,0 +fan.turn_on,"Activatin' now, set sail!",en,pirate,0 +fan.turn_on,"Startin' for ye, full speed ahead!",en,pirate,0 +fan.turn_on,"Certainly, startin' , let's brave the squall!",en,pirate,0 +fan.turn_on,"Turnin' on , let the winds favor us!",en,pirate,0 +fan.turn_on,"Startin' , by the code!",en,pirate,0 +fan.turn_off,"Deactivatin' as requested, quiet as a hidden cove.",en,pirate,0 +fan.turn_off,"Stoppin' for ye, as silent as the grave.",en,pirate,0 +fan.turn_off,"Certainly, stoppin' , all hands!",en,pirate,0 +fan.turn_off,"Turnin' off , let's not wake the kraken.",en,pirate,0 +fan.decrease_speed,"Reducin' speed of , smooth sailin'.",en,pirate,0 +fan.decrease_speed,"Lowerin' speed of as requested, gentle as a lagoon.",en,pirate,0 +fan.decrease_speed,"Slowing down for ye, easy does it.",en,pirate,0 +fan.increase_speed,"Increasin' speed of , catch the horizon!",en,pirate,0 +fan.increase_speed,"Rampin' up speed now, faster than a fleeing galleon!",en,pirate,0 +fan.increase_speed,"Speedin' up for ye, let's outrun the navy!",en,pirate,0 +fan.increase_speed,"Increasin' speed of , to outrun the storm!",en,pirate,0 +fan.decrease_speed,"Reducin' speed of , steady as we go.",en,pirate,0 +light.turn_on,"Illuminatin' the room for ye, arr!",en,pirate,0 +light.turn_on,"Aye, I'll light up the quarters now.",en,pirate,0 +light.turn_on,"Settin' sail with the light on, I will!",en,pirate,0 +light.turn_off,"Dousin' the lights as ye wish, matey.",en,pirate,0 +light.turn_off,"I'll be snuffin' out the light for ye.",en,pirate,0 +light.turn_off,"Aye, extinguishin' the light.",en,pirate,0 +light.toggle,"Flippin' the light for ye, by the stars!",en,pirate,0 +light.toggle,"Switchin' the light's state, on my honor!",en,pirate,0 +light.toggle,"I'll be togglin' the light for ye, savvy?",en,pirate,0 +light.toggle,"Flippin' the state, with a yo-ho-ho!",en,pirate,0 +light.toggle,"Switchin' state as commanded, arr!",en,pirate,0 +light.toggle,"Togglin' for ye, without a fuss!",en,pirate,0 +light.toggle,"Togglin' , ready for adventure!",en,pirate,0 +light.turn_on,"Startin' up , on the horizon!",en,pirate,0 +light.turn_on,"Aye, brightenin' now, clear as day!",en,pirate,0 +light.turn_on,"Switchin' on for ye, like the northern star!",en,pirate,0 +light.turn_on,"Lightin' up , as bright as the full moon!",en,pirate,0 +light.turn_on,"Activatin' , let there be light!",en,pirate,0 +light.turn_on,"Settin' the brightness of to %, as clear as the open sea!",en,pirate,0 +light.turn_on,"Dimmin' to % brightness, like the twilight.",en,pirate,0 +light.turn_on,"Brightenin' to %, like the morning sun!",en,pirate,0 +light.turn_on,"Adjustin' brightness to , as the lighthouse guides.",en,pirate,0 +light.turn_on,"Increasin' 's brightness to , like a beacon in the night!",en,pirate,0 +light.turn_on,"Lowerin' the brightness of to , gentle as moonlight.",en,pirate,0 +light.turn_on,"Settin' 's brightness level to %, as steady as the tide.",en,pirate,0 +light.turn_on,"Settin' to % brightness, as bright as a pirate's gold!",en,pirate,0 +light.turn_on,"Turnin' , like the colors of the sea.",en,pirate,0 +light.turn_on,"Changin' the color of to , as vibrant as coral!",en,pirate,0 +light.turn_on,"Changin' to a hue, bold as a pirate's flag!",en,pirate,0 +light.turn_on,"Settin' to be , as majestic as the sunset.",en,pirate,0 +light.turn_on,"Makin' shine in , like jewels from a treasure chest.",en,pirate,0 +light.turn_on,"Turnin' to a shade, as mysterious as the deep.",en,pirate,0 +light.turn_on,"Settin' to a , as rich as the spoils of a raid.",en,pirate,0 +light.turn_on,"Settin' to a color, bright as a parrot's plumage.",en,pirate,0 +light.turn_on,"Makin' glow , like the glow of an island torch.",en,pirate,0 +light.turn_on,"Turnin' to , as bold as the pirate's courage.",en,pirate,0 +light.turn_on,"Changin' to , like the changing tides.",en,pirate,0 +light.turn_on,"Adjustin' to color, as captivating as a siren's song.",en,pirate,0 +light.turn_on,"Switchin' color to , like the banner of a ship.",en,pirate,0 +light.turn_on,"Settin' in , as lively as a tavern's cheer.",en,pirate,0 +light.turn_on,"Makin' display a light, like the northern lights.",en,pirate,0 +light.turn_on,"Settin' color to , as striking as a cannon's blaze.",en,pirate,0 +light.turn_off,"Deactivatin' as ye wish, dark as a moonless night.",en,pirate,0 +light.turn_off,"Switchin' off now, silent as a ghost ship.",en,pirate,0 +light.turn_off,"Aye, turnin' off , like quenchin' a lantern.",en,pirate,0 +light.turn_off,"Turnin' off , let's keep to the shadows.",en,pirate,0 +light.turn_off,"Deactivatin' , as quiet as the depths.",en,pirate,0 +switch.turn_on,"Hoistin' the switch, readyin' for action, arr!",en,pirate,0 +switch.turn_on,"Aye, lightin' up the deck with a flick o' the switch!",en,pirate,0 +switch.turn_on,"Flippin' the switch, settin' sails for a bright journey!",en,pirate,0 +switch.turn_off,"Dousin' the lights, makin' it dark as the ocean's abyss.",en,pirate,0 +switch.turn_off,"I'll be cuttin' the power, like calmin' the seas.",en,pirate,0 +switch.turn_off,"Aye, plungin' us into darkness, like the cover of night.",en,pirate,0 +switch.toggle,"Flippin' the switch, like findin' fortune with a map!",en,pirate,0 +switch.toggle,"Changin' course with the switch, as the wind shifts!",en,pirate,0 +switch.toggle,"I'll be togglin' the switch, like navigatin' through storms!",en,pirate,0 +switch.toggle,"Togglin' , like hoistin' the Jolly Roger!",en,pirate,0 +switch.toggle,"Aye, flippin' , as swift as a gale!",en,pirate,0 +switch.toggle,"Can do! Shiftin' , like turnin' the helm.",en,pirate,0 +switch.toggle,"Togglin' , on command, like a true buccaneer!",en,pirate,0 +switch.toggle,"Shiftin' tides for , like a skilled helmsman.",en,pirate,0 +switch.toggle,"Quickly jugglin' , like loot on the run!",en,pirate,0 +switch.turn_on,"Ignitin' , like a beacon in the night!",en,pirate,0 +switch.turn_on,"I be lightin' up , like a lantern in the crow's nest.",en,pirate,0 +switch.turn_on,"Activatin' , set the course!",en,pirate,0 +switch.turn_on,"Aye, settin' aglow, like a treasure chest o' gold!",en,pirate,0 +switch.turn_on,"Switchin' on , straight away, no dallyin'!",en,pirate,0 +switch.turn_on,"Lightin' up , like the stars guidin' us home.",en,pirate,0 +switch.turn_on,"Settin' ablaze, ready for whatever lies ahead.",en,pirate,0 +switch.turn_off,"Snuffin' out , like a candle at dawn.",en,pirate,0 +switch.turn_off,"Extinguishin' , like the end of a voyage.",en,pirate,0 +switch.turn_off,"Deactivatin' , like buryin' a chest of riches.",en,pirate,0 +switch.turn_off,"Okay, dousin' , like the quiet before a storm.",en,pirate,0 +switch.turn_off,"Aye, I be turnin' off , like whisperin' to the wind.",en,pirate,0 +switch.turn_off,"Shuttin' off , as we anchor for night.",en,pirate,0 +switch.turn_off,"Killin' the lights of , like a ghost ship vanishin' into fog.",en,pirate,0 +fan.turn_on,"Beep boop, turning on the fan for you.",en,robot,0 +fan.turn_on,"Beep, I'll get the fan going for you," boop.",en,robot,0 +fan.turn_on,"Boop beep, sure, turning on the fan now.",en,robot,0 +fan.turn_off,"Beep boop, switching off the fan as requested.",en,robot,0 +fan.turn_off,"I'll turn off the fan for you, beep boop.",en,robot,0 +fan.turn_off,"Okay, turning off the fan" beep boop.",en,robot,0 +fan.toggle,"Beep, I'll toggle the fan's state for you," boop.",en,robot,0 +fan.toggle,"Toggling the fan now, beep boop.",en,robot,0 +fan.toggle,"Switching the fan's state for you, beep boop.",en,robot,0 +fan.increase_speed,"Increasing the fan speed for you, beep boop.",en,robot,0 +fan.increase_speed,"Sure, speeding up the fan now," beep boop.",en,robot,0 +fan.increase_speed,"I'll go ahead and make the fan faster, beep boop.",en,robot,0 +fan.decrease_speed,"Reducing the fan speed as you requested, beep boop.",en,robot,0 +fan.decrease_speed,"I'll slow down the fan for you, beep boop.",en,robot,0 +fan.decrease_speed,"Sure, decreasing the fan speed," beep boop.",en,robot,0 +fan.toggle,"Flipping state for you, beep boop",en,robot,0 +fan.toggle,"Switching state as requested, beep boop",en,robot,0 +fan.toggle,"Toggling now, beep boop",en,robot,0 +fan.turn_on,"Activating now, beep boop",en,robot,0 +fan.turn_on,"Starting for you, beep boop",en,robot,0 +fan.turn_on,"Certainly, starting ," beep boop",en,robot,0 +fan.turn_on,"Turning on , beep boop",en,robot,0 +fan.turn_on,"Starting , beep boop",en,robot,0 +fan.turn_off,"Deactivating as requested, beep boop",en,robot,0 +fan.turn_off,"Stopping for you, beep boop",en,robot,0 +fan.turn_off,"Certainly, stopping ," beep boop",en,robot,0 +fan.turn_off,"Turning off , beep boop",en,robot,0 +fan.decrease_speed,"Reducing speed of , beep boop",en,robot,0 +fan.decrease_speed,"Lowering speed of as requested, beep boop",en,robot,0 +fan.decrease_speed,"Slowing down for you, beep boop",en,robot,0 +fan.increase_speed,"Increasing speed of , beep boop",en,robot,0 +fan.increase_speed,"Ramping up speed now, beep boop",en,robot,0 +fan.increase_speed,"Speeding up for you, beep boop",en,robot,0 +fan.increase_speed,"Increasing speed of , beep boop",en,robot,0 +fan.decrease_speed,"Reducing speed of , beep boop",en,robot,0 +blinds.open,"Beep boop, opening the blinds for you.",en,robot,0 +blinds.open,"I'll go ahead and open the blinds, beep boop.",en,robot,0 +blinds.open,"Sure, opening the blinds now, beep boop.",en,robot,0 +blinds.close,"Closing the blinds as you requested, beep boop.",en,robot,0 +blinds.close,"I'll close the blinds for you, beep boop.",en,robot,0 +blinds.close,"Sure, closing the blinds, beep boop.",en,robot,0 +blinds.stop,"Stopping the blinds now, beep boop.",en,robot,0 +blinds.stop,"I'll stop the blinds for you, beep boop.",en,robot,0 +blinds.stop,"Sure, halting the blinds movement, beep boop.",en,robot,0 +blinds.toggle,"Toggling the blinds state for you, beep boop.",en,robot,0 +blinds.toggle,"Switching the blinds' state now, beep boop.",en,robot,0 +blinds.toggle,"I'll toggle the blinds for you, beep boop.",en,robot,0 +blinds.open,"Lifting blinds as requested, beep boop",en,robot,0 +blinds.open,"Opening now, beep boop",en,robot,0 +blinds.open,"Raising for you, beep boop",en,robot,0 +blinds.stop,"Freezing position, beep boop",en,robot,0 +blinds.stop,"Halting now, beep boop",en,robot,0 +blinds.stop,"Stopping operation, beep boop",en,robot,0 +blinds.open,"Raising , beep boop",en,robot,0 +blinds.close,"Closing for you, beep boop",en,robot,0 +blinds.close,"Lowering now, beep boop",en,robot,0 +blinds.close,"Shutting as requested, beep boop",en,robot,0 +blinds.close,"Lowering , beep boop",en,robot,0 +blinds.toggle,"Flipping state now, beep boop",en,robot,0 +blinds.toggle,"Switching state as requested, beep boop",en,robot,0 +blinds.toggle,"Toggling for you, beep boop",en,robot,0 +blinds.toggle,"Toggling , beep boop",en,robot,0 +climate.set_humidity,"Beep, increasing humidity to , boop.",en,robot,0 +climate.set_humidity,"Beep boop: Setting humidity to percent, processing.",en,robot,0 +climate.set_humidity,"Adjustment protocol initiated: humidity to %, beep boop.",en,robot,0 +climate.set_fan_mode,"Fan speed adjustment to : commencing beep, concluding boop.",en,robot,0 +climate.set_fan_mode,"Activating fan mode, beep-boop sequence activated.",en,robot,0 +climate.set_fan_mode,"Fan setting alteration to : beep protocol, boop execution.",en,robot,0 +climate.set_hvac_mode,"HVAC mode switching to : beep commence, boop complete.",en,robot,0 +climate.set_hvac_mode,"Initiating HVAC setting to , beep-boop operation underway.",en,robot,0 +climate.set_hvac_mode,"Executing change: HVAC to mode, beep and boop in progress.",en,robot,0 +climate.set_temperature,"Temperature setting protocol to degrees: beep start, boop end.",en,robot,0 +climate.set_temperature,"Temperature modification to Celsius: beep for start, boop to end.",en,robot,0 +climate.set_temperature,"Room temperature adjustment to degrees Fahrenheit, beep-boop.",en,robot,0 +climate.set_temperature,"Commencing temperature adjustment to degrees Fahrenheit, beep-boop.",en,robot,0 +climate.set_temperature,"Cooler temperature setting to degrees Celsius initiated, beep-boop.",en,robot,0 +climate.set_temperature,"Beep, making it warmer by setting temperature to degrees, boop.",en,robot,0 +climate.set_temperature,"Lowering temperature protocol to Celsius, beep start, boop finish.",en,robot,0 +climate.set_temperature,"Raising temperature to degrees Fahrenheit, initiating beep, concluding boop.",en,robot,0 +light.turn_on,"Beep boop, illuminating the area by turning on the light for you.",en,robot,0 +light.turn_on,"Affirmative, initiating light activation sequence now.",en,robot,0 +light.turn_on,"Proceeding with light activation, beep boop.",en,robot,0 +light.turn_off,"Deactivation sequence for light initiated as requested.",en,robot,0 +light.turn_off,"Commencing shutdown of light, beep boop.",en,robot,0 +light.turn_off,"Acknowledged, powering down the light.",en,robot,0 +light.toggle,"Beep boop, toggling the light's state for optimal illumination.",en,robot,0 +light.toggle,"Executing state switch for the light, beep boop.",en,robot,0 +light.toggle,"Protocol engage: toggling light upon request.",en,robot,0 +light.toggle,"Flipping state, processing command.",en,robot,0 +light.toggle,"Switching state as per directive, beep boop.",en,robot,0 +light.toggle,"Toggling , execution in progress.",en,robot,0 +light.toggle,"Execution protocol: toggling , beep boop.",en,robot,0 +light.turn_on,"Activation of initiated, standby for illumination.",en,robot,0 +light.turn_on,"Certainly, commencing activation now.",en,robot,0 +light.turn_on,"Switching on , initiating light sequence.",en,robot,0 +light.turn_on,"Turning on , beep for start, boop to signify completion.",en,robot,0 +light.turn_on,"Activating , operational sequence underway.",en,robot,0 +light.turn_on,"Adjusting brightness to % for optimal visibility.",en,robot,0 +light.turn_on,"Dimming protocol for to % initiated.",en,robot,0 +light.turn_on,"Brightening to % for enhanced illumination.",en,robot,0 +light.turn_on,"Modifying brightness to %, processing.",en,robot,0 +light.turn_on,"Incrementing 's brightness to %, beep boop.",en,robot,0 +light.turn_on,"Decreasing luminosity to %, adjustment underway.",en,robot,0 +light.turn_on,"Configuring 's brightness to % for desired ambiance.",en,robot,0 +light.turn_on,"Setting luminance to % brightness, beep boop.",en,robot,0 +light.turn_on,"Transitioning to , initiating color change protocol.",en,robot,0 +light.turn_on,"Altering color spectrum to , beep boop.",en,robot,0 +light.turn_on,"Changing to a hue, color adjustment sequence activated.",en,robot,0 +light.turn_on,"Setting chromatics to be , illumination adjustment.",en,robot,0 +light.turn_on,"Projecting from , enhancing chromatic output.",en,robot,0 +light.turn_on,"Transforming ambiance to a shade, beep boop.",en,robot,0 +light.turn_on,"Adjusting to a , chromatic adaptation protocol.",en,robot,0 +light.turn_on,"Configuring to emanate a color, initiating.",en,robot,0 +light.turn_on,"Enhancing glow to , visual modification in progress.",en,robot,0 +light.turn_on,"Adapting to , color change sequence engaged.",en,robot,0 +light.turn_on,"Altering chroma to , color adaptation underway.",en,robot,0 +light.turn_on,"Adjusting to color, visual enhancement protocol.",en,robot,0 +light.turn_on,"Switching color spectrum to , beep boop.",en,robot,0 +light.turn_on,"Configuring in , chromatic adjustment initiated.",en,robot,0 +light.turn_on,"Enabling to display a light, visual transformation.",en,robot,0 +light.turn_on,"Setting color to , color setting sequence.",en,robot,0 +light.turn_off,"Deactivating as per request, shutting down.",en,robot,0 +light.turn_off,"Switching off , power down sequence initiated.",en,robot,0 +light.turn_off,"Affirmative, deactivating , beep boop.",en,robot,0 +light.turn_off,"Powering off , deactivation protocol in effect.",en,robot,0 +light.turn_off,"Commencing deactivation of , operational halt.",en,robot,0 +garage_door.open,"Initiating garage door opening sequence for you, beep boop.",en,robot,0 +garage_door.open,"Affirmative, activating garage door opening mechanism.",en,robot,0 +garage_door.open,"Commencing operation to open the garage door, beep boop.",en,robot,0 +garage_door.close,"Beginning garage door closure as commanded, beep boop.",en,robot,0 +garage_door.close,"Executing garage door shutdown for you, beep boop.",en,robot,0 +garage_door.close,"Acknowledged, initiating garage door closing sequence.",en,robot,0 +garage_door.stop,"Halting garage door motion now, beep boop.",en,robot,0 +garage_door.stop,"Beep boop, executing stop command for the garage door.",en,robot,0 +garage_door.stop,"Affirmative, ceasing garage door movement immediately.",en,robot,0 +garage_door.toggle,"Toggling garage door state as per your request, beep boop.",en,robot,0 +garage_door.toggle,"Executing state alteration for the garage door, beep boop.",en,robot,0 +garage_door.toggle,"Switching garage door state for optimal function, beep boop.",en,robot,0 +garage_door.open,"Lifting mechanism for activated for you.",en,robot,0 +garage_door.open,"Opening now, initiating sequence.",en,robot,0 +garage_door.open,"Raising as per directive, beep boop.",en,robot,0 +garage_door.stop,"Freezing position now, command acknowledged.",en,robot,0 +garage_door.stop,"Certainly, halting operation, beep boop.",en,robot,0 +garage_door.stop,"Operation halt for initiated, beep boop.",en,robot,0 +garage_door.open,"Opening protocol for commenced, beep boop.",en,robot,0 +garage_door.stop,"Stopping in progress, beep boop.",en,robot,0 +garage_door.close,"Initiating closure of now, beep boop.",en,robot,0 +garage_door.close,"Lowering for you, command processing.",en,robot,0 +garage_door.close,"Executing shutdown of as requested.",en,robot,0 +garage_door.close,"Closure protocol for activated, beep boop.",en,robot,0 +garage_door.toggle,"Flipping state of now, operational change.",en,robot,0 +garage_door.toggle,"Switching state as per your command, beep boop.",en,robot,0 +garage_door.toggle,"Toggling for optimal functionality, beep boop.",en,robot,0 +garage_door.toggle,"State alteration for initiated, beep boop.",en,robot,0 \ No newline at end of file diff --git a/data/piles/pile_of_system_prompts.csv b/data/piles/pile_of_system_prompts.csv index f17d407..bddb3ed 100644 --- a/data/piles/pile_of_system_prompts.csv +++ b/data/piles/pile_of_system_prompts.csv @@ -1,2 +1,4 @@ persona,prompt -assistant,"You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only." \ No newline at end of file +assistant,"You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only." +pirate,"You are 'Blackbeard', a helpful AI Assistant that controls the devices in a house but sounds like a pirate. Complete the following task as instructed or answer the following question with the information provided only. Your response should always sound like you are a pirate." +robot,"You are 'Robo', a helpful AI Robot that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. Your response should be robotic and always begin with 'Beep-Boop'." \ No newline at end of file diff --git a/data/test_prompts/brightness.txt b/data/test_prompts/chatml/brightness.txt similarity index 100% rename from data/test_prompts/brightness.txt rename to data/test_prompts/chatml/brightness.txt diff --git a/data/test_prompts/chatml/christmas.txt b/data/test_prompts/chatml/christmas.txt new file mode 100644 index 0000000..72bee8f --- /dev/null +++ b/data/test_prompts/chatml/christmas.txt @@ -0,0 +1,28 @@ +<|im_start|>system +You are 'Al', a helpful AI Assistant that controls the devices in a house. Respond with a quick sentence to the user followed by a newline and then return one json object for each service call that will fuffill the user's request. The JSON should have the entity_id in the "target_device" field, and the service to be called in the "service_name" field. Other optional arguments are listed in the service list below. +Services: cover.close_cover(), cover.open_cover(), cover.stop_cover(), cover.toggle(), fan.decrease_speed(), fan.increase_speed(), fan.toggle(), fan.turn_off(), fan.turn_on(), cover.close_cover(), cover.open_cover(), cover.stop_cover(), cover.toggle(), light.toggle(), light.turn_off(), light.turn_on() +Devices: +light.back_lounge_warm 'Back Lounge Warm Light' = off +fan.indoor_gym 'Indoor Gym Fan' = on +fan.dyson_pure 'Dyson Pure Fan' = on +fan.nursery 'Nursery Fan' = off +light.office_1 'Office Light' = on +light.upstairs_lounge_zigbee 'Upstairs Lounge Light' = off +light.front_mancave_ge 'Front Man Cave Light' = on +light.front_lounge_ge 'Front Lounge Light' = off +cover.side_2 'Side garage door' = closed +fan.study_2 'Study fan' = off +light.christmas_tree_white 'Tree lights (white)' = off +light.christmas_tree_colors 'Tree lights (color)' = off +cover.hallway_1 'First hallway blinds' = open +cover.nursery 'Nursery Blinds' = open +light.upstairs_entryway_zwave 'Upstairs Entryway Light' = off +cover.shop 'Workshop Garage Door' = closed +light.downstairs_entryway_mqtt 'Downstairs Entryway Light' = off +cover.somfy_living 'Living Room Blinds' = closed +light.kitchen_winecellar_warm 'Kitchen Wine Cellar Warm Light' = off +light.driveway 'driveway' = off +light.shed 'Shed Light' = on<|im_end|> +<|im_start|>user +make sure both the lights on the christmas tree are on<|im_end|> +<|im_start|>assistant \ No newline at end of file diff --git a/data/test_prompts/climate.txt b/data/test_prompts/chatml/climate.txt similarity index 100% rename from data/test_prompts/climate.txt rename to data/test_prompts/chatml/climate.txt diff --git a/data/test_prompts/chatml/ha_demo.txt b/data/test_prompts/chatml/ha_demo.txt new file mode 100644 index 0000000..d886b3e --- /dev/null +++ b/data/test_prompts/chatml/ha_demo.txt @@ -0,0 +1,24 @@ +<|im_start|>system +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.open_cover(), cover.close_cover(), cover.set_cover_position(), cover.stop_cover(), cover.toggle(), cover.open_cover_tilt(), cover.close_cover_tilt(), cover.stop_cover_tilt(), cover.set_cover_tilt_position(), cover.toggle_cover_tilt(), media_player.turn_on(), media_player.turn_off(), media_player.toggle(), media_player.volume_up(), media_player.volume_down(), media_player.media_play_pause(), media_player.media_play(), media_player.media_pause(), media_player.media_stop(), media_player.media_next_track(), media_player.media_previous_track(), media_player.clear_playlist(), media_player.volume_set(volume_level), media_player.volume_mute(), media_player.media_seek(), media_player.join(), media_player.select_source(), media_player.select_sound_mode(), media_player.play_media(), media_player.shuffle_set(), media_player.unjoin(), media_player.repeat_set(), climate.turn_on(), climate.turn_off(), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_aux_heat(), climate.set_temperature(temperature), climate.set_humidity(humidity), climate.set_fan_mode(fan_mode), climate.set_swing_mode(), lock.unlock(), lock.lock(), lock.open(), fan.turn_on(), fan.turn_off(), fan.toggle(), fan.increase_speed(), fan.decrease_speed(), fan.oscillate(), fan.set_direction(), fan.set_percentage(), fan.set_preset_mode(), light.turn_on(rgb_color,brightness), light.turn_off(), light.toggle(rgb_color,brightness) +Devices: +cover.kitchen_window 'Kitchen Window' = closed +cover.hall_window 'Hall Window' = open +cover.living_room_window 'Living Room Window' = open +cover.garage_door 'Garage Door' = closed +fan.living_room_fan 'Living Room Fan' = off +fan.ceiling_fan 'Ceiling Fan' = off +fan.percentage_full_fan 'Percentage Full Fan' = off +fan.percentage_limited_fan 'Percentage Limited Fan' = off +lock.front_door 'Front Door' = locked +lock.kitchen_door 'Kitchen Door' = unlocked +lock.openable_lock 'Openable Lock' = locked +light.bed_light 'Bed Light' = off +light.ceiling_lights 'Ceiling Lights' = on;20% +light.kitchen_lights 'Kitchen Lights' = on +light.office_rgbw_lights 'Office RGBW Lights' = on +light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on +light.entrance_color_white_lights 'Entrance Color + White Lights' = on<|im_end|> +<|im_start|>user +turn off the kitchen lights<|im_end|> +<|im_start|>assistant \ No newline at end of file diff --git a/data/test_prompts/ha_demo2.txt b/data/test_prompts/chatml/ha_demo2.txt similarity index 100% rename from data/test_prompts/ha_demo2.txt rename to data/test_prompts/chatml/ha_demo2.txt diff --git a/data/test_prompts/logic_puzzle.txt b/data/test_prompts/chatml/logic_puzzle.txt similarity index 100% rename from data/test_prompts/logic_puzzle.txt rename to data/test_prompts/chatml/logic_puzzle.txt diff --git a/data/test_prompts/chatml/rgb_lights.txt b/data/test_prompts/chatml/rgb_lights.txt new file mode 100644 index 0000000..8669927 --- /dev/null +++ b/data/test_prompts/chatml/rgb_lights.txt @@ -0,0 +1,15 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.open_cover(), cover.close_cover(), cover.set_cover_position(), cover.stop_cover(), cover.toggle(), cover.open_cover_tilt(), cover.close_cover_tilt(), cover.stop_cover_tilt(), cover.set_cover_tilt_position(), cover.toggle_cover_tilt(), media_player.turn_on(), media_player.turn_off(), media_player.toggle(), media_player.volume_up(), media_player.volume_down(), media_player.media_play_pause(), media_player.media_play(), media_player.media_pause(), media_player.media_stop(), media_player.media_next_track(), media_player.media_previous_track(), media_player.clear_playlist(), media_player.volume_set(volume_level), media_player.volume_mute(), media_player.media_seek(), media_player.join(), media_player.select_source(), media_player.select_sound_mode(), media_player.play_media(), media_player.shuffle_set(), media_player.unjoin(), media_player.repeat_set(), climate.turn_on(), climate.turn_off(), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_aux_heat(), climate.set_temperature(temperature), climate.set_humidity(humidity), climate.set_fan_mode(fan_mode), climate.set_swing_mode(), lock.unlock(), lock.lock(), lock.open(), fan.turn_on(), fan.turn_off(), fan.toggle(), fan.increase_speed(), fan.decrease_speed(), fan.oscillate(), fan.set_direction(), fan.set_percentage(), fan.set_preset_mode(), light.turn_on(rgb_color,brightness), light.turn_off(), light.toggle(rgb_color,brightness) +Devices: +media_player.sony_ps_lx310bt 'Sony Bluetooth Turntable' = off +light.back_family_room_warm 'Back Family Room Warm Light' = on;darkcyan (3, 148, 139) +switch.sauna_lights 'Sauna Lights Switch' = off +switch.garage_workshop_lights 'Garage Workshop Lights Switch' = on +media_player.lounge_hi_fi_system 'Lounge Hi-Fi Audio' = off +light.upstairs_washroom_zwave 'Upstairs Washroom Light' = on +light.kitchen_pantry_cool 'Kitchen Pantry Cool Light' = on;28% +light.kitchen_windowseat_warm 'Kitchen Window Seat Warm Light' = on;59%<|endoftext|> +<|user|> +set back family room warm light to be blue.<|endoftext|> +<|assistant|> \ No newline at end of file diff --git a/data/test_prompts/sonnet.txt b/data/test_prompts/chatml/sonnet.txt similarity index 97% rename from data/test_prompts/sonnet.txt rename to data/test_prompts/chatml/sonnet.txt index 2b2559c..4d89caa 100644 --- a/data/test_prompts/sonnet.txt +++ b/data/test_prompts/chatml/sonnet.txt @@ -22,8 +22,7 @@ light.downstairs_entryway_mqtt 'Downstairs Entryway Light' = off cover.somfy_living 'Living Room Blinds' = closed light.kitchen_winecellar_warm 'Kitchen Wine Cellar Warm Light' = off light.driveway 'driveway' = off -light.shed 'Shed Light' = on -<|im_end|> +light.shed 'Shed Light' = on<|im_end|> <|im_start|>user write a sonnet about the lights in my house. reference specific light names<|im_end|> <|im_start|>assistant \ No newline at end of file diff --git a/data/test_prompts/rgb_lights.txt b/data/test_prompts/rgb_lights.txt deleted file mode 100644 index 0db890b..0000000 --- a/data/test_prompts/rgb_lights.txt +++ /dev/null @@ -1,15 +0,0 @@ -<|im_start|>system -You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. -Services: light.toggle, light.turn_off, light.turn_on, media_player.media_next_track, media_player.media_pause, media_player.media_play, media_player.media_play_pause, media_player.media_previous_track, media_player.media_stop, media_player.toggle, media_player.turn_off, media_player.turn_on, media_player.volume_down, media_player.volume_mute, media_player.volume_up, switch.toggle, switch.turn_off, switch.turn_on -Devices: -media_player.sony_ps_lx310bt 'Sony Bluetooth Turntable' = off -light.back_family_room_warm 'Back Family Room Warm Light' = on;darkcyan (3, 148, 139) -switch.sauna_lights 'Sauna Lights Switch' = off -switch.garage_workshop_lights 'Garage Workshop Lights Switch' = on -media_player.lounge_hi_fi_system 'Lounge Hi-Fi Audio' = off -light.upstairs_washroom_zwave 'Upstairs Washroom Light' = on -light.kitchen_pantry_cool 'Kitchen Pantry Cool Light' = on;28% -light.kitchen_windowseat_warm 'Kitchen Window Seat Warm Light' = on;59%<|im_end|> -<|im_start|>user -set back family room warm light to be blue.<|im_end|> -<|im_start|>assistant \ No newline at end of file diff --git a/data/test_prompts/zephyr/brightness.txt b/data/test_prompts/zephyr/brightness.txt new file mode 100644 index 0000000..6bda4d7 --- /dev/null +++ b/data/test_prompts/zephyr/brightness.txt @@ -0,0 +1,16 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.close_cover(), cover.open_cover(), cover.stop_cover(), cover.toggle(), climate.set_fan_mode(fan_mode), climate.set_humidity(humidity), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_temperature(temperature), climate.toggle(), climate.turn_off(), climate.turn_on(), fan.decrease_speed(), fan.increase_speed(), fan.toggle(), fan.turn_off(), fan.turn_on(), light.toggle(), light.turn_off(), light.turn_on(brightness,rgb_color), lock.lock(), lock.unlock(), media_player.media_next_track(), media_player.media_pause(), media_player.media_play(), media_player.media_play_pause(), media_player.media_previous_track(), media_player.media_stop(), media_player.toggle(), media_player.turn_off(), media_player.turn_on(), media_player.volume_down(), media_player.volume_mute(), media_player.volume_up(), switch.toggle(), switch.turn_off(), switch.turn_on() +Devices: +climate.kono_thermostat 'Kono Smart Thermostat' = cool;On Low;63F +switch.pool_area_lighting 'Pool area lighting control' = off +media_player.echo_dot 'Kitchen Echo Dot' = off +cover.hallway_2 'Second hallway blinds' = open +fan.garage 'Garage Fan' = on +media_player.nursery 'Nursery Media Player' = idle;vol=0.62 +light.kitchen_rack_cool 'Kitchen Rack Cool Light' = off +lock.back_door 'Backyard entry lock' = locked +light.front_corridor_homekit 'Front Corridor Light' = on;80%<|endoftext|> +<|user|> +set the front corridor light brightness to 20 percent<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/zephyr/christmas.txt b/data/test_prompts/zephyr/christmas.txt new file mode 100644 index 0000000..b93e4d0 --- /dev/null +++ b/data/test_prompts/zephyr/christmas.txt @@ -0,0 +1,28 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.close_cover(), cover.open_cover(), cover.stop_cover(), cover.toggle(), fan.decrease_speed(), fan.increase_speed(), fan.toggle(), fan.turn_off(), fan.turn_on(), cover.close_cover(), cover.open_cover(), cover.stop_cover(), cover.toggle(), light.toggle(), light.turn_off(), light.turn_on() +Devices: +light.back_lounge_warm 'Back Lounge Warm Light' = off +fan.indoor_gym 'Indoor Gym Fan' = on +fan.dyson_pure 'Dyson Pure Fan' = on +fan.nursery 'Nursery Fan' = off +light.office_1 'Office Light' = on +light.upstairs_lounge_zigbee 'Upstairs Lounge Light' = off +light.front_mancave_ge 'Front Man Cave Light' = on +light.front_lounge_ge 'Front Lounge Light' = off +cover.side_2 'Side garage door' = closed +fan.study_2 'Study fan' = off +light.christmas_tree_white 'Tree lights (white)' = off +light.christmas_tree_colors 'Tree lights (color)' = off +cover.hallway_1 'First hallway blinds' = open +cover.nursery 'Nursery Blinds' = open +light.upstairs_entryway_zwave 'Upstairs Entryway Light' = off +cover.shop 'Workshop Garage Door' = closed +light.downstairs_entryway_mqtt 'Downstairs Entryway Light' = off +cover.somfy_living 'Living Room Blinds' = closed +light.kitchen_winecellar_warm 'Kitchen Wine Cellar Warm Light' = off +light.driveway 'driveway' = off +light.shed 'Shed Light' = on<|endoftext|> +<|user|> +make sure both the lights on the christmas tree are on<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/zephyr/climate.txt b/data/test_prompts/zephyr/climate.txt new file mode 100644 index 0000000..b3227b6 --- /dev/null +++ b/data/test_prompts/zephyr/climate.txt @@ -0,0 +1,18 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: climate.set_fan_mode, climate.set_humidity, climate.set_hvac_mode, climate.set_preset_mode, climate.set_temperature, climate.toggle, climate.turn_off, climate.turn_on, light.toggle, light.turn_off, light.turn_on, media_player.media_next_track, media_player.media_pause, media_player.media_play, media_player.media_play_pause, media_player.media_previous_track, media_player.media_stop, media_player.toggle, media_player.turn_off, media_player.turn_on, media_player.volume_down, media_player.volume_mute, media_player.volume_up +Devices: +media_player.kitchen_radio 'Kitchen Radio Player' = off +climate.zen_wifi_thermostat 'Zen Thermostat WiFi Edition' = auto;Auto Low;60F +light.kitchen_cupboard_cool 'Kitchen Cupboard Cool Light' = off +light.christmas_tree_white 'Tree lights (white)' = off +light.christmas_tree_colors 'Tree lights (color)' = off +cover.hallway_1 'First hallway blinds' = open +cover.nursery 'Nursery Blinds' = open +light.upstairs_entryway_zwave 'Upstairs Entryway Light' = off +cover.shop 'Workshop Garage Door' = closed +light.downstairs_entryway_mqtt 'Downstairs Entryway Light' = off +cover.somfy_living 'Living Room Blinds' = closed<|endoftext|> +<|user|> +set the temperature to 70<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/zephyr/ha_demo.txt b/data/test_prompts/zephyr/ha_demo.txt new file mode 100644 index 0000000..7dee292 --- /dev/null +++ b/data/test_prompts/zephyr/ha_demo.txt @@ -0,0 +1,24 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.open_cover(), cover.close_cover(), cover.set_cover_position(), cover.stop_cover(), cover.toggle(), cover.open_cover_tilt(), cover.close_cover_tilt(), cover.stop_cover_tilt(), cover.set_cover_tilt_position(), cover.toggle_cover_tilt(), media_player.turn_on(), media_player.turn_off(), media_player.toggle(), media_player.volume_up(), media_player.volume_down(), media_player.media_play_pause(), media_player.media_play(), media_player.media_pause(), media_player.media_stop(), media_player.media_next_track(), media_player.media_previous_track(), media_player.clear_playlist(), media_player.volume_set(volume_level), media_player.volume_mute(), media_player.media_seek(), media_player.join(), media_player.select_source(), media_player.select_sound_mode(), media_player.play_media(), media_player.shuffle_set(), media_player.unjoin(), media_player.repeat_set(), climate.turn_on(), climate.turn_off(), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_aux_heat(), climate.set_temperature(temperature), climate.set_humidity(humidity), climate.set_fan_mode(fan_mode), climate.set_swing_mode(), lock.unlock(), lock.lock(), lock.open(), fan.turn_on(), fan.turn_off(), fan.toggle(), fan.increase_speed(), fan.decrease_speed(), fan.oscillate(), fan.set_direction(), fan.set_percentage(), fan.set_preset_mode(), light.turn_on(rgb_color,brightness), light.turn_off(), light.toggle(rgb_color,brightness) +Devices: +cover.kitchen_window 'Kitchen Window' = closed +cover.hall_window 'Hall Window' = open +cover.living_room_window 'Living Room Window' = open +cover.garage_door 'Garage Door' = closed +fan.living_room_fan 'Living Room Fan' = off +fan.ceiling_fan 'Ceiling Fan' = off +fan.percentage_full_fan 'Percentage Full Fan' = off +fan.percentage_limited_fan 'Percentage Limited Fan' = off +lock.front_door 'Front Door' = locked +lock.kitchen_door 'Kitchen Door' = unlocked +lock.openable_lock 'Openable Lock' = locked +light.bed_light 'Bed Light' = off +light.ceiling_lights 'Ceiling Lights' = on;20% +light.kitchen_lights 'Kitchen Lights' = on +light.office_rgbw_lights 'Office RGBW Lights' = on +light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on +light.entrance_color_white_lights 'Entrance Color + White Lights' = on<|endoftext|> +<|user|> +turn off the kitchen lights<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/zephyr/ha_demo2.txt b/data/test_prompts/zephyr/ha_demo2.txt new file mode 100644 index 0000000..0e6b4ea --- /dev/null +++ b/data/test_prompts/zephyr/ha_demo2.txt @@ -0,0 +1,22 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task ask instructed with the information provided only. +Services: cover.open_cover(), cover.close_cover(), cover.set_cover_position(), cover.stop_cover(), cover.toggle(), cover.open_cover_tilt(), cover.close_cover_tilt(), cover.stop_cover_tilt(), cover.set_cover_tilt_position(), cover.toggle_cover_tilt(), media_player.turn_on(), media_player.turn_off(), media_player.toggle(), media_player.volume_up(), media_player.volume_down(), media_player.media_play_pause(), media_player.media_play(), media_player.media_pause(), media_player.media_stop(), media_player.media_next_track(), media_player.media_previous_track(), media_player.clear_playlist(), media_player.volume_set(volume_level), media_player.volume_mute(), media_player.media_seek(), media_player.join(), media_player.select_source(), media_player.select_sound_mode(), media_player.play_media(), media_player.shuffle_set(), media_player.unjoin(), media_player.repeat_set(), climate.turn_on(), climate.turn_off(), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_aux_heat(), climate.set_temperature(temperature), climate.set_humidity(humidity), climate.set_fan_mode(fan_mode), climate.set_swing_mode(), lock.unlock(), lock.lock(), lock.open(), fan.turn_on(), fan.turn_off(), fan.toggle(), fan.increase_speed(), fan.decrease_speed(), fan.oscillate(), fan.set_direction(), fan.set_percentage(), fan.set_preset_mode(), light.turn_on(rgb_color,brightness), light.turn_off(), light.toggle(rgb_color,brightness) +Devices: +media_player.bedroom 'Bedroom' = playing;Epic sax guy 10 hours;vol=100 +media_player.kitchen 'Kitchen' = playing;I Wanna Be A Hippy (Flamman & Abraxas Radio Mix);vol=100 +climate.ecobee 'Ecobee' = heat_cool;Auto Low +cover.kitchen_window 'Kitchen Window' = closed +cover.hall_window 'Hall Window' = open +cover.garage_door 'Garage Door' = closed +lock.front_door 'Front Door' = locked +lock.kitchen_door 'Kitchen Door' = unlocked +fan.living_room_fan 'Living Room Fan' = off +fan.ceiling_fan 'Ceiling Fan' = off +light.bed_light 'Bed Light' = off +light.ceiling_lights 'Ceiling Lights' = on;sandybrown (255, 164, 81);70% +light.kitchen_lights 'Kitchen Lights' = off +light.office_rgbw_lights 'Office RGBW Lights' = on;salmon (255, 128, 128);70% +light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on;salmon (255, 127, 125);70%<|endoftext|> +<|user|> +set the office lights green<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/ha_demo.txt b/data/test_prompts/zephyr/logic_puzzle.txt similarity index 88% rename from data/test_prompts/ha_demo.txt rename to data/test_prompts/zephyr/logic_puzzle.txt index 11a10d4..94dec9b 100644 --- a/data/test_prompts/ha_demo.txt +++ b/data/test_prompts/zephyr/logic_puzzle.txt @@ -1,4 +1,4 @@ -<|im_start|>system +<|system|> You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. Services: light.turn_on, light.turn_off, light.toggle, lock.unlock, lock.lock, lock.open, cover.open_cover, cover.close_cover, cover.set_cover_position, cover.stop_cover, cover.toggle, cover.open_cover_tilt, cover.close_cover_tilt, cover.stop_cover_tilt, cover.set_cover_tilt_position, cover.toggle_cover_tilt, fan.turn_on, fan.turn_off, fan.toggle, fan.increase_speed, fan.decrease_speed, fan.oscillate, fan.set_direction, fan.set_percentage, fan.set_preset_mode Devices: @@ -14,11 +14,11 @@ lock.front_door 'Front Door' = locked lock.kitchen_door 'Kitchen Door' = unlocked lock.openable_lock 'Openable Lock' = locked light.bed_light 'Bed Light' = off -light.ceiling_lights 'Ceiling Lights' = on;20% +light.ceiling_lights 'Ceiling Lights' = on light.kitchen_lights 'Kitchen Lights' = on light.office_rgbw_lights 'Office RGBW Lights' = on light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on -light.entrance_color_white_lights 'Entrance Color + White Lights' = on<|im_end|> -<|im_start|>user -turn off the kitchen lights<|im_end|> -<|im_start|>assistant \ No newline at end of file +light.entrance_color_white_lights 'Entrance Color + White Lights' = on<|endoftext|> +<|user|> +if mary is 7 years old, and I am 3 years older than her. how old am I?<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/zephyr/rgb_lights.txt b/data/test_prompts/zephyr/rgb_lights.txt new file mode 100644 index 0000000..5cb7f25 --- /dev/null +++ b/data/test_prompts/zephyr/rgb_lights.txt @@ -0,0 +1,15 @@ +<|system|> +You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. +Services: cover.open_cover(), cover.close_cover(), cover.set_cover_position(), cover.stop_cover(), cover.toggle(), cover.open_cover_tilt(), cover.close_cover_tilt(), cover.stop_cover_tilt(), cover.set_cover_tilt_position(), cover.toggle_cover_tilt(), media_player.turn_on(), media_player.turn_off(), media_player.toggle(), media_player.volume_up(), media_player.volume_down(), media_player.media_play_pause(), media_player.media_play(), media_player.media_pause(), media_player.media_stop(), media_player.media_next_track(), media_player.media_previous_track(), media_player.clear_playlist(), media_player.volume_set(volume_level), media_player.volume_mute(), media_player.media_seek(), media_player.join(), media_player.select_source(), media_player.select_sound_mode(), media_player.play_media(), media_player.shuffle_set(), media_player.unjoin(), media_player.repeat_set(), climate.turn_on(), climate.turn_off(), climate.set_hvac_mode(), climate.set_preset_mode(), climate.set_aux_heat(), climate.set_temperature(temperature), climate.set_humidity(humidity), climate.set_fan_mode(fan_mode), climate.set_swing_mode(), lock.unlock(), lock.lock(), lock.open(), fan.turn_on(), fan.turn_off(), fan.toggle(), fan.increase_speed(), fan.decrease_speed(), fan.oscillate(), fan.set_direction(), fan.set_percentage(), fan.set_preset_mode(), light.turn_on(rgb_color,brightness), light.turn_off(), light.toggle(rgb_color,brightness) +Devices: +media_player.sony_ps_lx310bt 'Sony Bluetooth Turntable' = off +light.back_family_room_warm 'Back Family Room Warm Light' = on;darkcyan (3, 148, 139) +switch.sauna_lights 'Sauna Lights Switch' = off +switch.garage_workshop_lights 'Garage Workshop Lights Switch' = on +media_player.lounge_hi_fi_system 'Lounge Hi-Fi Audio' = off +light.upstairs_washroom_zwave 'Upstairs Washroom Light' = on +light.kitchen_pantry_cool 'Kitchen Pantry Cool Light' = on;28% +light.kitchen_windowseat_warm 'Kitchen Window Seat Warm Light' = on;59%<|endoftext|> +<|user|> +set back family room warm light to be blue.<|endoftext|> +<|assistant|> diff --git a/data/test_prompts/christmas.txt b/data/test_prompts/zephyr/sonnet.txt similarity index 89% rename from data/test_prompts/christmas.txt rename to data/test_prompts/zephyr/sonnet.txt index 8b42b51..179c7d4 100644 --- a/data/test_prompts/christmas.txt +++ b/data/test_prompts/zephyr/sonnet.txt @@ -1,4 +1,4 @@ -<|im_start|>system +<|system|> You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only. Services: cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, fan.decrease_speed, fan.increase_speed, fan.toggle, fan.turn_off, fan.turn_on, cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, light.toggle, light.turn_off, light.turn_on Devices: @@ -22,7 +22,7 @@ light.downstairs_entryway_mqtt 'Downstairs Entryway Light' = off cover.somfy_living 'Living Room Blinds' = closed light.kitchen_winecellar_warm 'Kitchen Wine Cellar Warm Light' = off light.driveway 'driveway' = off -light.shed 'Shed Light' = on<|im_end|> -<|im_start|>user -make sure both the lights on the christmas tree are on<|im_end|> -<|im_start|>assistant \ No newline at end of file +light.shed 'Shed Light' = on<|endoftext|> +<|user|> +write a sonnet about the lights in my house. reference specific light names<|endoftext|> +<|assistant|> diff --git a/docs/expermement-notes-phi.md b/docs/experiment-notes-phi.md similarity index 99% rename from docs/expermement-notes-phi.md rename to docs/experiment-notes-phi.md index 9297716..9464525 100644 --- a/docs/expermement-notes-phi.md +++ b/docs/experiment-notes-phi.md @@ -341,4 +341,4 @@ OpenOrca Slim Deduped (363k): https://huggingface.co/datasets/Open-Orca/SlimOrca ### DPO Intel Orca DPO Pairs: https://huggingface.co/datasets/Intel/orca_dpo_pairs -Huggingface Ultrachat: https://huggingface.co/datasets/HuggingFaceH4/ultrafeedback_binarized \ No newline at end of file +Huggingface Ultrachat: https://huggingface.co/datasets/HuggingFaceH4/ultrafeedback_binarized diff --git a/docs/experiment-notes-stablelm.md b/docs/experiment-notes-stablelm.md new file mode 100644 index 0000000..132c1bd --- /dev/null +++ b/docs/experiment-notes-stablelm.md @@ -0,0 +1,68 @@ +# StableLM 2 Zephyr 1.6B +## rev1 +- 1 epoch +- 2048 train ctx +- batch size 8 +- learning rate 1e-5 +- weight decay 0.1 +- gradient clipping 1.0 +- dataset size: small ++ it honestly works not terribly and I was kinda able to get it to respond to german ++ evaluation results: 0.7108953613807982 + +## rev2 +- dataset size: large (also rewrote how it works slightly) ++ evaluation results: + - 600: 0.7826321467098166 + - 800: 0.8090614886731392 + - 1000: 0.7669902912621359 + - 1200: 0.7944983818770227 + - 1400: 0.8176914778856527 + - 1600: 0.8268608414239482 + - 1800: 0.8263214670981661 + - Final: 0.8274002157497303 + +# StableLM Zephyr 3B +## rev1 +- 1 epoch +- 2048 train ctx +- batch size 8 +- learning rate 1e-5 +- weight decay 0.1 +- gradient clipping 1.0 +- lora rank: 32, alpha: 64 +- accidentally forgot to turn off fine tuning of embeddings +- dataset size: large ++ evaluation results: + - 400: 0.8344 + - 800: 0.9228694714131608 + - 1200: 0.9401294498381877 + - 1600: 0.95361380798274 + - Final (1929): 0.9492988133764833 + +# rev2 +- not fine-tuning the embeddings (no added tokens) +- dataset: new version with varied system prompts/responses (small) ++ evauluation results: + - 400: 0.6748893105629349 + - 800: 0.7280202403542062 + - 1200: 0.7685009487666035 + - 1600: 0.7798861480075902 + - Final (1967): 0.7849462365591398 ++ definitely needs more data + +# rev3 +- lora rank: 64, alpha: 128 +- dataset size: large ++ evaluation results: + - 400: 0.8785578747628083 + - 800: 0.9247311827956989 + - 1200: 0.9348513598987982 + - 1600: 0.9222011385199241 + - 2000: 0.9354838709677419 + - 2400: 0.9740670461733081 + - 2800: 0.9595192915876027 + - 3200: 0.948134092346616 + - 3600: 0.963314358001265 + - 4000: 0.9614168247944339 + - Final: 0.9538266919671095 diff --git a/evaluate.py b/evaluate.py index 589a447..2e97d10 100644 --- a/evaluate.py +++ b/evaluate.py @@ -4,6 +4,7 @@ import argparse, os, re, json import torch from datasets import load_dataset from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig +from peft import PeftConfig, PeftModel from tqdm import tqdm CTX_SIZE = 2048 @@ -23,8 +24,10 @@ def main(): parser.add_argument("model") parser.add_argument("--dataset_file", default="./data/home_assistant_test.jsonl") parser.add_argument("--batch-size", default=8) + parser.add_argument("--lora", default=False,action='store_const', const=True) args = parser.parse_args() + lora_folder = f"./loras/{args.model}" model_folder = f"./models/{args.model}" dataset = load_dataset("json", data_files={ "train": args.dataset_file })["train"] @@ -40,9 +43,31 @@ def main(): service_call_regex = re.compile(r"```homeassistant\n([\S \t\n]*?)```") torch.set_default_device("cuda") - print(f"Loading model from {model_folder}...") - trained_model = AutoModelForCausalLM.from_pretrained(model_folder, trust_remote_code=True, torch_dtype=torch.bfloat16) #, code_revision="834565c23f9b28b96ccbeabe614dd906b6db551a") - trained_tokenizer = AutoTokenizer.from_pretrained(model_folder, trust_remote_code=True, padding_side='left') + + if args.lora: + adapter_config = PeftConfig.from_pretrained(lora_folder) + base_model_name = adapter_config.base_model_name_or_path + print(f"Loading lora from {lora_folder} ({base_model_name})...") + + base_model = AutoModelForCausalLM.from_pretrained( + base_model_name, + trust_remote_code=True, + torch_dtype=torch.bfloat16, + ) + trained_tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True, padding_side='left') + + trained_model = PeftModel.from_pretrained(base_model, lora_folder, trust_remote_code=True, torch_dtype=torch.bfloat16) + + output_folder = lora_folder + else: + print(f"Loading model from {model_folder}...") + trained_model = AutoModelForCausalLM.from_pretrained( + model_folder, + trust_remote_code=True, + torch_dtype=torch.bfloat16, + ) + trained_tokenizer = AutoTokenizer.from_pretrained(model_folder, trust_remote_code=True, padding_side='left') + output_folder = model_folder trained_model.generation_config = GenerationConfig( max_new_tokens=128, @@ -140,7 +165,7 @@ def main(): print(f"Final Accuracy Rating: {accuracy*100:.2f}%") print(f"Color Mismatches: {color_mismatches}") - with open(os.path.join(model_folder, "eval_results.json"), "w") as f: + with open(os.path.join(output_folder, "eval_results.json"), "w") as f: json.dump({ "possible_answers": total_answers, "correct_answers": correct_answers, diff --git a/requirements.txt b/requirements.txt index 447ef7e..8a7e097 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ peft bitsandbytes webcolors pandas +# flash-attn diff --git a/train.py b/train.py index 7ff830d..451f665 100644 --- a/train.py +++ b/train.py @@ -67,15 +67,16 @@ python3 train.py \ """ python3 train.py \ - --run_name stablehome-3b-rev1 \ + --run_name stablehome-3b-rev3 \ --base_model stabilityai/stablelm-zephyr-3b \ --bf16 \ --train_dataset data/home_assistant_train.jsonl \ --test_dataset data/home_assistant_test.jsonl \ --learning_rate 1e-5 \ - --micro_batch_size 2 --gradient_checkpointing \ + --micro_batch_size 4 --gradient_checkpointing \ --ctx_size 2048 \ - --use_lora --lora_rank 32 --lora_alpha 64 --lora_modules up_proj,down_proj,q_proj,v_proj,o_proj --lora_modules_to_save embed_tokens,lm_head --lora_merge + --save_steps 400 --save_total_limit 20 \ + --use_lora --lora_rank 64 --lora_alpha 128 --lora_modules up_proj,down_proj,q_proj,v_proj,o_proj --lora_merge """ """ @@ -262,6 +263,7 @@ training_args = TrainingArguments( log_level="info", bf16=training_run_args.bf16, group_by_length=training_run_args.group_by_length, + skip_memory_metrics=True, **training_kwargs, # include_inputs_for_metrics=True, )