more fixes

TODO.md

@@ -23,4 +23,5 @@
 [ ] prime kv cache with current "state" so that requests are faster
 [ ] make a proper evaluation framework to run. not just loss. should test accuracy on the function calling
 [ ] add LocalAI backend
-[x] more config options for prompt template (allow other than chatml)
+[x] more config options for prompt template (allow other than chatml)
+[ ] publish snapshot of dataset on HF

convert_and_quantize.sh

@@ -12,6 +12,8 @@ fi
 echo "Converting to GGUF..."
 $LLAMA_CPP/convert-hf-to-gguf.py --outfile ./models/$MODEL_NAME/$MODEL_NAME.f16.gguf --outtype f16 ./models/$MODEL_NAME/
 
+exit -1
+
 DESIRED_QUANTS=("Q8_0" "Q5_K_M" "Q4_K_M" "Q3_K_M" "Q2_K")
 for QUANT in "${DESIRED_QUANTS[@]}"
 do

data/test_prompts/_sys_prompt.txt

@@ -1,4 +1,4 @@
-You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only.
+You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only.
 Services: cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, fan.decrease_speed, fan.increase_speed, fan.toggle, fan.turn_off, fan.turn_on, cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, light.toggle, light.turn_off, light.turn_on
 Devices:
 light.back_lounge_warm 'Back Lounge Warm Light' = off

data/test_prompts/christmas.txt

@@ -1,5 +1,5 @@
 <|im_start|>system
-You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only.
+You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only.
 Services: cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, fan.decrease_speed, fan.increase_speed, fan.toggle, fan.turn_off, fan.turn_on, cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, light.toggle, light.turn_off, light.turn_on
 Devices:
 light.back_lounge_warm 'Back Lounge Warm Light' = off

data/test_prompts/ha_demo.txt

@@ -1,5 +1,5 @@
 <|im_start|>system
-You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task ask instructed with the information provided only.
+You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only.
 Services: light.turn_on, light.turn_off, light.toggle, lock.unlock, lock.lock, lock.open, cover.open_cover, cover.close_cover, cover.set_cover_position, cover.stop_cover, cover.toggle, cover.open_cover_tilt, cover.close_cover_tilt, cover.stop_cover_tilt, cover.set_cover_tilt_position, cover.toggle_cover_tilt, fan.turn_on, fan.turn_off, fan.toggle, fan.increase_speed, fan.decrease_speed, fan.oscillate, fan.set_direction, fan.set_percentage, fan.set_preset_mode
 Devices:
 cover.kitchen_window 'Kitchen Window' = closed
@@ -14,7 +14,7 @@ lock.front_door 'Front Door' = locked
 lock.kitchen_door 'Kitchen Door' = unlocked
 lock.openable_lock 'Openable Lock' = locked
 light.bed_light 'Bed Light' = off
-light.ceiling_lights 'Ceiling Lights' = on
+light.ceiling_lights 'Ceiling Lights' = on;20%
 light.kitchen_lights 'Kitchen Lights' = on
 light.office_rgbw_lights 'Office RGBW Lights' = on
 light.living_room_rgbww_lights 'Living Room RGBWW Lights' = on

data/test_prompts/logic_puzzle.txt

@@ -1,5 +1,5 @@
 <|im_start|>system
-You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task ask instructed with the information provided only.
+You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only.
 Services: light.turn_on, light.turn_off, light.toggle, lock.unlock, lock.lock, lock.open, cover.open_cover, cover.close_cover, cover.set_cover_position, cover.stop_cover, cover.toggle, cover.open_cover_tilt, cover.close_cover_tilt, cover.stop_cover_tilt, cover.set_cover_tilt_position, cover.toggle_cover_tilt, fan.turn_on, fan.turn_off, fan.toggle, fan.increase_speed, fan.decrease_speed, fan.oscillate, fan.set_direction, fan.set_percentage, fan.set_preset_mode
 Devices:
 cover.kitchen_window 'Kitchen Window' = closed

data/test_prompts/sonnet.txt

@@ -1,5 +1,5 @@
 <|im_start|>system
-You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed with the information provided only.
+You are 'Al', a helpful AI Assistant that controls the devices in a house. Complete the following task as instructed or answer the following question with the information provided only.
 Services: cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, fan.decrease_speed, fan.increase_speed, fan.toggle, fan.turn_off, fan.turn_on, cover.close_cover, cover.open_cover, cover.stop_cover, cover.toggle, light.toggle, light.turn_off, light.turn_on
 Devices:
 light.back_lounge_warm 'Back Lounge Warm Light' = off

generate.sh

@@ -2,7 +2,7 @@
 
 LLAMA_CPP=../llama.cpp
 MODEL_NAME=$1
-PROMPT_SRC=${2:-./data/default_test_prompt.txt}
+PROMPT_SRC=${2:-./data/test_prompts/ha_demo.txt}
 QUANT_TYPE=${3:-f16}
 
 if [[ ! -d "./models/$MODEL_NAME" ]]; then
@@ -11,4 +11,4 @@ if [[ ! -d "./models/$MODEL_NAME" ]]; then
 fi
 
 PROMPT=$(cat $PROMPT_SRC)
-$LLAMA_CPP/build/bin/main --model "./models/$MODEL_NAME/$MODEL_NAME.$QUANT_TYPE.gguf" --temp 0.1 --ctx-size 2048 --prompt "$PROMPT" # --chatml
+$LLAMA_CPP/build/bin/main --model "./models/$MODEL_NAME/$MODEL_NAME.$QUANT_TYPE.gguf" --temp 0.1 --ctx-size 2048 --prompt "$PROMPT" --grammar-file ./custom_components/llama_conversation/output.gbnf

train.py

@@ -24,7 +24,7 @@ python3 train.py \
     --test_dataset data/home_assistant_alpaca_merged_test.json \
     --learning_rate 1e-5 \
     --save_steps 1000 \
-    --micro_batch_size 2 --gradient_checkpointing \
+    --micro_batch_size 4 --gradient_checkpointing \
     --ctx_size 2048 \
     --use_lora --lora_rank 32 --lora_alpha 64 --lora_modules fc1,fc2,Wqkv,out_proj --lora_modules_to_save wte,lm_head.linear --lora_merge
 """
@@ -147,6 +147,9 @@ if training_run_args.add_chatml_tokens:
         'eos_token': '<|im_end|>'
     })
 
+    model.config.bos_token_id = tokenizer.bos_token_id
+    model.config.eos_token_id = tokenizer.eos_token_id
+
 embeddings_len = math.ceil(len(tokenizer) / 32) * 32
 if model.get_input_embeddings().num_embeddings < embeddings_len:
     model.resize_token_embeddings(embeddings_len)
@@ -173,7 +176,7 @@ if training_run_args.use_lora:
         )
     model = get_peft_model(model, peft_config)
     model.enable_input_require_grads()
-    
+
     model.print_trainable_parameters()