diff --git a/tests/backend/patches/lora_conversions/lora_state_dicts/flux_dora_onetrainer_format.py b/tests/backend/patches/lora_conversions/lora_state_dicts/flux_dora_onetrainer_format.py
new file mode 100644
index 0000000000..10157636de
--- /dev/null
+++ b/tests/backend/patches/lora_conversions/lora_state_dicts/flux_dora_onetrainer_format.py
@@ -0,0 +1,2029 @@
+# A sample state dict in the OneTrainer FLUX DoRA format.
+# This state dict is based on the ball_flux.safetensors file from here:
+# https://github.com/invoke-ai/InvokeAI/issues/6912
+state_dict_keys = {
+    "lora_te1_text_model_encoder_layers_0_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_0_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_0_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_0_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_0_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_0_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_0_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_0_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_10_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_10_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_10_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_10_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_10_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_10_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_10_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_11_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_11_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_11_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_11_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_11_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_11_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_11_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_1_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_1_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_1_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_1_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_1_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_1_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_1_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_2_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_2_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_2_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_2_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_2_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_2_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_2_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_3_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_3_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_3_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_3_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_3_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_3_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_3_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_4_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_4_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_4_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_4_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_4_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_4_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_4_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_5_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_5_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_5_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_5_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_5_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_5_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_5_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_6_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_6_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_6_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_6_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_6_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_6_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_6_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_7_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_7_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_7_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_7_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_7_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_7_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_7_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_8_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_8_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_8_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_8_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_8_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_8_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_8_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc1.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc1.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc1.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc1.lora_up.weight": [3072, 4],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc2.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc2.dora_scale": [1, 3072],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc2.lora_down.weight": [4, 3072],
+    "lora_te1_text_model_encoder_layers_9_mlp_fc2.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_9_self_attn_k_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_self_attn_k_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_k_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_k_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_9_self_attn_out_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_self_attn_out_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_out_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_out_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_9_self_attn_q_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_self_attn_q_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_q_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_q_proj.lora_up.weight": [768, 4],
+    "lora_te1_text_model_encoder_layers_9_self_attn_v_proj.alpha": [],
+    "lora_te1_text_model_encoder_layers_9_self_attn_v_proj.dora_scale": [1, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_v_proj.lora_down.weight": [4, 768],
+    "lora_te1_text_model_encoder_layers_9_self_attn_v_proj.lora_up.weight": [768, 4],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_0_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_10_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_11_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_12_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_13_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_14_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_15_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_16_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_17_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_18_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_19_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_1_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_20_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_21_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_22_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_23_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_2_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_3_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_4_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_5_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_6_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_7_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_8_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_k.alpha": [],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_k.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_k.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_k.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_o.alpha": [],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_o.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_o.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_o.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_q.alpha": [],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_q.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_q.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_q.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_v.alpha": [],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_v.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_v.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_0_SelfAttention_v.lora_up.weight": [4096, 4],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_0.alpha": [],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_0.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_0.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_0.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_1.alpha": [],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_1.dora_scale": [1, 4096],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_1.lora_down.weight": [4, 4096],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wi_1.lora_up.weight": [10240, 4],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wo.alpha": [],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wo.dora_scale": [1, 10240],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wo.lora_down.weight": [4, 10240],
+    "lora_te2_encoder_block_9_layer_1_DenseReluDense_wo.lora_up.weight": [4096, 4],
+    "lora_transformer_single_transformer_blocks_0_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_0_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_0_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_0_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_0_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_0_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_0_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_10_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_10_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_10_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_10_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_10_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_10_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_10_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_11_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_11_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_11_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_11_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_11_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_11_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_11_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_12_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_12_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_12_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_12_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_12_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_12_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_12_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_13_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_13_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_13_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_13_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_13_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_13_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_13_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_14_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_14_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_14_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_14_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_14_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_14_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_14_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_15_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_15_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_15_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_15_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_15_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_15_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_15_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_16_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_16_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_16_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_16_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_16_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_16_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_16_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_17_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_17_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_17_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_17_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_17_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_17_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_17_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_18_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_18_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_18_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_18_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_18_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_18_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_18_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_19_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_19_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_19_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_19_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_19_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_19_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_19_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_1_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_1_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_1_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_1_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_1_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_1_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_1_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_20_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_20_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_20_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_20_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_20_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_20_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_20_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_21_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_21_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_21_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_21_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_21_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_21_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_21_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_22_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_22_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_22_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_22_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_22_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_22_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_22_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_23_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_23_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_23_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_23_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_23_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_23_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_23_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_24_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_24_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_24_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_24_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_24_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_24_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_24_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_25_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_25_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_25_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_25_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_25_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_25_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_25_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_26_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_26_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_26_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_26_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_26_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_26_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_26_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_27_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_27_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_27_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_27_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_27_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_27_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_27_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_28_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_28_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_28_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_28_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_28_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_28_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_28_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_29_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_29_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_29_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_29_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_29_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_29_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_29_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_2_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_2_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_2_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_2_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_2_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_2_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_2_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_30_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_30_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_30_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_30_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_30_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_30_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_30_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_31_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_31_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_31_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_31_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_31_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_31_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_31_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_32_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_32_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_32_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_32_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_32_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_32_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_32_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_33_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_33_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_33_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_33_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_33_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_33_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_33_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_34_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_34_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_34_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_34_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_34_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_34_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_34_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_35_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_35_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_35_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_35_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_35_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_35_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_35_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_36_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_36_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_36_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_36_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_36_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_36_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_36_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_37_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_37_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_37_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_37_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_37_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_37_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_37_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_3_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_3_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_3_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_3_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_3_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_3_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_3_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_4_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_4_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_4_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_4_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_4_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_4_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_4_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_5_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_5_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_5_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_5_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_5_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_5_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_5_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_6_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_6_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_6_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_6_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_6_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_6_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_6_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_7_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_7_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_7_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_7_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_7_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_7_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_7_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_8_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_8_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_8_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_8_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_8_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_8_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_8_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_9_attn_to_k.alpha": [],
+    "lora_transformer_single_transformer_blocks_9_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_9_attn_to_q.alpha": [],
+    "lora_transformer_single_transformer_blocks_9_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_single_transformer_blocks_9_attn_to_v.alpha": [],
+    "lora_transformer_single_transformer_blocks_9_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_single_transformer_blocks_9_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_0_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_0_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_0_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_10_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_10_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_10_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_11_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_11_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_11_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_12_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_12_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_12_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_13_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_13_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_13_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_14_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_14_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_14_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_15_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_15_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_15_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_16_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_16_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_16_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_17_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_17_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_17_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_18_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_18_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_18_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_1_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_1_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_1_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_2_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_2_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_2_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_3_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_3_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_3_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_4_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_4_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_4_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_5_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_5_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_5_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_6_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_6_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_6_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_7_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_7_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_7_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_8_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_8_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_8_attn_to_v.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_add_k_proj.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_add_k_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_k_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_k_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_add_q_proj.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_add_q_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_q_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_q_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_add_v_proj.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_add_v_proj.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_v_proj.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_add_v_proj.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_to_add_out.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_to_add_out.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_add_out.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_add_out.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_to_k.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_to_k.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_k.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_k.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_to_out_0.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_to_out_0.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_out_0.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_out_0.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_to_q.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_to_q.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_q.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_q.lora_up.weight": [3072, 4],
+    "lora_transformer_transformer_blocks_9_attn_to_v.alpha": [],
+    "lora_transformer_transformer_blocks_9_attn_to_v.dora_scale": [1, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_v.lora_down.weight": [4, 3072],
+    "lora_transformer_transformer_blocks_9_attn_to_v.lora_up.weight": [3072, 4],
+}