chore: 4.2.9 version bump

Currently translated at 97.6% (1342 of 1374 strings)

Co-authored-by: Phrixus2023 <920414016@qq.com>
Translate-URL: https://hosted.weblate.org/projects/invokeai/web-ui/zh_Hans/
Translation: InvokeAI/Web UI

.github/workflows/build-container.yml

@@ -13,6 +13,12 @@ on:
     tags:
       - 'v*.*.*'
   workflow_dispatch:
+    inputs:
+      push-to-registry:
+        description: Push the built image to the container registry
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: write
@@ -50,16 +56,15 @@ jobs:
           df -h
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           images: |
             ghcr.io/${{ github.repository }}
-            ${{ env.DOCKERHUB_REPOSITORY }}
           tags: |
             type=ref,event=branch
             type=ref,event=tag
@@ -72,49 +77,33 @@ jobs:
             suffix=-${{ matrix.gpu-driver }},onlatest=false
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           platforms: ${{ env.PLATFORMS }}
 
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # - name: Login to Docker Hub
-      #   if: github.event_name != 'pull_request' && vars.DOCKERHUB_REPOSITORY != ''
-      #   uses: docker/login-action@v2
-      #   with:
-      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Build container
         timeout-minutes: 40
         id: docker_build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/Dockerfile
           platforms: ${{ env.PLATFORMS }}
-          push: ${{ github.ref == 'refs/heads/main' || github.ref_type == 'tag' }}
+          push: ${{ github.ref == 'refs/heads/main' || github.ref_type == 'tag' || github.event.inputs.push-to-registry }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: |
             type=gha,scope=${{ github.ref_name }}-${{ matrix.gpu-driver }}
             type=gha,scope=main-${{ matrix.gpu-driver }}
           cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ matrix.gpu-driver }}
-
-      # - name: Docker Hub Description
-      #   if: github.ref == 'refs/heads/main' || github.ref == 'refs/tags/*' && vars.DOCKERHUB_REPOSITORY != ''
-      #   uses: peter-evans/dockerhub-description@v3
-      #   with:
-      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
-      #     repository: ${{ vars.DOCKERHUB_REPOSITORY }}
-      #     short-description: ${{ github.event.repository.description }}

invokeai/app/invocations/compel.py

@@ -19,8 +19,8 @@ from invokeai.app.invocations.model import CLIPField
 from invokeai.app.invocations.primitives import ConditioningOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.util.ti_utils import generate_ti_list
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_patcher import ModelPatcher
-from invokeai.backend.peft.lora import LoRAModelRaw
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
     BasicConditioningInfo,
     ConditioningFieldData,

invokeai/app/invocations/denoise_latents.py

@@ -36,9 +36,9 @@ from invokeai.app.invocations.t2i_adapter import T2IAdapterField
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.util.controlnet_utils import prepare_control_image
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager import BaseModelType, ModelVariantType
 from invokeai.backend.model_patcher import ModelPatcher
-from invokeai.backend.peft.lora import LoRAModelRaw
 from invokeai.backend.stable_diffusion import PipelineIntermediateState
 from invokeai.backend.stable_diffusion.denoise_context import DenoiseContext, DenoiseInputs
 from invokeai.backend.stable_diffusion.diffusers_pipeline import (

invokeai/app/invocations/flux_denoise.py

@@ -1,4 +1,4 @@
-from typing import Callable, Iterator, Optional, Tuple
+from typing import Callable, Optional
 
 import torch
 import torchvision.transforms as tv_transforms
@@ -17,7 +17,6 @@ from invokeai.app.invocations.fields import (
 )
 from invokeai.app.invocations.model import TransformerField
 from invokeai.app.invocations.primitives import LatentsOutput
-from invokeai.app.services.session_processor.session_processor_common import CanceledException
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.flux.denoise import denoise
 from invokeai.backend.flux.inpaint_extension import InpaintExtension
@@ -30,8 +29,7 @@ from invokeai.backend.flux.sampling_utils import (
     pack,
     unpack,
 )
-from invokeai.backend.peft.lora import LoRAModelRaw
-from invokeai.backend.peft.peft_patcher import PeftPatcher
+from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import FLUXConditioningInfo
 from invokeai.backend.util.devices import TorchDevice
 
@@ -189,16 +187,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
                 noise=noise,
             )
 
-        with (
-            transformer_info.model_on_device() as (cached_weights, transformer),
-            # Apply the LoRA after transformer has been moved to its target device for faster patching.
-            PeftPatcher.apply_peft_patches(
-                model=transformer,
-                patches=self._lora_iterator(context),
-                prefix="",
-                cached_weights=cached_weights,
-            ),
-        ):
+        with transformer_info as transformer:
             assert isinstance(transformer, Flux)
 
             x = denoise(
@@ -252,41 +241,9 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         # `latents`.
         return mask.expand_as(latents)
 
-    def _lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[LoRAModelRaw, float]]:
-        for lora in self.transformer.loras:
-            lora_info = context.models.load(lora.lora)
-            assert isinstance(lora_info.model, LoRAModelRaw)
-            yield (lora_info.model, lora.weight)
-            del lora_info
-
-    def _build_step_callback(self, context: InvocationContext) -> Callable[[], None]:
-        def step_callback() -> None:
-            if context.util.is_canceled():
-                raise CanceledException
-
-            # TODO: Make this look like the image before re-enabling
-            # latent_image = unpack(img.float(), self.height, self.width)
-            # latent_image = latent_image.squeeze()  # Remove unnecessary dimensions
-            # flattened_tensor = latent_image.reshape(-1)  # Flatten to shape [48*128*128]
-
-            # # Create a new tensor of the required shape [255, 255, 3]
-            # latent_image = flattened_tensor[: 255 * 255 * 3].reshape(255, 255, 3)  # Reshape to RGB format
-
-            # # Convert to a NumPy array and then to a PIL Image
-            # image = Image.fromarray(latent_image.cpu().numpy().astype(np.uint8))
-
-            # (width, height) = image.size
-            # width *= 8
-            # height *= 8
-
-            # dataURL = image_to_dataURL(image, image_format="JPEG")
-
-            # # TODO: move this whole function to invocation context to properly reference these variables
-            # context._services.events.emit_invocation_denoise_progress(
-            #     context._data.queue_item,
-            #     context._data.invocation,
-            #     state,
-            #     ProgressImage(dataURL=dataURL, width=width, height=height),
-            # )
+    def _build_step_callback(self, context: InvocationContext) -> Callable[[PipelineIntermediateState], None]:
+        def step_callback(state: PipelineIntermediateState) -> None:
+            state.latents = unpack(state.latents.float(), self.height, self.width).squeeze()
+            context.util.flux_step_callback(state)
 
         return step_callback

invokeai/app/invocations/flux_lora_loader.py

@@ -1,53 +0,0 @@
-from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output
-from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField, UIType
-from invokeai.app.invocations.model import LoRAField, ModelIdentifierField, TransformerField
-from invokeai.app.services.shared.invocation_context import InvocationContext
-
-
-@invocation_output("flux_lora_loader_output")
-class FluxLoRALoaderOutput(BaseInvocationOutput):
-    """FLUX LoRA Loader Output"""
-
-    transformer: TransformerField = OutputField(
-        default=None, description=FieldDescriptions.transformer, title="FLUX Transformer"
-    )
-
-
-@invocation(
-    "flux_lora_loader",
-    title="FLUX LoRA",
-    tags=["lora", "model", "flux"],
-    category="model",
-    version="1.0.0",
-)
-class FluxLoRALoaderInvocation(BaseInvocation):
-    """Apply a LoRA model to a FLUX transformer."""
-
-    lora: ModelIdentifierField = InputField(
-        description=FieldDescriptions.lora_model, title="LoRA", ui_type=UIType.LoRAModel
-    )
-    weight: float = InputField(default=0.75, description=FieldDescriptions.lora_weight)
-    transformer: TransformerField = InputField(
-        description=FieldDescriptions.transformer,
-        input=Input.Connection,
-        title="FLUX Transformer",
-    )
-
-    def invoke(self, context: InvocationContext) -> FluxLoRALoaderOutput:
-        lora_key = self.lora.key
-
-        if not context.models.exists(lora_key):
-            raise ValueError(f"Unknown lora: {lora_key}!")
-
-        if any(lora.lora.key == lora_key for lora in self.transformer.loras):
-            raise Exception(f'LoRA "{lora_key}" already applied to transformer.')
-
-        transformer = self.transformer.model_copy(deep=True)
-        transformer.loras.append(
-            LoRAField(
-                lora=self.lora,
-                weight=self.weight,
-            )
-        )
-
-        return FluxLoRALoaderOutput(transformer=transformer)

invokeai/app/invocations/model.py

@@ -69,7 +69,6 @@ class CLIPField(BaseModel):
 
 class TransformerField(BaseModel):
     transformer: ModelIdentifierField = Field(description="Info to load Transformer submodel")
-    loras: List[LoRAField] = Field(description="LoRAs to apply on model loading")
 
 
 class T5EncoderField(BaseModel):
@@ -203,7 +202,7 @@ class FluxModelLoaderInvocation(BaseInvocation):
         assert isinstance(transformer_config, CheckpointConfigBase)
 
         return FluxModelLoaderOutput(
-            transformer=TransformerField(transformer=transformer, loras=[]),
+            transformer=TransformerField(transformer=transformer),
             clip=CLIPField(tokenizer=tokenizer, text_encoder=clip_encoder, loras=[], skipped_layers=0),
             t5_encoder=T5EncoderField(tokenizer=tokenizer2, text_encoder=t5_encoder),
             vae=VAEField(vae=vae),

invokeai/app/invocations/tiled_multi_diffusion_denoise_latents.py

@@ -22,8 +22,8 @@ from invokeai.app.invocations.fields import (
 from invokeai.app.invocations.model import UNetField
 from invokeai.app.invocations.primitives import LatentsOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_patcher import ModelPatcher
-from invokeai.backend.peft.lora import LoRAModelRaw
 from invokeai.backend.stable_diffusion.diffusers_pipeline import ControlNetData, PipelineIntermediateState
 from invokeai.backend.stable_diffusion.multi_diffusion_pipeline import (
     MultiDiffusionPipeline,

invokeai/app/services/shared/invocation_context.py

@@ -14,7 +14,7 @@ from invokeai.app.services.image_records.image_records_common import ImageCatego
 from invokeai.app.services.images.images_common import ImageDTO
 from invokeai.app.services.invocation_services import InvocationServices
 from invokeai.app.services.model_records.model_records_base import UnknownModelException
-from invokeai.app.util.step_callback import stable_diffusion_step_callback
+from invokeai.app.util.step_callback import flux_step_callback, stable_diffusion_step_callback
 from invokeai.backend.model_manager.config import (
     AnyModel,
     AnyModelConfig,
@@ -557,6 +557,24 @@ class UtilInterface(InvocationContextInterface):
             is_canceled=self.is_canceled,
         )
 
+    def flux_step_callback(self, intermediate_state: PipelineIntermediateState) -> None:
+        """
+        The step callback emits a progress event with the current step, the total number of
+        steps, a preview image, and some other internal metadata.
+
+        This should be called after each denoising step.
+
+        Args:
+            intermediate_state: The intermediate state of the diffusion pipeline.
+        """
+
+        flux_step_callback(
+            context_data=self._data,
+            intermediate_state=intermediate_state,
+            events=self._services.events,
+            is_canceled=self.is_canceled,
+        )
+
 
 class InvocationContext:
     """Provides access to various services and data for the current invocation.

invokeai/app/util/step_callback.py

@@ -38,6 +38,25 @@ SD1_5_LATENT_RGB_FACTORS = [
     [-0.1307, -0.1874, -0.7445],  # L4
 ]
 
+FLUX_LATENT_RGB_FACTORS = [
+    [-0.0412, 0.0149, 0.0521],
+    [0.0056, 0.0291, 0.0768],
+    [0.0342, -0.0681, -0.0427],
+    [-0.0258, 0.0092, 0.0463],
+    [0.0863, 0.0784, 0.0547],
+    [-0.0017, 0.0402, 0.0158],
+    [0.0501, 0.1058, 0.1152],
+    [-0.0209, -0.0218, -0.0329],
+    [-0.0314, 0.0083, 0.0896],
+    [0.0851, 0.0665, -0.0472],
+    [-0.0534, 0.0238, -0.0024],
+    [0.0452, -0.0026, 0.0048],
+    [0.0892, 0.0831, 0.0881],
+    [-0.1117, -0.0304, -0.0789],
+    [0.0027, -0.0479, -0.0043],
+    [-0.1146, -0.0827, -0.0598],
+]
+
 
 def sample_to_lowres_estimated_image(
     samples: torch.Tensor, latent_rgb_factors: torch.Tensor, smooth_matrix: Optional[torch.Tensor] = None
@@ -94,3 +113,32 @@ def stable_diffusion_step_callback(
         intermediate_state,
         ProgressImage(dataURL=dataURL, width=width, height=height),
     )
+
+
+def flux_step_callback(
+    context_data: "InvocationContextData",
+    intermediate_state: PipelineIntermediateState,
+    events: "EventServiceBase",
+    is_canceled: Callable[[], bool],
+) -> None:
+    if is_canceled():
+        raise CanceledException
+    sample = intermediate_state.latents
+    latent_rgb_factors = torch.tensor(FLUX_LATENT_RGB_FACTORS, dtype=sample.dtype, device=sample.device)
+    latent_image_perm = sample.permute(1, 2, 0).to(dtype=sample.dtype, device=sample.device)
+    latent_image = latent_image_perm @ latent_rgb_factors
+    latents_ubyte = (
+        ((latent_image + 1) / 2).clamp(0, 1).mul(0xFF)  # change scale from -1..1 to 0..1  # to 0..255
+    ).to(device="cpu", dtype=torch.uint8)
+    image = Image.fromarray(latents_ubyte.cpu().numpy())
+    (width, height) = image.size
+    width *= 8
+    height *= 8
+    dataURL = image_to_dataURL(image, image_format="JPEG")
+
+    events.emit_invocation_denoise_progress(
+        context_data.queue_item,
+        context_data.invocation,
+        intermediate_state,
+        ProgressImage(dataURL=dataURL, width=width, height=height),
+    )

invokeai/backend/flux/denoise.py

@@ -5,6 +5,7 @@ from tqdm import tqdm
 
 from invokeai.backend.flux.inpaint_extension import InpaintExtension
 from invokeai.backend.flux.model import Flux
+from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState
 
 
 def denoise(
@@ -17,10 +18,11 @@ def denoise(
     vec: torch.Tensor,
     # sampling parameters
     timesteps: list[float],
-    step_callback: Callable[[], None],
+    step_callback: Callable[[PipelineIntermediateState], None],
     guidance: float,
     inpaint_extension: InpaintExtension | None,
 ):
+    step = 0
     # guidance_vec is ignored for schnell.
     guidance_vec = torch.full((img.shape[0],), guidance, device=img.device, dtype=img.dtype)
     for t_curr, t_prev in tqdm(list(zip(timesteps[:-1], timesteps[1:], strict=True))):
@@ -34,12 +36,21 @@ def denoise(
             timesteps=t_vec,
             guidance=guidance_vec,
         )
-
+        preview_img = img - t_curr * pred
         img = img + (t_prev - t_curr) * pred
 
         if inpaint_extension is not None:
             img = inpaint_extension.merge_intermediate_latents_with_init_latents(img, t_prev)
 
-        step_callback()
+        step_callback(
+            PipelineIntermediateState(
+                step=step,
+                order=1,
+                total_steps=len(timesteps),
+                timestep=int(t_curr),
+                latents=preview_img,
+            ),
+        )
+        step += 1
 
     return img

invokeai/backend/lora.py

@@ -0,0 +1,672 @@
+# Copyright (c) 2024 The InvokeAI Development team
+"""LoRA model support."""
+
+import bisect
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple, Union
+
+import torch
+from safetensors.torch import load_file
+from typing_extensions import Self
+
+import invokeai.backend.util.logging as logger
+from invokeai.backend.model_manager import BaseModelType
+from invokeai.backend.raw_model import RawModel
+
+
+class LoRALayerBase:
+    # rank: Optional[int]
+    # alpha: Optional[float]
+    # bias: Optional[torch.Tensor]
+    # layer_key: str
+
+    # @property
+    # def scale(self):
+    #    return self.alpha / self.rank if (self.alpha and self.rank) else 1.0
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        if "alpha" in values:
+            self.alpha = values["alpha"].item()
+        else:
+            self.alpha = None
+
+        if "bias_indices" in values and "bias_values" in values and "bias_size" in values:
+            self.bias: Optional[torch.Tensor] = torch.sparse_coo_tensor(
+                values["bias_indices"],
+                values["bias_values"],
+                tuple(values["bias_size"]),
+            )
+
+        else:
+            self.bias = None
+
+        self.rank = None  # set in layer implementation
+        self.layer_key = layer_key
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        raise NotImplementedError()
+
+    def get_bias(self, orig_bias: torch.Tensor) -> Optional[torch.Tensor]:
+        return self.bias
+
+    def get_parameters(self, orig_module: torch.nn.Module) -> Dict[str, torch.Tensor]:
+        params = {"weight": self.get_weight(orig_module.weight)}
+        bias = self.get_bias(orig_module.bias)
+        if bias is not None:
+            params["bias"] = bias
+        return params
+
+    def calc_size(self) -> int:
+        model_size = 0
+        for val in [self.bias]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        if self.bias is not None:
+            self.bias = self.bias.to(device=device, dtype=dtype)
+
+    def check_keys(self, values: Dict[str, torch.Tensor], known_keys: Set[str]):
+        """Log a warning if values contains unhandled keys."""
+        # {"alpha", "bias_indices", "bias_values", "bias_size"} are hard-coded, because they are handled by
+        # `LoRALayerBase`. Sub-classes should provide the known_keys that they handled.
+        all_known_keys = known_keys | {"alpha", "bias_indices", "bias_values", "bias_size"}
+        unknown_keys = set(values.keys()) - all_known_keys
+        if unknown_keys:
+            logger.warning(
+                f"Unexpected keys found in LoRA/LyCORIS layer, model might work incorrectly! Keys: {unknown_keys}"
+            )
+
+
+# TODO: find and debug lora/locon with bias
+class LoRALayer(LoRALayerBase):
+    # up: torch.Tensor
+    # mid: Optional[torch.Tensor]
+    # down: torch.Tensor
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        super().__init__(layer_key, values)
+
+        self.up = values["lora_up.weight"]
+        self.down = values["lora_down.weight"]
+        self.mid = values.get("lora_mid.weight", None)
+
+        self.rank = self.down.shape[0]
+        self.check_keys(
+            values,
+            {
+                "lora_up.weight",
+                "lora_down.weight",
+                "lora_mid.weight",
+            },
+        )
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        if self.mid is not None:
+            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
+            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
+            weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
+        else:
+            weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.up, self.mid, self.down]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        super().to(device=device, dtype=dtype)
+
+        self.up = self.up.to(device=device, dtype=dtype)
+        self.down = self.down.to(device=device, dtype=dtype)
+
+        if self.mid is not None:
+            self.mid = self.mid.to(device=device, dtype=dtype)
+
+
+class LoHALayer(LoRALayerBase):
+    # w1_a: torch.Tensor
+    # w1_b: torch.Tensor
+    # w2_a: torch.Tensor
+    # w2_b: torch.Tensor
+    # t1: Optional[torch.Tensor] = None
+    # t2: Optional[torch.Tensor] = None
+
+    def __init__(self, layer_key: str, values: Dict[str, torch.Tensor]):
+        super().__init__(layer_key, values)
+
+        self.w1_a = values["hada_w1_a"]
+        self.w1_b = values["hada_w1_b"]
+        self.w2_a = values["hada_w2_a"]
+        self.w2_b = values["hada_w2_b"]
+        self.t1 = values.get("hada_t1", None)
+        self.t2 = values.get("hada_t2", None)
+
+        self.rank = self.w1_b.shape[0]
+        self.check_keys(
+            values,
+            {
+                "hada_w1_a",
+                "hada_w1_b",
+                "hada_w2_a",
+                "hada_w2_b",
+                "hada_t1",
+                "hada_t2",
+            },
+        )
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        if self.t1 is None:
+            weight: torch.Tensor = (self.w1_a @ self.w1_b) * (self.w2_a @ self.w2_b)
+
+        else:
+            rebuild1 = torch.einsum("i j k l, j r, i p -> p r k l", self.t1, self.w1_b, self.w1_a)
+            rebuild2 = torch.einsum("i j k l, j r, i p -> p r k l", self.t2, self.w2_b, self.w2_a)
+            weight = rebuild1 * rebuild2
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.w1_a, self.w1_b, self.w2_a, self.w2_b, self.t1, self.t2]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        super().to(device=device, dtype=dtype)
+
+        self.w1_a = self.w1_a.to(device=device, dtype=dtype)
+        self.w1_b = self.w1_b.to(device=device, dtype=dtype)
+        if self.t1 is not None:
+            self.t1 = self.t1.to(device=device, dtype=dtype)
+
+        self.w2_a = self.w2_a.to(device=device, dtype=dtype)
+        self.w2_b = self.w2_b.to(device=device, dtype=dtype)
+        if self.t2 is not None:
+            self.t2 = self.t2.to(device=device, dtype=dtype)
+
+
+class LoKRLayer(LoRALayerBase):
+    # w1: Optional[torch.Tensor] = None
+    # w1_a: Optional[torch.Tensor] = None
+    # w1_b: Optional[torch.Tensor] = None
+    # w2: Optional[torch.Tensor] = None
+    # w2_a: Optional[torch.Tensor] = None
+    # w2_b: Optional[torch.Tensor] = None
+    # t2: Optional[torch.Tensor] = None
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        super().__init__(layer_key, values)
+
+        self.w1 = values.get("lokr_w1", None)
+        if self.w1 is None:
+            self.w1_a = values["lokr_w1_a"]
+            self.w1_b = values["lokr_w1_b"]
+        else:
+            self.w1_b = None
+            self.w1_a = None
+
+        self.w2 = values.get("lokr_w2", None)
+        if self.w2 is None:
+            self.w2_a = values["lokr_w2_a"]
+            self.w2_b = values["lokr_w2_b"]
+        else:
+            self.w2_a = None
+            self.w2_b = None
+
+        self.t2 = values.get("lokr_t2", None)
+
+        if self.w1_b is not None:
+            self.rank = self.w1_b.shape[0]
+        elif self.w2_b is not None:
+            self.rank = self.w2_b.shape[0]
+        else:
+            self.rank = None  # unscaled
+
+        self.check_keys(
+            values,
+            {
+                "lokr_w1",
+                "lokr_w1_a",
+                "lokr_w1_b",
+                "lokr_w2",
+                "lokr_w2_a",
+                "lokr_w2_b",
+                "lokr_t2",
+            },
+        )
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        w1: Optional[torch.Tensor] = self.w1
+        if w1 is None:
+            assert self.w1_a is not None
+            assert self.w1_b is not None
+            w1 = self.w1_a @ self.w1_b
+
+        w2 = self.w2
+        if w2 is None:
+            if self.t2 is None:
+                assert self.w2_a is not None
+                assert self.w2_b is not None
+                w2 = self.w2_a @ self.w2_b
+            else:
+                w2 = torch.einsum("i j k l, i p, j r -> p r k l", self.t2, self.w2_a, self.w2_b)
+
+        if len(w2.shape) == 4:
+            w1 = w1.unsqueeze(2).unsqueeze(2)
+        w2 = w2.contiguous()
+        assert w1 is not None
+        assert w2 is not None
+        weight = torch.kron(w1, w2)
+
+        return weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        for val in [self.w1, self.w1_a, self.w1_b, self.w2, self.w2_a, self.w2_b, self.t2]:
+            if val is not None:
+                model_size += val.nelement() * val.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        super().to(device=device, dtype=dtype)
+
+        if self.w1 is not None:
+            self.w1 = self.w1.to(device=device, dtype=dtype)
+        else:
+            assert self.w1_a is not None
+            assert self.w1_b is not None
+            self.w1_a = self.w1_a.to(device=device, dtype=dtype)
+            self.w1_b = self.w1_b.to(device=device, dtype=dtype)
+
+        if self.w2 is not None:
+            self.w2 = self.w2.to(device=device, dtype=dtype)
+        else:
+            assert self.w2_a is not None
+            assert self.w2_b is not None
+            self.w2_a = self.w2_a.to(device=device, dtype=dtype)
+            self.w2_b = self.w2_b.to(device=device, dtype=dtype)
+
+        if self.t2 is not None:
+            self.t2 = self.t2.to(device=device, dtype=dtype)
+
+
+class FullLayer(LoRALayerBase):
+    # bias handled in LoRALayerBase(calc_size, to)
+    # weight: torch.Tensor
+    # bias: Optional[torch.Tensor]
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        super().__init__(layer_key, values)
+
+        self.weight = values["diff"]
+        self.bias = values.get("diff_b", None)
+
+        self.rank = None  # unscaled
+        self.check_keys(values, {"diff", "diff_b"})
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        return self.weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        model_size += self.weight.nelement() * self.weight.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        super().to(device=device, dtype=dtype)
+
+        self.weight = self.weight.to(device=device, dtype=dtype)
+
+
+class IA3Layer(LoRALayerBase):
+    # weight: torch.Tensor
+    # on_input: torch.Tensor
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        super().__init__(layer_key, values)
+
+        self.weight = values["weight"]
+        self.on_input = values["on_input"]
+
+        self.rank = None  # unscaled
+        self.check_keys(values, {"weight", "on_input"})
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        weight = self.weight
+        if not self.on_input:
+            weight = weight.reshape(-1, 1)
+        assert orig_weight is not None
+        return orig_weight * weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        model_size += self.weight.nelement() * self.weight.element_size()
+        model_size += self.on_input.nelement() * self.on_input.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None):
+        super().to(device=device, dtype=dtype)
+
+        self.weight = self.weight.to(device=device, dtype=dtype)
+        self.on_input = self.on_input.to(device=device, dtype=dtype)
+
+
+class NormLayer(LoRALayerBase):
+    # bias handled in LoRALayerBase(calc_size, to)
+    # weight: torch.Tensor
+    # bias: Optional[torch.Tensor]
+
+    def __init__(
+        self,
+        layer_key: str,
+        values: Dict[str, torch.Tensor],
+    ):
+        super().__init__(layer_key, values)
+
+        self.weight = values["w_norm"]
+        self.bias = values.get("b_norm", None)
+
+        self.rank = None  # unscaled
+        self.check_keys(values, {"w_norm", "b_norm"})
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
+        return self.weight
+
+    def calc_size(self) -> int:
+        model_size = super().calc_size()
+        model_size += self.weight.nelement() * self.weight.element_size()
+        return model_size
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        super().to(device=device, dtype=dtype)
+
+        self.weight = self.weight.to(device=device, dtype=dtype)
+
+
+AnyLoRALayer = Union[LoRALayer, LoHALayer, LoKRLayer, FullLayer, IA3Layer, NormLayer]
+
+
+class LoRAModelRaw(RawModel):  # (torch.nn.Module):
+    _name: str
+    layers: Dict[str, AnyLoRALayer]
+
+    def __init__(
+        self,
+        name: str,
+        layers: Dict[str, AnyLoRALayer],
+    ):
+        self._name = name
+        self.layers = layers
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
+        # TODO: try revert if exception?
+        for _key, layer in self.layers.items():
+            layer.to(device=device, dtype=dtype)
+
+    def calc_size(self) -> int:
+        model_size = 0
+        for _, layer in self.layers.items():
+            model_size += layer.calc_size()
+        return model_size
+
+    @classmethod
+    def _convert_sdxl_keys_to_diffusers_format(cls, state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
+        """Convert the keys of an SDXL LoRA state_dict to diffusers format.
+
+        The input state_dict can be in either Stability AI format or diffusers format. If the state_dict is already in
+        diffusers format, then this function will have no effect.
+
+        This function is adapted from:
+        https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L385-L409
+
+        Args:
+            state_dict (Dict[str, Tensor]): The SDXL LoRA state_dict.
+
+        Raises:
+            ValueError: If state_dict contains an unrecognized key, or not all keys could be converted.
+
+        Returns:
+            Dict[str, Tensor]: The diffusers-format state_dict.
+        """
+        converted_count = 0  # The number of Stability AI keys converted to diffusers format.
+        not_converted_count = 0  # The number of keys that were not converted.
+
+        # Get a sorted list of Stability AI UNet keys so that we can efficiently search for keys with matching prefixes.
+        # For example, we want to efficiently find `input_blocks_4_1` in the list when searching for
+        # `input_blocks_4_1_proj_in`.
+        stability_unet_keys = list(SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP)
+        stability_unet_keys.sort()
+
+        new_state_dict = {}
+        for full_key, value in state_dict.items():
+            if full_key.startswith("lora_unet_"):
+                search_key = full_key.replace("lora_unet_", "")
+                # Use bisect to find the key in stability_unet_keys that *may* match the search_key's prefix.
+                position = bisect.bisect_right(stability_unet_keys, search_key)
+                map_key = stability_unet_keys[position - 1]
+                # Now, check if the map_key *actually* matches the search_key.
+                if search_key.startswith(map_key):
+                    new_key = full_key.replace(map_key, SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP[map_key])
+                    new_state_dict[new_key] = value
+                    converted_count += 1
+                else:
+                    new_state_dict[full_key] = value
+                    not_converted_count += 1
+            elif full_key.startswith("lora_te1_") or full_key.startswith("lora_te2_"):
+                # The CLIP text encoders have the same keys in both Stability AI and diffusers formats.
+                new_state_dict[full_key] = value
+                continue
+            else:
+                raise ValueError(f"Unrecognized SDXL LoRA key prefix: '{full_key}'.")
+
+        if converted_count > 0 and not_converted_count > 0:
+            raise ValueError(
+                f"The SDXL LoRA could only be partially converted to diffusers format. converted={converted_count},"
+                f" not_converted={not_converted_count}"
+            )
+
+        return new_state_dict
+
+    @classmethod
+    def from_checkpoint(
+        cls,
+        file_path: Union[str, Path],
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+        base_model: Optional[BaseModelType] = None,
+    ) -> Self:
+        device = device or torch.device("cpu")
+        dtype = dtype or torch.float32
+
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        model = cls(
+            name=file_path.stem,
+            layers={},
+        )
+
+        if file_path.suffix == ".safetensors":
+            sd = load_file(file_path.absolute().as_posix(), device="cpu")
+        else:
+            sd = torch.load(file_path, map_location="cpu")
+
+        state_dict = cls._group_state(sd)
+
+        if base_model == BaseModelType.StableDiffusionXL:
+            state_dict = cls._convert_sdxl_keys_to_diffusers_format(state_dict)
+
+        for layer_key, values in state_dict.items():
+            # Detect layers according to LyCORIS detection logic(`weight_list_det`)
+            # https://github.com/KohakuBlueleaf/LyCORIS/tree/8ad8000efb79e2b879054da8c9356e6143591bad/lycoris/modules
+
+            # lora and locon
+            if "lora_up.weight" in values:
+                layer: AnyLoRALayer = LoRALayer(layer_key, values)
+
+            # loha
+            elif "hada_w1_a" in values:
+                layer = LoHALayer(layer_key, values)
+
+            # lokr
+            elif "lokr_w1" in values or "lokr_w1_a" in values:
+                layer = LoKRLayer(layer_key, values)
+
+            # diff
+            elif "diff" in values:
+                layer = FullLayer(layer_key, values)
+
+            # ia3
+            elif "on_input" in values:
+                layer = IA3Layer(layer_key, values)
+
+            # norms
+            elif "w_norm" in values:
+                layer = NormLayer(layer_key, values)
+
+            else:
+                print(f">> Encountered unknown lora layer module in {model.name}: {layer_key} - {list(values.keys())}")
+                raise Exception("Unknown lora format!")
+
+            # lower memory consumption by removing already parsed layer values
+            state_dict[layer_key].clear()
+
+            layer.to(device=device, dtype=dtype)
+            model.layers[layer_key] = layer
+
+        return model
+
+    @staticmethod
+    def _group_state(state_dict: Dict[str, torch.Tensor]) -> Dict[str, Dict[str, torch.Tensor]]:
+        state_dict_groupped: Dict[str, Dict[str, torch.Tensor]] = {}
+
+        for key, value in state_dict.items():
+            stem, leaf = key.split(".", 1)
+            if stem not in state_dict_groupped:
+                state_dict_groupped[stem] = {}
+            state_dict_groupped[stem][leaf] = value
+
+        return state_dict_groupped
+
+
+# code from
+# https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L15C1-L97C32
+def make_sdxl_unet_conversion_map() -> List[Tuple[str, str]]:
+    """Create a dict mapping state_dict keys from Stability AI SDXL format to diffusers SDXL format."""
+    unet_conversion_map_layer = []
+
+    for i in range(3):  # num_blocks is 3 in sdxl
+        # loop over downblocks/upblocks
+        for j in range(2):
+            # loop over resnets/attentions for downblocks
+            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
+            sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0."
+            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
+
+            if i < 3:
+                # no attention layers in down_blocks.3
+                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
+                sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1."
+                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
+
+        for j in range(3):
+            # loop over resnets/attentions for upblocks
+            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
+            sd_up_res_prefix = f"output_blocks.{3*i + j}.0."
+            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
+
+            # if i > 0: commentout for sdxl
+            # no attention layers in up_blocks.0
+            hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
+            sd_up_atn_prefix = f"output_blocks.{3*i + j}.1."
+            unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
+
+        if i < 3:
+            # no downsample in down_blocks.3
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
+            sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op."
+            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
+
+            # no upsample in up_blocks.3
+            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
+            sd_upsample_prefix = f"output_blocks.{3*i + 2}.{2}."  # change for sdxl
+            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
+
+    hf_mid_atn_prefix = "mid_block.attentions.0."
+    sd_mid_atn_prefix = "middle_block.1."
+    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
+
+    for j in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{j}."
+        sd_mid_res_prefix = f"middle_block.{2*j}."
+        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    unet_conversion_map_resnet = [
+        # (stable-diffusion, HF Diffusers)
+        ("in_layers.0.", "norm1."),
+        ("in_layers.2.", "conv1."),
+        ("out_layers.0.", "norm2."),
+        ("out_layers.3.", "conv2."),
+        ("emb_layers.1.", "time_emb_proj."),
+        ("skip_connection.", "conv_shortcut."),
+    ]
+
+    unet_conversion_map = []
+    for sd, hf in unet_conversion_map_layer:
+        if "resnets" in hf:
+            for sd_res, hf_res in unet_conversion_map_resnet:
+                unet_conversion_map.append((sd + sd_res, hf + hf_res))
+        else:
+            unet_conversion_map.append((sd, hf))
+
+    for j in range(2):
+        hf_time_embed_prefix = f"time_embedding.linear_{j+1}."
+        sd_time_embed_prefix = f"time_embed.{j*2}."
+        unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix))
+
+    for j in range(2):
+        hf_label_embed_prefix = f"add_embedding.linear_{j+1}."
+        sd_label_embed_prefix = f"label_emb.0.{j*2}."
+        unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix))
+
+    unet_conversion_map.append(("input_blocks.0.0.", "conv_in."))
+    unet_conversion_map.append(("out.0.", "conv_norm_out."))
+    unet_conversion_map.append(("out.2.", "conv_out."))
+
+    return unet_conversion_map
+
+
+SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP = {
+    sd.rstrip(".").replace(".", "_"): hf.rstrip(".").replace(".", "_") for sd, hf in make_sdxl_unet_conversion_map()
+}

invokeai/backend/model_manager/load/model_loaders/flux.py

@@ -32,6 +32,9 @@ from invokeai.backend.model_manager.config import (
 )
 from invokeai.backend.model_manager.load.load_default import ModelLoader
 from invokeai.backend.model_manager.load.model_loader_registry import ModelLoaderRegistry
+from invokeai.backend.model_manager.util.model_util import (
+    convert_bundle_to_flux_transformer_checkpoint,
+)
 from invokeai.backend.util.silence_warnings import SilenceWarnings
 
 try:
@@ -190,6 +193,13 @@ class FluxCheckpointModel(ModelLoader):
         with SilenceWarnings():
             model = Flux(params[config.config_path])
             sd = load_file(model_path)
+            if "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in sd:
+                sd = convert_bundle_to_flux_transformer_checkpoint(sd)
+            new_sd_size = sum([ten.nelement() * torch.bfloat16.itemsize for ten in sd.values()])
+            self._ram_cache.make_room(new_sd_size)
+            for k in sd.keys():
+                # We need to cast to bfloat16 due to it being the only currently supported dtype for inference
+                sd[k] = sd[k].to(torch.bfloat16)
             model.load_state_dict(sd, assign=True)
         return model
 
@@ -230,5 +240,7 @@ class FluxBnbQuantizednf4bCheckpointModel(ModelLoader):
                 model = Flux(params[config.config_path])
                 model = quantize_model_nf4(model, modules_to_not_convert=set(), compute_dtype=torch.bfloat16)
             sd = load_file(model_path)
+            if "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in sd:
+                sd = convert_bundle_to_flux_transformer_checkpoint(sd)
             model.load_state_dict(sd, assign=True)
         return model

invokeai/backend/model_manager/load/model_loaders/lora.py

@@ -5,10 +5,8 @@ from logging import Logger
 from pathlib import Path
 from typing import Optional
 
-import torch
-from safetensors.torch import load_file
-
 from invokeai.app.services.config import InvokeAIAppConfig
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager import (
     AnyModel,
     AnyModelConfig,
@@ -20,11 +18,6 @@ from invokeai.backend.model_manager import (
 from invokeai.backend.model_manager.load.load_default import ModelLoader
 from invokeai.backend.model_manager.load.model_cache.model_cache_base import ModelCacheBase
 from invokeai.backend.model_manager.load.model_loader_registry import ModelLoaderRegistry
-from invokeai.backend.peft.conversions.flux_kohya_lora_conversion_utils import (
-    lora_model_from_flux_kohya_state_dict,
-)
-from invokeai.backend.peft.conversions.sd_lora_conversion_utils import lora_model_from_sd_state_dict
-from invokeai.backend.peft.conversions.sdxl_lora_conversion_utils import convert_sdxl_keys_to_diffusers_format
 
 
 @ModelLoaderRegistry.register(base=BaseModelType.Any, type=ModelType.LoRA, format=ModelFormat.Diffusers)
@@ -52,28 +45,14 @@ class LoRALoader(ModelLoader):
             raise ValueError("There are no submodels in a LoRA model.")
         model_path = Path(config.path)
         assert self._model_base is not None
-
-        # Load the state dict from the model file.
-        if model_path.suffix == ".safetensors":
-            state_dict = load_file(model_path.absolute().as_posix(), device="cpu")
-        else:
-            state_dict = torch.load(model_path, map_location="cpu")
-
-        # Apply state_dict key conversions, if necessary.
-        if self._model_base == BaseModelType.StableDiffusionXL:
-            state_dict = convert_sdxl_keys_to_diffusers_format(state_dict)
-            model = lora_model_from_sd_state_dict(state_dict=state_dict)
-        elif self._model_base == BaseModelType.Flux:
-            model = lora_model_from_flux_kohya_state_dict(state_dict=state_dict)
-        elif self._model_base in [BaseModelType.StableDiffusion1, BaseModelType.StableDiffusion2]:
-            # Currently, we don't apply any conversions for SD1 and SD2 LoRA models.
-            model = lora_model_from_sd_state_dict(state_dict=state_dict)
-        else:
-            raise ValueError(f"Unsupported LoRA base model: {self._model_base}")
-
-        model.to(dtype=self._torch_dtype)
+        model = LoRAModelRaw.from_checkpoint(
+            file_path=model_path,
+            dtype=self._torch_dtype,
+            base_model=self._model_base,
+        )
         return model
 
+    # override
     def _get_model_path(self, config: AnyModelConfig) -> Path:
         # cheating a little - we remember this variable for using in the subsequent call to _load_model()
         self._model_base = config.base

invokeai/backend/model_manager/load/model_util.py

@@ -15,9 +15,9 @@ from invokeai.backend.image_util.depth_anything.depth_anything_pipeline import D
 from invokeai.backend.image_util.grounding_dino.grounding_dino_pipeline import GroundingDinoPipeline
 from invokeai.backend.image_util.segment_anything.segment_anything_pipeline import SegmentAnythingPipeline
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager.config import AnyModel
 from invokeai.backend.onnx.onnx_runtime import IAIOnnxRuntimeModel
-from invokeai.backend.peft.lora import LoRAModelRaw
 from invokeai.backend.spandrel_image_to_image_model import SpandrelImageToImageModel
 from invokeai.backend.textual_inversion import TextualInversionModelRaw
 

invokeai/backend/model_manager/probe.py

@@ -26,7 +26,6 @@ from invokeai.backend.model_manager.config import (
     SchedulerPredictionType,
 )
 from invokeai.backend.model_manager.util.model_util import lora_token_vector_length, read_checkpoint_meta
-from invokeai.backend.peft.conversions.flux_kohya_lora_conversion_utils import is_state_dict_likely_in_flux_kohya_format
 from invokeai.backend.spandrel_image_to_image_model import SpandrelImageToImageModel
 from invokeai.backend.util.silence_warnings import SilenceWarnings
 
@@ -109,6 +108,8 @@ class ModelProbe(object):
         "CLIPVisionModelWithProjection": ModelType.CLIPVision,
         "T2IAdapter": ModelType.T2IAdapter,
         "CLIPModel": ModelType.CLIPEmbed,
+        "CLIPTextModel": ModelType.CLIPEmbed,
+        "T5EncoderModel": ModelType.T5Encoder,
     }
 
     @classmethod
@@ -225,7 +226,18 @@ class ModelProbe(object):
         ckpt = ckpt.get("state_dict", ckpt)
 
         for key in [str(k) for k in ckpt.keys()]:
-            if key.startswith(("cond_stage_model.", "first_stage_model.", "model.diffusion_model.", "double_blocks.")):
+            if key.startswith(
+                (
+                    "cond_stage_model.",
+                    "first_stage_model.",
+                    "model.diffusion_model.",
+                    # FLUX models in the official BFL format contain keys with the "double_blocks." prefix.
+                    "double_blocks.",
+                    # Some FLUX checkpoint files contain transformer keys prefixed with "model.diffusion_model".
+                    # This prefix is typically used to distinguish between multiple models bundled in a single file.
+                    "model.diffusion_model.double_blocks.",
+                )
+            ):
                 # Keys starting with double_blocks are associated with Flux models
                 return ModelType.Main
             elif key.startswith(("encoder.conv_in", "decoder.conv_in")):
@@ -284,9 +296,16 @@ class ModelProbe(object):
         if (folder_path / "image_encoder.txt").exists():
             return ModelType.IPAdapter
 
-        i = folder_path / "model_index.json"
-        c = folder_path / "config.json"
-        config_path = i if i.exists() else c if c.exists() else None
+        config_path = None
+        for p in [
+            folder_path / "model_index.json",  # pipeline
+            folder_path / "config.json",  # most diffusers
+            folder_path / "text_encoder_2" / "config.json",  # T5 text encoder
+            folder_path / "text_encoder" / "config.json",  # T5 CLIP
+        ]:
+            if p.exists():
+                config_path = p
+                break
 
         if config_path:
             with open(config_path, "r") as file:
@@ -329,7 +348,10 @@ class ModelProbe(object):
                 # TODO: Decide between dev/schnell
                 checkpoint = ModelProbe._scan_and_load_checkpoint(model_path)
                 state_dict = checkpoint.get("state_dict") or checkpoint
-                if "guidance_in.out_layer.weight" in state_dict:
+                if (
+                    "guidance_in.out_layer.weight" in state_dict
+                    or "model.diffusion_model.guidance_in.out_layer.weight" in state_dict
+                ):
                     # For flux, this is a key in invokeai.backend.flux.util.params
                     #   Due to model type and format being the descriminator for model configs this
                     #   is used rather than attempting to support flux with separate model types and format
@@ -337,7 +359,7 @@ class ModelProbe(object):
                     config_file = "flux-dev"
                 else:
                     # For flux, this is a key in invokeai.backend.flux.util.params
-                    #   Due to model type and format being the descriminator for model configs this
+                    #   Due to model type and format being the discriminator for model configs this
                     #   is used rather than attempting to support flux with separate model types and format
                     #   If changed in the future, please fix me
                     config_file = "flux-schnell"
@@ -444,7 +466,10 @@ class CheckpointProbeBase(ProbeBase):
 
     def get_format(self) -> ModelFormat:
         state_dict = self.checkpoint.get("state_dict") or self.checkpoint
-        if "double_blocks.0.img_attn.proj.weight.quant_state.bitsandbytes__nf4" in state_dict:
+        if (
+            "double_blocks.0.img_attn.proj.weight.quant_state.bitsandbytes__nf4" in state_dict
+            or "model.diffusion_model.double_blocks.0.img_attn.proj.weight.quant_state.bitsandbytes__nf4" in state_dict
+        ):
             return ModelFormat.BnbQuantizednf4b
         return ModelFormat("checkpoint")
 
@@ -471,7 +496,10 @@ class PipelineCheckpointProbe(CheckpointProbeBase):
     def get_base_type(self) -> BaseModelType:
         checkpoint = self.checkpoint
         state_dict = self.checkpoint.get("state_dict") or checkpoint
-        if "double_blocks.0.img_attn.norm.key_norm.scale" in state_dict:
+        if (
+            "double_blocks.0.img_attn.norm.key_norm.scale" in state_dict
+            or "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in state_dict
+        ):
             return BaseModelType.Flux
         key_name = "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weight"
         if key_name in state_dict and state_dict[key_name].shape[-1] == 768:
@@ -529,11 +557,9 @@ class LoRACheckpointProbe(CheckpointProbeBase):
         return ModelFormat("lycoris")
 
     def get_base_type(self) -> BaseModelType:
-        if is_state_dict_likely_in_flux_kohya_format(self.checkpoint):
-            return BaseModelType.Flux
+        checkpoint = self.checkpoint
+        token_vector_length = lora_token_vector_length(checkpoint)
 
-        # If we've gotten here, we assume that the model is a Stable Diffusion model.
-        token_vector_length = lora_token_vector_length(self.checkpoint)
         if token_vector_length == 768:
             return BaseModelType.StableDiffusion1
         elif token_vector_length == 1024:
@@ -750,8 +776,27 @@ class TextualInversionFolderProbe(FolderProbeBase):
 
 
 class T5EncoderFolderProbe(FolderProbeBase):
+    def get_base_type(self) -> BaseModelType:
+        return BaseModelType.Any
+
     def get_format(self) -> ModelFormat:
-        return ModelFormat.T5Encoder
+        path = self.model_path / "text_encoder_2"
+        if (path / "model.safetensors.index.json").exists():
+            return ModelFormat.T5Encoder
+        files = list(path.glob("*.safetensors"))
+        if len(files) == 0:
+            raise InvalidModelConfigException(f"{self.model_path.as_posix()}: no .safetensors files found")
+
+        # shortcut: look for the quantization in the name
+        if any(x for x in files if "llm_int8" in x.as_posix()):
+            return ModelFormat.BnbQuantizedLlmInt8b
+
+        # more reliable path: probe contents for a 'SCB' key
+        ckpt = read_checkpoint_meta(files[0], scan=True)
+        if any("SCB" in x for x in ckpt.keys()):
+            return ModelFormat.BnbQuantizedLlmInt8b
+
+        raise InvalidModelConfigException(f"{self.model_path.as_posix()}: unknown model format")
 
 
 class ONNXFolderProbe(PipelineFolderProbe):

invokeai/backend/model_manager/util/model_util.py

@@ -133,3 +133,29 @@ def lora_token_vector_length(checkpoint: Dict[str, torch.Tensor]) -> Optional[in
             break
 
     return lora_token_vector_length
+
+
+def convert_bundle_to_flux_transformer_checkpoint(
+    transformer_state_dict: dict[str, torch.Tensor],
+) -> dict[str, torch.Tensor]:
+    original_state_dict: dict[str, torch.Tensor] = {}
+    keys_to_remove: list[str] = []
+
+    for k, v in transformer_state_dict.items():
+        if not k.startswith("model.diffusion_model"):
+            keys_to_remove.append(k)  # This can be removed in the future if we only want to delete transformer keys
+            continue
+        if k.endswith("scale"):
+            # Scale math must be done at bfloat16 due to our current flux model
+            # support limitations at inference time
+            v = v.to(dtype=torch.bfloat16)
+        new_key = k.replace("model.diffusion_model.", "")
+        original_state_dict[new_key] = v
+        keys_to_remove.append(k)
+
+    # Remove processed keys from the original dictionary, leaving others in case
+    # other model state dicts need to be pulled
+    for k in keys_to_remove:
+        del transformer_state_dict[k]
+
+    return original_state_dict

invokeai/backend/model_patcher.py

@@ -13,10 +13,10 @@ from diffusers import OnnxRuntimeModel, UNet2DConditionModel
 from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
 
 from invokeai.app.shared.models import FreeUConfig
+from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager import AnyModel
 from invokeai.backend.model_manager.load.optimizations import skip_torch_weight_init
 from invokeai.backend.onnx.onnx_runtime import IAIOnnxRuntimeModel
-from invokeai.backend.peft.lora import LoRAModelRaw
 from invokeai.backend.stable_diffusion.extensions.lora import LoRAExt
 from invokeai.backend.textual_inversion import TextualInversionManager, TextualInversionModelRaw
 from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage

invokeai/backend/peft/conversions/flux_kohya_lora_conversion_utils.py

@@ -1,84 +0,0 @@
-import re
-from typing import Any, Dict, TypeVar
-
-import torch
-
-from invokeai.backend.peft.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.peft.layers.utils import peft_layer_from_state_dict
-from invokeai.backend.peft.lora import LoRAModelRaw
-
-# A regex pattern that matches all of the keys in the Kohya FLUX LoRA format.
-# Example keys:
-#   lora_unet_double_blocks_0_img_attn_proj.alpha
-#   lora_unet_double_blocks_0_img_attn_proj.lora_down.weight
-#   lora_unet_double_blocks_0_img_attn_proj.lora_up.weight
-FLUX_KOHYA_KEY_REGEX = (
-    r"lora_unet_(\w+_blocks)_(\d+)_(img_attn|img_mlp|img_mod|txt_attn|txt_mlp|txt_mod|linear1|linear2|modulation)_?(.*)"
-)
-
-
-def is_state_dict_likely_in_flux_kohya_format(state_dict: Dict[str, Any]) -> bool:
-    """Checks if the provided state dict is likely in the Kohya FLUX LoRA format.
-
-    This is intended to be a high-precision detector, but it is not guaranteed to have perfect precision. (A
-    perfect-precision detector would require checking all keys against a whitelist and verifying tensor shapes.)
-    """
-    for k in state_dict.keys():
-        if not re.match(FLUX_KOHYA_KEY_REGEX, k):
-            return False
-    return True
-
-
-def lora_model_from_flux_kohya_state_dict(state_dict: Dict[str, torch.Tensor]) -> LoRAModelRaw:
-    # Group keys by layer.
-    grouped_state_dict: dict[str, dict[str, torch.Tensor]] = {}
-    for key, value in state_dict.items():
-        layer_name, param_name = key.split(".", 1)
-        if layer_name not in grouped_state_dict:
-            grouped_state_dict[layer_name] = {}
-        grouped_state_dict[layer_name][param_name] = value
-
-    # Convert the state dict to the InvokeAI format.
-    grouped_state_dict = convert_flux_kohya_state_dict_to_invoke_format(grouped_state_dict)
-
-    # Create LoRA layers.
-    layers: dict[str, AnyLoRALayer] = {}
-    for layer_key, layer_state_dict in grouped_state_dict.items():
-        layer = peft_layer_from_state_dict(layer_key, layer_state_dict)
-        layers[layer_key] = layer
-
-    # Create and return the LoRAModelRaw.
-    return LoRAModelRaw(layers=layers)
-
-
-T = TypeVar("T")
-
-
-def convert_flux_kohya_state_dict_to_invoke_format(state_dict: Dict[str, T]) -> Dict[str, T]:
-    """Converts a state dict from the Kohya FLUX LoRA format to LoRA weight format used internally by InvokeAI.
-
-    Example key conversions:
-    "lora_unet_double_blocks_0_img_attn_proj" -> "double_blocks.0.img_attn.proj"
-    "lora_unet_double_blocks_0_img_attn_proj" -> "double_blocks.0.img_attn.proj"
-    "lora_unet_double_blocks_0_img_attn_proj" -> "double_blocks.0.img_attn.proj"
-    "lora_unet_double_blocks_0_img_attn_qkv" -> "double_blocks.0.img_attn.qkv"
-    "lora_unet_double_blocks_0_img_attn_qkv" -> "double_blocks.0.img.attn.qkv"
-    "lora_unet_double_blocks_0_img_attn_qkv" -> "double_blocks.0.img.attn.qkv"
-    """
-
-    def replace_func(match: re.Match[str]) -> str:
-        s = f"{match.group(1)}.{match.group(2)}.{match.group(3)}"
-        if match.group(4):
-            s += f".{match.group(4)}"
-        return s
-
-    converted_dict: dict[str, T] = {}
-    for k, v in state_dict.items():
-        match = re.match(FLUX_KOHYA_KEY_REGEX, k)
-        if match:
-            new_key = re.sub(FLUX_KOHYA_KEY_REGEX, replace_func, k)
-            converted_dict[new_key] = v
-        else:
-            raise ValueError(f"Key '{k}' does not match the expected pattern for FLUX LoRA weights.")
-
-    return converted_dict

invokeai/backend/peft/conversions/sd_lora_conversion_utils.py

@@ -1,30 +0,0 @@
-from typing import Dict
-
-import torch
-
-from invokeai.backend.peft.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.peft.layers.utils import peft_layer_from_state_dict
-from invokeai.backend.peft.lora import LoRAModelRaw
-
-
-def lora_model_from_sd_state_dict(state_dict: Dict[str, torch.Tensor]) -> LoRAModelRaw:
-    grouped_state_dict: dict[str, dict[str, torch.Tensor]] = _group_state(state_dict)
-
-    layers: dict[str, AnyLoRALayer] = {}
-    for layer_key, values in grouped_state_dict.items():
-        layer = peft_layer_from_state_dict(layer_key, values)
-        layers[layer_key] = layer
-
-    return LoRAModelRaw(layers=layers)
-
-
-def _group_state(state_dict: Dict[str, torch.Tensor]) -> Dict[str, Dict[str, torch.Tensor]]:
-    state_dict_groupped: Dict[str, Dict[str, torch.Tensor]] = {}
-
-    for key, value in state_dict.items():
-        stem, leaf = key.split(".", 1)
-        if stem not in state_dict_groupped:
-            state_dict_groupped[stem] = {}
-        state_dict_groupped[stem][leaf] = value
-
-    return state_dict_groupped

invokeai/backend/peft/conversions/sdxl_lora_conversion_utils.py

@@ -1,154 +0,0 @@
-import bisect
-from typing import Dict, List, Tuple, TypeVar
-
-T = TypeVar("T")
-
-
-def convert_sdxl_keys_to_diffusers_format(state_dict: Dict[str, T]) -> dict[str, T]:
-    """Convert the keys of an SDXL LoRA state_dict to diffusers format.
-
-    The input state_dict can be in either Stability AI format or diffusers format. If the state_dict is already in
-    diffusers format, then this function will have no effect.
-
-    This function is adapted from:
-    https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L385-L409
-
-    Args:
-        state_dict (Dict[str, Tensor]): The SDXL LoRA state_dict.
-
-    Raises:
-        ValueError: If state_dict contains an unrecognized key, or not all keys could be converted.
-
-    Returns:
-        Dict[str, Tensor]: The diffusers-format state_dict.
-    """
-    converted_count = 0  # The number of Stability AI keys converted to diffusers format.
-    not_converted_count = 0  # The number of keys that were not converted.
-
-    # Get a sorted list of Stability AI UNet keys so that we can efficiently search for keys with matching prefixes.
-    # For example, we want to efficiently find `input_blocks_4_1` in the list when searching for
-    # `input_blocks_4_1_proj_in`.
-    stability_unet_keys = list(SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP)
-    stability_unet_keys.sort()
-
-    new_state_dict: dict[str, T] = {}
-    for full_key, value in state_dict.items():
-        if full_key.startswith("lora_unet_"):
-            search_key = full_key.replace("lora_unet_", "")
-            # Use bisect to find the key in stability_unet_keys that *may* match the search_key's prefix.
-            position = bisect.bisect_right(stability_unet_keys, search_key)
-            map_key = stability_unet_keys[position - 1]
-            # Now, check if the map_key *actually* matches the search_key.
-            if search_key.startswith(map_key):
-                new_key = full_key.replace(map_key, SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP[map_key])
-                new_state_dict[new_key] = value
-                converted_count += 1
-            else:
-                new_state_dict[full_key] = value
-                not_converted_count += 1
-        elif full_key.startswith("lora_te1_") or full_key.startswith("lora_te2_"):
-            # The CLIP text encoders have the same keys in both Stability AI and diffusers formats.
-            new_state_dict[full_key] = value
-            continue
-        else:
-            raise ValueError(f"Unrecognized SDXL LoRA key prefix: '{full_key}'.")
-
-    if converted_count > 0 and not_converted_count > 0:
-        raise ValueError(
-            f"The SDXL LoRA could only be partially converted to diffusers format. converted={converted_count},"
-            f" not_converted={not_converted_count}"
-        )
-
-    return new_state_dict
-
-
-# code from
-# https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L15C1-L97C32
-def _make_sdxl_unet_conversion_map() -> List[Tuple[str, str]]:
-    """Create a dict mapping state_dict keys from Stability AI SDXL format to diffusers SDXL format."""
-    unet_conversion_map_layer: list[tuple[str, str]] = []
-
-    for i in range(3):  # num_blocks is 3 in sdxl
-        # loop over downblocks/upblocks
-        for j in range(2):
-            # loop over resnets/attentions for downblocks
-            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
-            sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0."
-            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
-
-            if i < 3:
-                # no attention layers in down_blocks.3
-                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
-                sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1."
-                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
-
-        for j in range(3):
-            # loop over resnets/attentions for upblocks
-            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
-            sd_up_res_prefix = f"output_blocks.{3*i + j}.0."
-            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
-
-            # if i > 0: commentout for sdxl
-            # no attention layers in up_blocks.0
-            hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
-            sd_up_atn_prefix = f"output_blocks.{3*i + j}.1."
-            unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
-
-        if i < 3:
-            # no downsample in down_blocks.3
-            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
-            sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op."
-            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
-
-            # no upsample in up_blocks.3
-            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
-            sd_upsample_prefix = f"output_blocks.{3*i + 2}.{2}."  # change for sdxl
-            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
-
-    hf_mid_atn_prefix = "mid_block.attentions.0."
-    sd_mid_atn_prefix = "middle_block.1."
-    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
-
-    for j in range(2):
-        hf_mid_res_prefix = f"mid_block.resnets.{j}."
-        sd_mid_res_prefix = f"middle_block.{2*j}."
-        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
-
-    unet_conversion_map_resnet = [
-        # (stable-diffusion, HF Diffusers)
-        ("in_layers.0.", "norm1."),
-        ("in_layers.2.", "conv1."),
-        ("out_layers.0.", "norm2."),
-        ("out_layers.3.", "conv2."),
-        ("emb_layers.1.", "time_emb_proj."),
-        ("skip_connection.", "conv_shortcut."),
-    ]
-
-    unet_conversion_map: list[tuple[str, str]] = []
-    for sd, hf in unet_conversion_map_layer:
-        if "resnets" in hf:
-            for sd_res, hf_res in unet_conversion_map_resnet:
-                unet_conversion_map.append((sd + sd_res, hf + hf_res))
-        else:
-            unet_conversion_map.append((sd, hf))
-
-    for j in range(2):
-        hf_time_embed_prefix = f"time_embedding.linear_{j+1}."
-        sd_time_embed_prefix = f"time_embed.{j*2}."
-        unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix))
-
-    for j in range(2):
-        hf_label_embed_prefix = f"add_embedding.linear_{j+1}."
-        sd_label_embed_prefix = f"label_emb.0.{j*2}."
-        unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix))
-
-    unet_conversion_map.append(("input_blocks.0.0.", "conv_in."))
-    unet_conversion_map.append(("out.0.", "conv_norm_out."))
-    unet_conversion_map.append(("out.2.", "conv_out."))
-
-    return unet_conversion_map
-
-
-SDXL_UNET_STABILITY_TO_DIFFUSERS_MAP = {
-    sd.rstrip(".").replace(".", "_"): hf.rstrip(".").replace(".", "_") for sd, hf in _make_sdxl_unet_conversion_map()
-}

invokeai/backend/peft/layers/any_lora_layer.py

@@ -1,10 +0,0 @@
-from typing import Union
-
-from invokeai.backend.peft.layers.full_layer import FullLayer
-from invokeai.backend.peft.layers.ia3_layer import IA3Layer
-from invokeai.backend.peft.layers.loha_layer import LoHALayer
-from invokeai.backend.peft.layers.lokr_layer import LoKRLayer
-from invokeai.backend.peft.layers.lora_layer import LoRALayer
-from invokeai.backend.peft.layers.norm_layer import NormLayer
-
-AnyLoRALayer = Union[LoRALayer, LoHALayer, LoKRLayer, FullLayer, IA3Layer, NormLayer]

invokeai/backend/peft/layers/full_layer.py

@@ -1,37 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-class FullLayer(LoRALayerBase):
-    # bias handled in LoRALayerBase(calc_size, to)
-    # weight: torch.Tensor
-    # bias: Optional[torch.Tensor]
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        super().__init__(layer_key, values)
-
-        self.weight = values["diff"]
-        self.bias = values.get("diff_b", None)
-
-        self.rank = None  # unscaled
-        self.check_keys(values, {"diff", "diff_b"})
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        return self.weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        model_size += self.weight.nelement() * self.weight.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        super().to(device=device, dtype=dtype)
-
-        self.weight = self.weight.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/ia3_layer.py

@@ -1,42 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-class IA3Layer(LoRALayerBase):
-    # weight: torch.Tensor
-    # on_input: torch.Tensor
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        super().__init__(layer_key, values)
-
-        self.weight = values["weight"]
-        self.on_input = values["on_input"]
-
-        self.rank = None  # unscaled
-        self.check_keys(values, {"weight", "on_input"})
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        weight = self.weight
-        if not self.on_input:
-            weight = weight.reshape(-1, 1)
-        assert orig_weight is not None
-        return orig_weight * weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        model_size += self.weight.nelement() * self.weight.element_size()
-        model_size += self.on_input.nelement() * self.on_input.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None):
-        super().to(device=device, dtype=dtype)
-
-        self.weight = self.weight.to(device=device, dtype=dtype)
-        self.on_input = self.on_input.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/loha_layer.py

@@ -1,68 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-class LoHALayer(LoRALayerBase):
-    # w1_a: torch.Tensor
-    # w1_b: torch.Tensor
-    # w2_a: torch.Tensor
-    # w2_b: torch.Tensor
-    # t1: Optional[torch.Tensor] = None
-    # t2: Optional[torch.Tensor] = None
-
-    def __init__(self, layer_key: str, values: Dict[str, torch.Tensor]):
-        super().__init__(layer_key, values)
-
-        self.w1_a = values["hada_w1_a"]
-        self.w1_b = values["hada_w1_b"]
-        self.w2_a = values["hada_w2_a"]
-        self.w2_b = values["hada_w2_b"]
-        self.t1 = values.get("hada_t1", None)
-        self.t2 = values.get("hada_t2", None)
-
-        self.rank = self.w1_b.shape[0]
-        self.check_keys(
-            values,
-            {
-                "hada_w1_a",
-                "hada_w1_b",
-                "hada_w2_a",
-                "hada_w2_b",
-                "hada_t1",
-                "hada_t2",
-            },
-        )
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        if self.t1 is None:
-            weight: torch.Tensor = (self.w1_a @ self.w1_b) * (self.w2_a @ self.w2_b)
-
-        else:
-            rebuild1 = torch.einsum("i j k l, j r, i p -> p r k l", self.t1, self.w1_b, self.w1_a)
-            rebuild2 = torch.einsum("i j k l, j r, i p -> p r k l", self.t2, self.w2_b, self.w2_a)
-            weight = rebuild1 * rebuild2
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.w1_a, self.w1_b, self.w2_a, self.w2_b, self.t1, self.t2]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        super().to(device=device, dtype=dtype)
-
-        self.w1_a = self.w1_a.to(device=device, dtype=dtype)
-        self.w1_b = self.w1_b.to(device=device, dtype=dtype)
-        if self.t1 is not None:
-            self.t1 = self.t1.to(device=device, dtype=dtype)
-
-        self.w2_a = self.w2_a.to(device=device, dtype=dtype)
-        self.w2_b = self.w2_b.to(device=device, dtype=dtype)
-        if self.t2 is not None:
-            self.t2 = self.t2.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/lokr_layer.py

@@ -1,114 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-class LoKRLayer(LoRALayerBase):
-    # w1: Optional[torch.Tensor] = None
-    # w1_a: Optional[torch.Tensor] = None
-    # w1_b: Optional[torch.Tensor] = None
-    # w2: Optional[torch.Tensor] = None
-    # w2_a: Optional[torch.Tensor] = None
-    # w2_b: Optional[torch.Tensor] = None
-    # t2: Optional[torch.Tensor] = None
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        super().__init__(layer_key, values)
-
-        self.w1 = values.get("lokr_w1", None)
-        if self.w1 is None:
-            self.w1_a = values["lokr_w1_a"]
-            self.w1_b = values["lokr_w1_b"]
-        else:
-            self.w1_b = None
-            self.w1_a = None
-
-        self.w2 = values.get("lokr_w2", None)
-        if self.w2 is None:
-            self.w2_a = values["lokr_w2_a"]
-            self.w2_b = values["lokr_w2_b"]
-        else:
-            self.w2_a = None
-            self.w2_b = None
-
-        self.t2 = values.get("lokr_t2", None)
-
-        if self.w1_b is not None:
-            self.rank = self.w1_b.shape[0]
-        elif self.w2_b is not None:
-            self.rank = self.w2_b.shape[0]
-        else:
-            self.rank = None  # unscaled
-
-        self.check_keys(
-            values,
-            {
-                "lokr_w1",
-                "lokr_w1_a",
-                "lokr_w1_b",
-                "lokr_w2",
-                "lokr_w2_a",
-                "lokr_w2_b",
-                "lokr_t2",
-            },
-        )
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        w1: Optional[torch.Tensor] = self.w1
-        if w1 is None:
-            assert self.w1_a is not None
-            assert self.w1_b is not None
-            w1 = self.w1_a @ self.w1_b
-
-        w2 = self.w2
-        if w2 is None:
-            if self.t2 is None:
-                assert self.w2_a is not None
-                assert self.w2_b is not None
-                w2 = self.w2_a @ self.w2_b
-            else:
-                w2 = torch.einsum("i j k l, i p, j r -> p r k l", self.t2, self.w2_a, self.w2_b)
-
-        if len(w2.shape) == 4:
-            w1 = w1.unsqueeze(2).unsqueeze(2)
-        w2 = w2.contiguous()
-        assert w1 is not None
-        assert w2 is not None
-        weight = torch.kron(w1, w2)
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.w1, self.w1_a, self.w1_b, self.w2, self.w2_a, self.w2_b, self.t2]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        super().to(device=device, dtype=dtype)
-
-        if self.w1 is not None:
-            self.w1 = self.w1.to(device=device, dtype=dtype)
-        else:
-            assert self.w1_a is not None
-            assert self.w1_b is not None
-            self.w1_a = self.w1_a.to(device=device, dtype=dtype)
-            self.w1_b = self.w1_b.to(device=device, dtype=dtype)
-
-        if self.w2 is not None:
-            self.w2 = self.w2.to(device=device, dtype=dtype)
-        else:
-            assert self.w2_a is not None
-            assert self.w2_b is not None
-            self.w2_a = self.w2_a.to(device=device, dtype=dtype)
-            self.w2_b = self.w2_b.to(device=device, dtype=dtype)
-
-        if self.t2 is not None:
-            self.t2 = self.t2.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/lora_layer.py

@@ -1,59 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-# TODO: find and debug lora/locon with bias
-class LoRALayer(LoRALayerBase):
-    # up: torch.Tensor
-    # mid: Optional[torch.Tensor]
-    # down: torch.Tensor
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        super().__init__(layer_key, values)
-
-        self.up = values["lora_up.weight"]
-        self.down = values["lora_down.weight"]
-        self.mid = values.get("lora_mid.weight", None)
-
-        self.rank = self.down.shape[0]
-        self.check_keys(
-            values,
-            {
-                "lora_up.weight",
-                "lora_down.weight",
-                "lora_mid.weight",
-            },
-        )
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        if self.mid is not None:
-            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
-            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
-            weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
-        else:
-            weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
-
-        return weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        for val in [self.up, self.mid, self.down]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        super().to(device=device, dtype=dtype)
-
-        self.up = self.up.to(device=device, dtype=dtype)
-        self.down = self.down.to(device=device, dtype=dtype)
-
-        if self.mid is not None:
-            self.mid = self.mid.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/lora_layer_base.py

@@ -1,74 +0,0 @@
-from typing import Dict, Optional, Set
-
-import torch
-
-import invokeai.backend.util.logging as logger
-
-
-class LoRALayerBase:
-    # rank: Optional[int]
-    # alpha: Optional[float]
-    # bias: Optional[torch.Tensor]
-    # layer_key: str
-
-    # @property
-    # def scale(self):
-    #    return self.alpha / self.rank if (self.alpha and self.rank) else 1.0
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        if "alpha" in values:
-            self.alpha = values["alpha"].item()
-        else:
-            self.alpha = None
-
-        if "bias_indices" in values and "bias_values" in values and "bias_size" in values:
-            self.bias: Optional[torch.Tensor] = torch.sparse_coo_tensor(
-                values["bias_indices"],
-                values["bias_values"],
-                tuple(values["bias_size"]),
-            )
-
-        else:
-            self.bias = None
-
-        self.rank = None  # set in layer implementation
-        self.layer_key = layer_key
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        raise NotImplementedError()
-
-    def get_bias(self, orig_bias: torch.Tensor) -> Optional[torch.Tensor]:
-        return self.bias
-
-    def get_parameters(self, orig_module: torch.nn.Module) -> Dict[str, torch.Tensor]:
-        params = {"weight": self.get_weight(orig_module.weight)}
-        bias = self.get_bias(orig_module.bias)
-        if bias is not None:
-            params["bias"] = bias
-        return params
-
-    def calc_size(self) -> int:
-        model_size = 0
-        for val in [self.bias]:
-            if val is not None:
-                model_size += val.nelement() * val.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        if self.bias is not None:
-            self.bias = self.bias.to(device=device, dtype=dtype)
-
-    def check_keys(self, values: Dict[str, torch.Tensor], known_keys: Set[str]):
-        """Log a warning if values contains unhandled keys."""
-        # {"alpha", "bias_indices", "bias_values", "bias_size"} are hard-coded, because they are handled by
-        # `LoRALayerBase`. Sub-classes should provide the known_keys that they handled.
-        all_known_keys = known_keys | {"alpha", "bias_indices", "bias_values", "bias_size"}
-        unknown_keys = set(values.keys()) - all_known_keys
-        if unknown_keys:
-            logger.warning(
-                f"Unexpected keys found in LoRA/LyCORIS layer, model might work incorrectly! Keys: {unknown_keys}"
-            )

invokeai/backend/peft/layers/norm_layer.py

@@ -1,37 +0,0 @@
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.lora_layer_base import LoRALayerBase
-
-
-class NormLayer(LoRALayerBase):
-    # bias handled in LoRALayerBase(calc_size, to)
-    # weight: torch.Tensor
-    # bias: Optional[torch.Tensor]
-
-    def __init__(
-        self,
-        layer_key: str,
-        values: Dict[str, torch.Tensor],
-    ):
-        super().__init__(layer_key, values)
-
-        self.weight = values["w_norm"]
-        self.bias = values.get("b_norm", None)
-
-        self.rank = None  # unscaled
-        self.check_keys(values, {"w_norm", "b_norm"})
-
-    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
-        return self.weight
-
-    def calc_size(self) -> int:
-        model_size = super().calc_size()
-        model_size += self.weight.nelement() * self.weight.element_size()
-        return model_size
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        super().to(device=device, dtype=dtype)
-
-        self.weight = self.weight.to(device=device, dtype=dtype)

invokeai/backend/peft/layers/utils.py

@@ -1,33 +0,0 @@
-from typing import Dict
-
-import torch
-
-from invokeai.backend.peft.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.peft.layers.full_layer import FullLayer
-from invokeai.backend.peft.layers.ia3_layer import IA3Layer
-from invokeai.backend.peft.layers.loha_layer import LoHALayer
-from invokeai.backend.peft.layers.lokr_layer import LoKRLayer
-from invokeai.backend.peft.layers.lora_layer import LoRALayer
-from invokeai.backend.peft.layers.norm_layer import NormLayer
-
-
-def peft_layer_from_state_dict(layer_key: str, state_dict: Dict[str, torch.Tensor]) -> AnyLoRALayer:
-    # Detect layers according to LyCORIS detection logic(`weight_list_det`)
-    # https://github.com/KohakuBlueleaf/LyCORIS/tree/8ad8000efb79e2b879054da8c9356e6143591bad/lycoris/modules
-
-    if "lora_up.weight" in state_dict:
-        # LoRA a.k.a LoCon
-        return LoRALayer(layer_key, state_dict)
-    elif "hada_w1_a" in state_dict:
-        return LoHALayer(layer_key, state_dict)
-    elif "lokr_w1" in state_dict or "lokr_w1_a" in state_dict:
-        return LoKRLayer(layer_key, state_dict)
-    elif "diff" in state_dict:
-        # Full a.k.a Diff
-        return FullLayer(layer_key, state_dict)
-    elif "on_input" in state_dict:
-        return IA3Layer(layer_key, state_dict)
-    elif "w_norm" in state_dict:
-        return NormLayer(layer_key, state_dict)
-    else:
-        raise ValueError(f"Unsupported lora format: {state_dict.keys()}")

invokeai/backend/peft/lora.py

@@ -1,22 +0,0 @@
-# Copyright (c) 2024 The InvokeAI Development team
-from typing import Dict, Optional
-
-import torch
-
-from invokeai.backend.peft.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.raw_model import RawModel
-
-
-class LoRAModelRaw(RawModel):  # (torch.nn.Module):
-    def __init__(self, layers: Dict[str, AnyLoRALayer]):
-        self.layers = layers
-
-    def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        for _key, layer in self.layers.items():
-            layer.to(device=device, dtype=dtype)
-
-    def calc_size(self) -> int:
-        model_size = 0
-        for _, layer in self.layers.items():
-            model_size += layer.calc_size()
-        return model_size

invokeai/backend/peft/peft_patcher.py

@@ -1,102 +0,0 @@
-from contextlib import contextmanager
-from typing import Dict, Iterator, Optional, Tuple
-
-import torch
-
-from invokeai.backend.peft.lora import LoRAModelRaw
-from invokeai.backend.util.devices import TorchDevice
-from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
-
-
-class PeftPatcher:
-    @classmethod
-    @torch.no_grad()
-    @contextmanager
-    def apply_peft_patches(
-        cls,
-        model: torch.nn.Module,
-        patches: Iterator[Tuple[LoRAModelRaw, float]],
-        prefix: str,
-        cached_weights: Optional[Dict[str, torch.Tensor]] = None,
-    ):
-        """Apply one or more PEFT patches to a model.
-
-        :param model: The model to patch.
-        :param loras: An iterator that returns tuples of PEFT patches and associated weights. An iterator is used so
-            that the PEFT patches do not need to be loaded into memory all at once.
-        :param prefix: The keys in the patches will be filtered to only include weights with this prefix.
-        :cached_weights: Read-only copy of the model's state dict in CPU, for efficient unpatching purposes.
-        """
-        original_weights = OriginalWeightsStorage(cached_weights)
-        try:
-            for patch, patch_weight in patches:
-                cls._apply_peft_patch(
-                    model=model,
-                    prefix=prefix,
-                    patch=patch,
-                    patch_weight=patch_weight,
-                    original_weights=original_weights,
-                )
-
-            yield
-        finally:
-            for param_key, weight in original_weights.get_changed_weights():
-                model.get_parameter(param_key).copy_(weight)
-
-    @classmethod
-    @torch.no_grad()
-    def _apply_peft_patch(
-        cls,
-        model: torch.nn.Module,
-        prefix: str,
-        patch: LoRAModelRaw,
-        patch_weight: float,
-        original_weights: OriginalWeightsStorage,
-    ):
-        """
-        Apply one a LoRA to a model.
-        :param model: The model to patch.
-        :param patch: LoRA model to patch in.
-        :param patch_weight: LoRA patch weight.
-        :param prefix: A string prefix that precedes keys used in the LoRAs weight layers.
-        :param original_weights: Storage with original weights, filled by weights which lora patches, used for unpatching.
-        """
-
-        if patch_weight == 0:
-            return
-
-        for layer_key, layer in patch.layers.items():
-            if not layer_key.startswith(prefix):
-                continue
-
-            module = model.get_submodule(layer_key)
-
-            # All of the LoRA weight calculations will be done on the same device as the module weight.
-            # (Performance will be best if this is a CUDA device.)
-            device = module.weight.device
-            dtype = module.weight.dtype
-
-            layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
-
-            # We intentionally move to the target device first, then cast. Experimentally, this was found to
-            # be significantly faster for 16-bit CPU tensors being moved to a CUDA device than doing the
-            # same thing in a single call to '.to(...)'.
-            layer.to(device=device)
-            layer.to(dtype=torch.float32)
-
-            # TODO(ryand): Using torch.autocast(...) over explicit casting may offer a speed benefit on CUDA
-            # devices here. Experimentally, it was found to be very slow on CPU. More investigation needed.
-            for param_name, lora_param_weight in layer.get_parameters(module).items():
-                param_key = layer_key + "." + param_name
-                module_param = module.get_parameter(param_name)
-
-                # Save original weight
-                original_weights.save(param_key, module_param)
-
-                if module_param.shape != lora_param_weight.shape:
-                    lora_param_weight = lora_param_weight.reshape(module_param.shape)
-
-                lora_param_weight *= patch_weight * layer_scale
-                module_param += lora_param_weight.to(dtype=dtype)
-
-            layer.to(device=TorchDevice.CPU_DEVICE)

invokeai/backend/stable_diffusion/extensions/lora.py

@@ -12,7 +12,7 @@ from invokeai.backend.util.devices import TorchDevice
 if TYPE_CHECKING:
     from invokeai.app.invocations.model import ModelIdentifierField
     from invokeai.app.services.shared.invocation_context import InvocationContext
-    from invokeai.backend.peft.lora import LoRAModelRaw
+    from invokeai.backend.lora import LoRAModelRaw
     from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 

invokeai/frontend/web/public/locales/de.json

@@ -127,7 +127,14 @@
         "bulkDownloadRequestedDesc": "Dein Download wird vorbereitet. Dies kann ein paar Momente dauern.",
         "bulkDownloadRequestFailed": "Problem beim Download vorbereiten",
         "bulkDownloadFailed": "Download fehlgeschlagen",
-        "alwaysShowImageSizeBadge": "Zeige immer Bilder Größe Abzeichen"
+        "alwaysShowImageSizeBadge": "Zeige immer Bilder Größe Abzeichen",
+        "selectForCompare": "Zum Vergleichen auswählen",
+        "compareImage": "Bilder vergleichen",
+        "exitSearch": "Suche beenden",
+        "newestFirst": "Neueste zuerst",
+        "oldestFirst": "Älteste zuerst",
+        "openInViewer": "Im Viewer öffnen",
+        "swapImages": "Bilder tauschen"
     },
     "hotkeys": {
         "keyboardShortcuts": "Tastenkürzel",
@@ -631,7 +638,8 @@
         "archived": "Archiviert",
         "noBoards": "Kein {boardType}} Ordner",
         "hideBoards": "Ordner verstecken",
-        "viewBoards": "Ordner ansehen"
+        "viewBoards": "Ordner ansehen",
+        "deletedPrivateBoardsCannotbeRestored": "Gelöschte Boards können nicht wiederhergestellt werden. Wenn Sie „Nur Board löschen“ wählen, werden die Bilder in einen privaten, nicht kategorisierten Status für den Ersteller des Bildes versetzt."
     },
     "controlnet": {
         "showAdvanced": "Zeige Erweitert",
@@ -781,7 +789,9 @@
         "batchFieldValues": "Stapelverarbeitungswerte",
         "batchQueued": "Stapelverarbeitung eingereiht",
         "graphQueued": "Graph eingereiht",
-        "graphFailedToQueue": "Fehler beim Einreihen des Graphen"
+        "graphFailedToQueue": "Fehler beim Einreihen des Graphen",
+        "generations_one": "Generation",
+        "generations_other": "Generationen"
     },
     "metadata": {
         "negativePrompt": "Negativ Beschreibung",
@@ -1146,5 +1156,10 @@
         "noMatchingTriggers": "Keine passenden Trigger",
         "addPromptTrigger": "Prompt-Trigger hinzufügen",
         "compatibleEmbeddings": "Kompatible Einbettungen"
+    },
+    "ui": {
+        "tabs": {
+            "queue": "Warteschlange"
+        }
     }
 }

invokeai/frontend/web/public/locales/es.json

@@ -86,15 +86,15 @@
         "loadMore": "Cargar más",
         "noImagesInGallery": "No hay imágenes para mostrar",
         "deleteImage_one": "Eliminar Imagen",
-        "deleteImage_many": "",
-        "deleteImage_other": "",
+        "deleteImage_many": "Eliminar {{count}} Imágenes",
+        "deleteImage_other": "Eliminar {{count}} Imágenes",
         "deleteImagePermanent": "Las imágenes eliminadas no se pueden restaurar.",
         "assets": "Activos",
         "autoAssignBoardOnClick": "Asignación automática de tableros al hacer clic"
     },
     "hotkeys": {
         "keyboardShortcuts": "Atajos de teclado",
-        "appHotkeys": "Atajos de applicación",
+        "appHotkeys": "Atajos de aplicación",
         "generalHotkeys": "Atajos generales",
         "galleryHotkeys": "Atajos de galería",
         "unifiedCanvasHotkeys": "Atajos de lienzo unificado",
@@ -535,7 +535,7 @@
         "bottomMessage": "Al eliminar este panel y las imágenes que contiene, se restablecerán las funciones que los estén utilizando actualmente.",
         "deleteBoardAndImages": "Borrar el panel y las imágenes",
         "loading": "Cargando...",
-        "deletedBoardsCannotbeRestored": "Los paneles eliminados no se pueden restaurar",
+        "deletedBoardsCannotbeRestored": "Los paneles eliminados no se pueden restaurar. Al Seleccionar 'Borrar Solo el Panel' transferirá las imágenes a un estado sin categorizar.",
         "move": "Mover",
         "menuItemAutoAdd": "Agregar automáticamente a este panel",
         "searchBoard": "Buscando paneles…",
@@ -549,7 +549,13 @@
         "imagesWithCount_other": "{{count}} imágenes",
         "assetsWithCount_one": "{{count}} activo",
         "assetsWithCount_many": "{{count}} activos",
-        "assetsWithCount_other": "{{count}} activos"
+        "assetsWithCount_other": "{{count}} activos",
+        "hideBoards": "Ocultar Paneles",
+        "addPrivateBoard": "Agregar un tablero privado",
+        "addSharedBoard": "Agregar Panel Compartido",
+        "boards": "Paneles",
+        "archiveBoard": "Archivar Panel",
+        "archived": "Archivado"
     },
     "accordions": {
         "compositing": {

invokeai/frontend/web/public/locales/it.json

@@ -496,7 +496,9 @@
         "main": "Principali",
         "noModelsInstalledDesc1": "Installa i modelli con",
         "ipAdapters": "Adattatori IP",
-        "noMatchingModels": "Nessun modello corrispondente"
+        "noMatchingModels": "Nessun modello corrispondente",
+        "starterModelsInModelManager": "I modelli iniziali possono essere trovati in Gestione Modelli",
+        "spandrelImageToImage": "Immagine a immagine (Spandrel)"
     },
     "parameters": {
         "images": "Immagini",
@@ -510,7 +512,7 @@
         "perlinNoise": "Rumore Perlin",
         "type": "Tipo",
         "strength": "Forza",
-        "upscaling": "Ampliamento",
+        "upscaling": "Amplia",
         "scale": "Scala",
         "imageFit": "Adatta l'immagine iniziale alle dimensioni di output",
         "scaleBeforeProcessing": "Scala prima dell'elaborazione",
@@ -593,7 +595,7 @@
         "globalPositivePromptPlaceholder": "Prompt positivo globale",
         "globalNegativePromptPlaceholder": "Prompt negativo globale",
         "processImage": "Elabora Immagine",
-        "sendToUpscale": "Invia a Ampliare",
+        "sendToUpscale": "Invia a Amplia",
         "postProcessing": "Post-elaborazione (Shift + U)"
     },
     "settings": {
@@ -1420,7 +1422,7 @@
         "paramUpscaleMethod": {
             "heading": "Metodo di ampliamento",
             "paragraphs": [
-                "Metodo utilizzato per eseguire l'ampliamento dell'immagine per la correzione ad alta risoluzione."
+                "Metodo utilizzato per ampliare l'immagine per la correzione ad alta risoluzione."
             ]
         },
         "patchmatchDownScaleSize": {
@@ -1528,7 +1530,7 @@
         },
         "upscaleModel": {
             "paragraphs": [
-                "Il modello di ampliamento (Upscale), scala l'immagine alle dimensioni di uscita prima di aggiungere i dettagli. È possibile utilizzare qualsiasi modello di ampliamento supportato, ma alcuni sono specializzati per diversi tipi di immagini, come foto o disegni al tratto."
+                "Il modello di ampliamento, scala l'immagine alle dimensioni di uscita prima di aggiungere i dettagli. È possibile utilizzare qualsiasi modello di ampliamento supportato, ma alcuni sono specializzati per diversi tipi di immagini, come foto o disegni al tratto."
             ],
             "heading": "Modello di ampliamento"
         },
@@ -1720,26 +1722,27 @@
             "modelsTab": "$t(ui.tabs.models) $t(common.tab)",
             "queue": "Coda",
             "queueTab": "$t(ui.tabs.queue) $t(common.tab)",
-            "upscaling": "Ampliamento",
+            "upscaling": "Amplia",
             "upscalingTab": "$t(ui.tabs.upscaling) $t(common.tab)"
         }
     },
     "upscaling": {
         "creativity": "Creatività",
         "structure": "Struttura",
-        "upscaleModel": "Modello di Ampliamento",
+        "upscaleModel": "Modello di ampliamento",
         "scale": "Scala",
         "missingModelsWarning": "Visita <LinkComponent>Gestione modelli</LinkComponent> per installare i modelli richiesti:",
         "mainModelDesc": "Modello principale (architettura SD1.5 o SDXL)",
         "tileControlNetModelDesc": "Modello Tile ControlNet per l'architettura del modello principale scelto",
-        "upscaleModelDesc": "Modello per l'ampliamento (da immagine a immagine)",
+        "upscaleModelDesc": "Modello per l'ampliamento (immagine a immagine)",
         "missingUpscaleInitialImage": "Immagine iniziale mancante per l'ampliamento",
         "missingUpscaleModel": "Modello per l’ampliamento mancante",
         "missingTileControlNetModel": "Nessun modello ControlNet Tile valido installato",
         "postProcessingModel": "Modello di post-elaborazione",
         "postProcessingMissingModelWarning": "Visita <LinkComponent>Gestione modelli</LinkComponent> per installare un modello di post-elaborazione (da immagine a immagine).",
         "exceedsMaxSize": "Le impostazioni di ampliamento superano il limite massimo delle dimensioni",
-        "exceedsMaxSizeDetails": "Il limite massimo di ampliamento è {{maxUpscaleDimension}}x{{maxUpscaleDimension}} pixel. Prova un'immagine più piccola o diminuisci la scala selezionata."
+        "exceedsMaxSizeDetails": "Il limite massimo di ampliamento è {{maxUpscaleDimension}}x{{maxUpscaleDimension}} pixel. Prova un'immagine più piccola o diminuisci la scala selezionata.",
+        "upscale": "Amplia"
     },
     "upsell": {
         "inviteTeammates": "Invita collaboratori",
@@ -1789,6 +1792,7 @@
         "positivePromptColumn": "'prompt' o 'positive_prompt'",
         "noTemplates": "Nessun modello",
         "acceptedColumnsKeys": "Colonne/chiavi accettate:",
-        "templateActions": "Azioni modello"
+        "templateActions": "Azioni modello",
+        "promptTemplateCleared": "Modello di prompt cancellato"
     }
 }

invokeai/frontend/web/public/locales/ru.json

@@ -501,7 +501,8 @@
         "noModelsInstalled": "Нет установленных моделей",
         "noModelsInstalledDesc1": "Установите модели с помощью",
         "noMatchingModels": "Нет подходящих моделей",
-        "ipAdapters": "IP адаптеры"
+        "ipAdapters": "IP адаптеры",
+        "starterModelsInModelManager": "Стартовые модели можно найти в Менеджере моделей"
     },
     "parameters": {
         "images": "Изображения",
@@ -1758,7 +1759,8 @@
         "postProcessingModel": "Модель постобработки",
         "tileControlNetModelDesc": "Модель ControlNet для выбранной архитектуры основной модели",
         "missingModelsWarning": "Зайдите в <LinkComponent>Менеджер моделей</LinkComponent> чтоб установить необходимые модели:",
-        "postProcessingMissingModelWarning": "Посетите <LinkComponent>Менеджер моделей</LinkComponent>, чтобы установить модель постобработки (img2img)."
+        "postProcessingMissingModelWarning": "Посетите <LinkComponent>Менеджер моделей</LinkComponent>, чтобы установить модель постобработки (img2img).",
+        "upscale": "Увеличить"
     },
     "stylePresets": {
         "noMatchingTemplates": "Нет подходящих шаблонов",
@@ -1804,7 +1806,8 @@
         "noTemplates": "Нет шаблонов",
         "promptTemplatesDesc2": "Используйте строку-заполнитель <Pre>{{placeholder}}</Pre>, чтобы указать место, куда должен быть включен ваш запрос в шаблоне.",
         "searchByName": "Поиск по имени",
-        "shared": "Общий"
+        "shared": "Общий",
+        "promptTemplateCleared": "Шаблон запроса создан"
     },
     "upsell": {
         "inviteTeammates": "Пригласите членов команды",

invokeai/frontend/web/public/locales/zh_CN.json

@@ -154,7 +154,8 @@
         "displaySearch": "显示搜索",
         "stretchToFit": "拉伸以适应",
         "exitCompare": "退出对比",
-        "compareHelp1": "在点击图库中的图片或使用箭头键切换比较图片时，请按住<Kbd>Alt</Kbd> 键。"
+        "compareHelp1": "在点击图库中的图片或使用箭头键切换比较图片时，请按住<Kbd>Alt</Kbd> 键。",
+        "go": "运行"
     },
     "hotkeys": {
         "keyboardShortcuts": "快捷键",
@@ -494,7 +495,9 @@
         "huggingFacePlaceholder": "所有者或模型名称",
         "huggingFaceRepoID": "HuggingFace仓库ID",
         "loraTriggerPhrases": "LoRA 触发词",
-        "ipAdapters": "IP适配器"
+        "ipAdapters": "IP适配器",
+        "spandrelImageToImage": "图生图(Spandrel)",
+        "starterModelsInModelManager": "您可以在模型管理器中找到初始模型"
     },
     "parameters": {
         "images": "图像",
@@ -695,7 +698,9 @@
         "outOfMemoryErrorDesc": "您当前的生成设置已超出系统处理能力.请调整设置后再次尝试.",
         "parametersSet": "参数已恢复",
         "errorCopied": "错误信息已复制",
-        "modelImportCanceled": "模型导入已取消"
+        "modelImportCanceled": "模型导入已取消",
+        "importFailed": "导入失败",
+        "importSuccessful": "导入成功"
     },
     "unifiedCanvas": {
         "layer": "图层",
@@ -1705,12 +1710,55 @@
         "missingModelsWarning": "请访问<LinkComponent>模型管理器</LinkComponent> 安装所需的模型：",
         "mainModelDesc": "主模型（SD1.5或SDXL架构）",
         "exceedsMaxSize": "放大设置超出了最大尺寸限制",
-        "exceedsMaxSizeDetails": "最大放大限制是 {{maxUpscaleDimension}}x{{maxUpscaleDimension}} 像素.请尝试一个较小的图像或减少您的缩放选择."
+        "exceedsMaxSizeDetails": "最大放大限制是 {{maxUpscaleDimension}}x{{maxUpscaleDimension}} 像素.请尝试一个较小的图像或减少您的缩放选择.",
+        "upscale": "放大"
     },
     "upsell": {
         "inviteTeammates": "邀请团队成员",
         "professional": "专业",
         "professionalUpsell": "可在 Invoke 的专业版中使用.点击此处或访问 invoke.com/pricing 了解更多详情.",
         "shareAccess": "共享访问权限"
+    },
+    "stylePresets": {
+        "positivePrompt": "正向提示词",
+        "preview": "预览",
+        "deleteImage": "删除图像",
+        "deleteTemplate": "删除模版",
+        "deleteTemplate2": "您确定要删除这个模板吗？请注意，删除后无法恢复.",
+        "importTemplates": "导入提示模板，支持CSV或JSON格式",
+        "insertPlaceholder": "插入一个占位符",
+        "myTemplates": "我的模版",
+        "name": "名称",
+        "type": "类型",
+        "unableToDeleteTemplate": "无法删除提示模板",
+        "updatePromptTemplate": "更新提示词模版",
+        "exportPromptTemplates": "导出我的提示模板为CSV格式",
+        "exportDownloaded": "导出已下载",
+        "noMatchingTemplates": "无匹配的模版",
+        "promptTemplatesDesc1": "提示模板可以帮助您在编写提示时添加预设的文本内容.",
+        "promptTemplatesDesc3": "如果您没有使用占位符，那么模板的内容将会被添加到您提示的末尾.",
+        "searchByName": "按名称搜索",
+        "shared": "已分享",
+        "sharedTemplates": "已分享的模版",
+        "templateActions": "模版操作",
+        "templateDeleted": "提示模版已删除",
+        "toggleViewMode": "切换显示模式",
+        "uploadImage": "上传图像",
+        "active": "激活",
+        "choosePromptTemplate": "选择提示词模板",
+        "clearTemplateSelection": "清除模版选择",
+        "copyTemplate": "拷贝模版",
+        "createPromptTemplate": "创建提示词模版",
+        "defaultTemplates": "默认模版",
+        "editTemplate": "编辑模版",
+        "exportFailed": "无法生成并下载CSV文件",
+        "flatten": "将选定的模板内容合并到当前提示中",
+        "negativePrompt": "反向提示词",
+        "promptTemplateCleared": "提示模板已清除",
+        "useForTemplate": "用于提示词模版",
+        "viewList": "预览模版列表",
+        "viewModeTooltip": "这是您的提示在当前选定的模板下的预览效果。如需编辑提示，请直接在文本框中点击进行修改.",
+        "noTemplates": "无模版",
+        "private": "私密"
     }
 }

invokeai/version/invokeai_version.py

@@ -1 +1 @@
-__version__ = "4.2.9rc1"
+__version__ = "4.2.9"

tests/backend/model_manager/test_lora.py

@@ -1,9 +1,12 @@
+# test that if the model's device changes while the lora is applied, the weights can still be restored
+
+# test that LoRA patching works on both CPU and CUDA
+
 import pytest
 import torch
 
+from invokeai.backend.lora import LoRALayer, LoRAModelRaw
 from invokeai.backend.model_patcher import ModelPatcher
-from invokeai.backend.peft.layers.lora_layer import LoRALayer
-from invokeai.backend.peft.lora import LoRAModelRaw
 
 
 @pytest.mark.parametrize(
@@ -35,7 +38,7 @@ def test_apply_lora(device):
             },
         )
     }
-    lora = LoRAModelRaw(lora_layers)
+    lora = LoRAModelRaw("lora_name", lora_layers)
 
     lora_weight = 0.5
     orig_linear_weight = model["linear_layer_1"].weight.data.detach().clone()
@@ -79,7 +82,7 @@ def test_apply_lora_change_device():
             },
         )
     }
-    lora = LoRAModelRaw(lora_layers)
+    lora = LoRAModelRaw("lora_name", lora_layers)
 
     orig_linear_weight = model["linear_layer_1"].weight.data.detach().clone()
 

tests/backend/peft/conversions/lora_state_dicts/flux_lora_diffusers_format.py

@@ -1,990 +0,0 @@
-state_dict_keys = [
-    "transformer.single_transformer_blocks.0.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.0.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.0.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.0.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.0.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.0.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.0.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.0.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.0.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.0.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.0.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.0.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.1.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.1.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.1.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.1.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.1.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.1.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.1.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.1.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.1.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.1.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.1.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.1.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.10.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.10.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.10.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.10.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.10.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.10.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.10.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.10.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.10.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.10.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.10.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.10.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.11.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.11.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.11.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.11.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.11.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.11.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.11.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.11.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.11.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.11.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.11.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.11.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.12.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.12.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.12.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.12.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.12.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.12.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.12.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.12.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.12.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.12.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.12.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.12.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.13.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.13.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.13.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.13.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.13.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.13.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.13.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.13.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.13.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.13.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.13.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.13.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.14.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.14.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.14.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.14.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.14.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.14.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.14.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.14.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.14.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.14.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.14.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.14.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.15.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.15.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.15.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.15.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.15.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.15.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.15.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.15.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.15.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.15.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.15.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.15.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.16.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.16.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.16.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.16.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.16.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.16.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.16.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.16.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.16.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.16.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.16.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.16.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.17.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.17.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.17.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.17.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.17.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.17.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.17.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.17.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.17.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.17.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.17.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.17.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.18.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.18.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.18.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.18.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.18.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.18.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.18.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.18.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.18.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.18.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.18.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.18.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.19.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.19.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.19.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.19.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.19.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.19.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.19.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.19.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.19.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.19.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.19.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.19.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.2.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.2.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.2.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.2.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.2.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.2.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.2.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.2.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.2.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.2.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.2.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.2.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.20.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.20.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.20.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.20.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.20.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.20.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.20.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.20.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.20.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.20.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.20.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.20.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.21.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.21.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.21.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.21.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.21.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.21.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.21.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.21.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.21.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.21.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.21.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.21.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.22.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.22.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.22.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.22.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.22.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.22.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.22.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.22.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.22.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.22.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.22.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.22.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.23.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.23.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.23.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.23.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.23.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.23.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.23.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.23.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.23.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.23.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.23.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.23.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.24.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.24.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.24.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.24.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.24.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.24.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.24.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.24.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.24.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.24.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.24.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.24.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.25.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.25.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.25.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.25.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.25.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.25.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.25.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.25.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.25.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.25.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.25.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.25.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.26.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.26.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.26.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.26.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.26.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.26.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.26.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.26.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.26.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.26.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.26.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.26.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.27.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.27.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.27.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.27.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.27.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.27.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.27.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.27.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.27.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.27.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.27.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.27.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.28.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.28.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.28.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.28.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.28.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.28.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.28.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.28.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.28.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.28.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.28.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.28.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.29.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.29.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.29.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.29.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.29.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.29.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.29.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.29.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.29.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.29.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.29.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.29.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.3.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.3.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.3.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.3.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.3.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.3.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.3.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.3.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.3.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.3.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.3.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.3.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.30.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.30.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.30.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.30.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.30.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.30.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.30.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.30.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.30.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.30.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.30.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.30.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.31.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.31.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.31.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.31.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.31.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.31.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.31.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.31.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.31.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.31.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.31.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.31.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.32.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.32.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.32.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.32.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.32.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.32.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.32.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.32.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.32.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.32.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.32.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.32.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.33.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.33.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.33.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.33.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.33.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.33.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.33.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.33.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.33.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.33.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.33.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.33.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.34.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.34.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.34.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.34.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.34.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.34.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.34.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.34.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.34.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.34.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.34.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.34.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.35.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.35.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.35.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.35.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.35.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.35.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.35.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.35.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.35.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.35.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.35.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.35.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.36.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.36.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.36.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.36.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.36.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.36.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.36.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.36.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.36.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.36.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.36.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.36.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.37.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.37.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.37.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.37.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.37.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.37.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.37.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.37.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.37.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.37.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.37.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.37.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.4.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.4.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.4.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.4.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.4.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.4.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.4.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.4.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.4.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.4.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.4.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.4.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.5.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.5.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.5.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.5.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.5.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.5.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.5.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.5.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.5.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.5.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.5.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.5.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.6.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.6.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.6.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.6.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.6.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.6.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.6.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.6.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.6.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.6.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.6.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.6.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.7.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.7.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.7.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.7.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.7.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.7.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.7.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.7.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.7.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.7.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.7.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.7.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.8.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.8.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.8.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.8.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.8.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.8.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.8.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.8.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.8.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.8.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.8.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.8.proj_out.lora_B.weight",
-    "transformer.single_transformer_blocks.9.attn.to_k.lora_A.weight",
-    "transformer.single_transformer_blocks.9.attn.to_k.lora_B.weight",
-    "transformer.single_transformer_blocks.9.attn.to_q.lora_A.weight",
-    "transformer.single_transformer_blocks.9.attn.to_q.lora_B.weight",
-    "transformer.single_transformer_blocks.9.attn.to_v.lora_A.weight",
-    "transformer.single_transformer_blocks.9.attn.to_v.lora_B.weight",
-    "transformer.single_transformer_blocks.9.norm.linear.lora_A.weight",
-    "transformer.single_transformer_blocks.9.norm.linear.lora_B.weight",
-    "transformer.single_transformer_blocks.9.proj_mlp.lora_A.weight",
-    "transformer.single_transformer_blocks.9.proj_mlp.lora_B.weight",
-    "transformer.single_transformer_blocks.9.proj_out.lora_A.weight",
-    "transformer.single_transformer_blocks.9.proj_out.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.0.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.0.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.0.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.0.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.0.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.0.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.0.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.0.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.0.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.0.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.0.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.0.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.0.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.0.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.1.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.1.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.1.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.1.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.1.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.1.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.1.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.1.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.1.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.1.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.1.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.1.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.1.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.1.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.10.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.10.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.10.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.10.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.10.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.10.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.10.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.10.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.10.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.10.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.10.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.10.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.10.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.10.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.11.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.11.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.11.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.11.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.11.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.11.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.11.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.11.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.11.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.11.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.11.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.11.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.11.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.11.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.12.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.12.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.12.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.12.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.12.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.12.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.12.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.12.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.12.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.12.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.12.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.12.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.12.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.12.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.13.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.13.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.13.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.13.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.13.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.13.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.13.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.13.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.13.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.13.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.13.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.13.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.13.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.13.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.14.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.14.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.14.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.14.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.14.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.14.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.14.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.14.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.14.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.14.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.14.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.14.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.14.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.14.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.15.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.15.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.15.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.15.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.15.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.15.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.15.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.15.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.15.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.15.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.15.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.15.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.15.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.15.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.16.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.16.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.16.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.16.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.16.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.16.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.16.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.16.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.16.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.16.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.16.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.16.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.16.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.16.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.17.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.17.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.17.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.17.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.17.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.17.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.17.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.17.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.17.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.17.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.17.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.17.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.17.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.17.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.18.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.18.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.18.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.18.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.18.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.18.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.18.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.18.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.18.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.18.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.18.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.18.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.18.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.18.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.2.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.2.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.2.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.2.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.2.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.2.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.2.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.2.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.2.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.2.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.2.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.2.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.2.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.2.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.3.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.3.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.3.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.3.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.3.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.3.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.3.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.3.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.3.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.3.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.3.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.3.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.3.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.3.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.4.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.4.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.4.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.4.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.4.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.4.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.4.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.4.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.4.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.4.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.4.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.4.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.4.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.4.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.5.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.5.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.5.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.5.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.5.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.5.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.5.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.5.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.5.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.5.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.5.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.5.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.5.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.5.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.6.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.6.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.6.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.6.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.6.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.6.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.6.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.6.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.6.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.6.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.6.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.6.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.6.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.6.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.7.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.7.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.7.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.7.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.7.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.7.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.7.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.7.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.7.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.7.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.7.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.7.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.7.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.7.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.8.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.8.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.8.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.8.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.8.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.8.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.8.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.8.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.8.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.8.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.8.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.8.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.8.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.8.norm1_context.linear.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.add_k_proj.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.add_k_proj.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.add_q_proj.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.add_q_proj.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.add_v_proj.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.add_v_proj.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.to_add_out.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.to_add_out.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.to_k.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.to_k.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.to_out.0.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.to_out.0.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.to_q.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.to_q.lora_B.weight",
-    "transformer.transformer_blocks.9.attn.to_v.lora_A.weight",
-    "transformer.transformer_blocks.9.attn.to_v.lora_B.weight",
-    "transformer.transformer_blocks.9.ff.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.9.ff.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.9.ff.net.2.lora_A.weight",
-    "transformer.transformer_blocks.9.ff.net.2.lora_B.weight",
-    "transformer.transformer_blocks.9.ff_context.net.0.proj.lora_A.weight",
-    "transformer.transformer_blocks.9.ff_context.net.0.proj.lora_B.weight",
-    "transformer.transformer_blocks.9.ff_context.net.2.lora_A.weight",
-    "transformer.transformer_blocks.9.ff_context.net.2.lora_B.weight",
-    "transformer.transformer_blocks.9.norm1.linear.lora_A.weight",
-    "transformer.transformer_blocks.9.norm1.linear.lora_B.weight",
-    "transformer.transformer_blocks.9.norm1_context.linear.lora_A.weight",
-    "transformer.transformer_blocks.9.norm1_context.linear.lora_B.weight",
-]

tests/backend/peft/conversions/lora_state_dicts/flux_lora_kohya_format.py

@@ -1,914 +0,0 @@
-state_dict_keys = [
-    "lora_unet_double_blocks_0_img_attn_proj.alpha",
-    "lora_unet_double_blocks_0_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_0_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_0_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_0_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_0_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_0_img_mlp_0.alpha",
-    "lora_unet_double_blocks_0_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_0_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_0_img_mlp_2.alpha",
-    "lora_unet_double_blocks_0_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_0_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_0_img_mod_lin.alpha",
-    "lora_unet_double_blocks_0_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_0_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_0_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_0_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_0_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_0_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_0_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_0_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_0_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_0_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_0_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_0_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_0_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_0_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_0_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_0_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_0_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_10_img_attn_proj.alpha",
-    "lora_unet_double_blocks_10_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_10_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_10_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_10_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_10_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_10_img_mlp_0.alpha",
-    "lora_unet_double_blocks_10_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_10_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_10_img_mlp_2.alpha",
-    "lora_unet_double_blocks_10_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_10_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_10_img_mod_lin.alpha",
-    "lora_unet_double_blocks_10_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_10_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_10_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_10_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_10_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_10_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_10_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_10_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_10_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_10_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_10_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_10_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_10_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_10_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_10_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_10_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_10_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_11_img_attn_proj.alpha",
-    "lora_unet_double_blocks_11_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_11_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_11_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_11_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_11_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_11_img_mlp_0.alpha",
-    "lora_unet_double_blocks_11_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_11_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_11_img_mlp_2.alpha",
-    "lora_unet_double_blocks_11_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_11_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_11_img_mod_lin.alpha",
-    "lora_unet_double_blocks_11_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_11_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_11_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_11_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_11_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_11_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_11_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_11_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_11_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_11_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_11_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_11_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_11_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_11_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_11_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_11_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_11_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_12_img_attn_proj.alpha",
-    "lora_unet_double_blocks_12_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_12_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_12_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_12_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_12_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_12_img_mlp_0.alpha",
-    "lora_unet_double_blocks_12_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_12_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_12_img_mlp_2.alpha",
-    "lora_unet_double_blocks_12_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_12_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_12_img_mod_lin.alpha",
-    "lora_unet_double_blocks_12_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_12_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_12_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_12_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_12_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_12_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_12_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_12_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_12_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_12_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_12_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_12_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_12_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_12_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_12_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_12_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_12_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_13_img_attn_proj.alpha",
-    "lora_unet_double_blocks_13_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_13_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_13_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_13_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_13_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_13_img_mlp_0.alpha",
-    "lora_unet_double_blocks_13_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_13_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_13_img_mlp_2.alpha",
-    "lora_unet_double_blocks_13_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_13_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_13_img_mod_lin.alpha",
-    "lora_unet_double_blocks_13_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_13_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_13_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_13_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_13_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_13_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_13_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_13_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_13_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_13_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_13_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_13_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_13_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_13_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_13_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_13_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_13_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_14_img_attn_proj.alpha",
-    "lora_unet_double_blocks_14_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_14_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_14_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_14_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_14_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_14_img_mlp_0.alpha",
-    "lora_unet_double_blocks_14_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_14_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_14_img_mlp_2.alpha",
-    "lora_unet_double_blocks_14_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_14_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_14_img_mod_lin.alpha",
-    "lora_unet_double_blocks_14_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_14_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_14_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_14_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_14_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_14_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_14_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_14_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_14_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_14_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_14_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_14_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_14_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_14_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_14_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_14_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_14_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_15_img_attn_proj.alpha",
-    "lora_unet_double_blocks_15_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_15_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_15_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_15_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_15_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_15_img_mlp_0.alpha",
-    "lora_unet_double_blocks_15_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_15_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_15_img_mlp_2.alpha",
-    "lora_unet_double_blocks_15_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_15_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_15_img_mod_lin.alpha",
-    "lora_unet_double_blocks_15_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_15_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_15_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_15_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_15_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_15_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_15_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_15_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_15_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_15_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_15_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_15_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_15_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_15_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_15_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_15_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_15_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_16_img_attn_proj.alpha",
-    "lora_unet_double_blocks_16_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_16_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_16_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_16_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_16_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_16_img_mlp_0.alpha",
-    "lora_unet_double_blocks_16_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_16_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_16_img_mlp_2.alpha",
-    "lora_unet_double_blocks_16_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_16_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_16_img_mod_lin.alpha",
-    "lora_unet_double_blocks_16_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_16_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_16_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_16_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_16_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_16_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_16_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_16_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_16_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_16_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_16_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_16_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_16_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_16_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_16_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_16_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_16_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_17_img_attn_proj.alpha",
-    "lora_unet_double_blocks_17_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_17_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_17_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_17_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_17_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_17_img_mlp_0.alpha",
-    "lora_unet_double_blocks_17_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_17_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_17_img_mlp_2.alpha",
-    "lora_unet_double_blocks_17_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_17_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_17_img_mod_lin.alpha",
-    "lora_unet_double_blocks_17_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_17_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_17_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_17_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_17_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_17_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_17_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_17_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_17_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_17_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_17_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_17_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_17_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_17_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_17_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_17_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_17_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_18_img_attn_proj.alpha",
-    "lora_unet_double_blocks_18_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_18_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_18_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_18_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_18_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_18_img_mlp_0.alpha",
-    "lora_unet_double_blocks_18_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_18_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_18_img_mlp_2.alpha",
-    "lora_unet_double_blocks_18_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_18_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_18_img_mod_lin.alpha",
-    "lora_unet_double_blocks_18_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_18_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_18_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_18_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_18_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_18_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_18_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_18_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_18_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_18_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_18_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_18_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_18_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_18_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_18_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_18_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_18_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_1_img_attn_proj.alpha",
-    "lora_unet_double_blocks_1_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_1_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_1_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_1_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_1_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_1_img_mlp_0.alpha",
-    "lora_unet_double_blocks_1_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_1_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_1_img_mlp_2.alpha",
-    "lora_unet_double_blocks_1_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_1_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_1_img_mod_lin.alpha",
-    "lora_unet_double_blocks_1_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_1_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_1_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_1_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_1_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_1_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_1_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_1_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_1_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_1_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_1_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_1_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_1_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_1_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_1_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_1_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_1_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_2_img_attn_proj.alpha",
-    "lora_unet_double_blocks_2_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_2_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_2_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_2_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_2_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_2_img_mlp_0.alpha",
-    "lora_unet_double_blocks_2_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_2_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_2_img_mlp_2.alpha",
-    "lora_unet_double_blocks_2_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_2_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_2_img_mod_lin.alpha",
-    "lora_unet_double_blocks_2_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_2_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_2_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_2_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_2_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_2_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_2_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_2_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_2_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_2_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_2_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_2_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_2_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_2_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_2_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_2_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_2_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_3_img_attn_proj.alpha",
-    "lora_unet_double_blocks_3_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_3_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_3_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_3_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_3_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_3_img_mlp_0.alpha",
-    "lora_unet_double_blocks_3_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_3_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_3_img_mlp_2.alpha",
-    "lora_unet_double_blocks_3_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_3_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_3_img_mod_lin.alpha",
-    "lora_unet_double_blocks_3_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_3_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_3_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_3_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_3_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_3_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_3_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_3_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_3_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_3_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_3_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_3_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_3_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_3_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_3_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_3_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_3_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_4_img_attn_proj.alpha",
-    "lora_unet_double_blocks_4_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_4_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_4_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_4_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_4_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_4_img_mlp_0.alpha",
-    "lora_unet_double_blocks_4_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_4_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_4_img_mlp_2.alpha",
-    "lora_unet_double_blocks_4_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_4_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_4_img_mod_lin.alpha",
-    "lora_unet_double_blocks_4_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_4_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_4_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_4_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_4_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_4_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_4_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_4_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_4_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_4_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_4_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_4_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_4_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_4_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_4_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_4_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_4_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_5_img_attn_proj.alpha",
-    "lora_unet_double_blocks_5_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_5_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_5_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_5_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_5_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_5_img_mlp_0.alpha",
-    "lora_unet_double_blocks_5_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_5_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_5_img_mlp_2.alpha",
-    "lora_unet_double_blocks_5_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_5_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_5_img_mod_lin.alpha",
-    "lora_unet_double_blocks_5_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_5_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_5_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_5_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_5_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_5_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_5_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_5_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_5_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_5_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_5_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_5_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_5_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_5_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_5_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_5_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_5_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_6_img_attn_proj.alpha",
-    "lora_unet_double_blocks_6_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_6_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_6_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_6_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_6_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_6_img_mlp_0.alpha",
-    "lora_unet_double_blocks_6_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_6_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_6_img_mlp_2.alpha",
-    "lora_unet_double_blocks_6_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_6_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_6_img_mod_lin.alpha",
-    "lora_unet_double_blocks_6_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_6_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_6_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_6_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_6_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_6_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_6_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_6_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_6_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_6_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_6_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_6_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_6_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_6_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_6_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_6_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_6_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_7_img_attn_proj.alpha",
-    "lora_unet_double_blocks_7_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_7_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_7_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_7_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_7_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_7_img_mlp_0.alpha",
-    "lora_unet_double_blocks_7_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_7_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_7_img_mlp_2.alpha",
-    "lora_unet_double_blocks_7_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_7_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_7_img_mod_lin.alpha",
-    "lora_unet_double_blocks_7_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_7_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_7_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_7_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_7_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_7_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_7_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_7_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_7_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_7_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_7_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_7_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_7_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_7_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_7_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_7_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_7_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_8_img_attn_proj.alpha",
-    "lora_unet_double_blocks_8_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_8_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_8_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_8_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_8_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_8_img_mlp_0.alpha",
-    "lora_unet_double_blocks_8_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_8_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_8_img_mlp_2.alpha",
-    "lora_unet_double_blocks_8_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_8_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_8_img_mod_lin.alpha",
-    "lora_unet_double_blocks_8_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_8_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_8_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_8_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_8_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_8_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_8_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_8_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_8_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_8_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_8_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_8_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_8_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_8_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_8_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_8_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_8_txt_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_9_img_attn_proj.alpha",
-    "lora_unet_double_blocks_9_img_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_9_img_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_9_img_attn_qkv.alpha",
-    "lora_unet_double_blocks_9_img_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_9_img_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_9_img_mlp_0.alpha",
-    "lora_unet_double_blocks_9_img_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_9_img_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_9_img_mlp_2.alpha",
-    "lora_unet_double_blocks_9_img_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_9_img_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_9_img_mod_lin.alpha",
-    "lora_unet_double_blocks_9_img_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_9_img_mod_lin.lora_up.weight",
-    "lora_unet_double_blocks_9_txt_attn_proj.alpha",
-    "lora_unet_double_blocks_9_txt_attn_proj.lora_down.weight",
-    "lora_unet_double_blocks_9_txt_attn_proj.lora_up.weight",
-    "lora_unet_double_blocks_9_txt_attn_qkv.alpha",
-    "lora_unet_double_blocks_9_txt_attn_qkv.lora_down.weight",
-    "lora_unet_double_blocks_9_txt_attn_qkv.lora_up.weight",
-    "lora_unet_double_blocks_9_txt_mlp_0.alpha",
-    "lora_unet_double_blocks_9_txt_mlp_0.lora_down.weight",
-    "lora_unet_double_blocks_9_txt_mlp_0.lora_up.weight",
-    "lora_unet_double_blocks_9_txt_mlp_2.alpha",
-    "lora_unet_double_blocks_9_txt_mlp_2.lora_down.weight",
-    "lora_unet_double_blocks_9_txt_mlp_2.lora_up.weight",
-    "lora_unet_double_blocks_9_txt_mod_lin.alpha",
-    "lora_unet_double_blocks_9_txt_mod_lin.lora_down.weight",
-    "lora_unet_double_blocks_9_txt_mod_lin.lora_up.weight",
-    "lora_unet_single_blocks_0_linear1.alpha",
-    "lora_unet_single_blocks_0_linear1.lora_down.weight",
-    "lora_unet_single_blocks_0_linear1.lora_up.weight",
-    "lora_unet_single_blocks_0_linear2.alpha",
-    "lora_unet_single_blocks_0_linear2.lora_down.weight",
-    "lora_unet_single_blocks_0_linear2.lora_up.weight",
-    "lora_unet_single_blocks_0_modulation_lin.alpha",
-    "lora_unet_single_blocks_0_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_0_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_10_linear1.alpha",
-    "lora_unet_single_blocks_10_linear1.lora_down.weight",
-    "lora_unet_single_blocks_10_linear1.lora_up.weight",
-    "lora_unet_single_blocks_10_linear2.alpha",
-    "lora_unet_single_blocks_10_linear2.lora_down.weight",
-    "lora_unet_single_blocks_10_linear2.lora_up.weight",
-    "lora_unet_single_blocks_10_modulation_lin.alpha",
-    "lora_unet_single_blocks_10_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_10_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_11_linear1.alpha",
-    "lora_unet_single_blocks_11_linear1.lora_down.weight",
-    "lora_unet_single_blocks_11_linear1.lora_up.weight",
-    "lora_unet_single_blocks_11_linear2.alpha",
-    "lora_unet_single_blocks_11_linear2.lora_down.weight",
-    "lora_unet_single_blocks_11_linear2.lora_up.weight",
-    "lora_unet_single_blocks_11_modulation_lin.alpha",
-    "lora_unet_single_blocks_11_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_11_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_12_linear1.alpha",
-    "lora_unet_single_blocks_12_linear1.lora_down.weight",
-    "lora_unet_single_blocks_12_linear1.lora_up.weight",
-    "lora_unet_single_blocks_12_linear2.alpha",
-    "lora_unet_single_blocks_12_linear2.lora_down.weight",
-    "lora_unet_single_blocks_12_linear2.lora_up.weight",
-    "lora_unet_single_blocks_12_modulation_lin.alpha",
-    "lora_unet_single_blocks_12_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_12_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_13_linear1.alpha",
-    "lora_unet_single_blocks_13_linear1.lora_down.weight",
-    "lora_unet_single_blocks_13_linear1.lora_up.weight",
-    "lora_unet_single_blocks_13_linear2.alpha",
-    "lora_unet_single_blocks_13_linear2.lora_down.weight",
-    "lora_unet_single_blocks_13_linear2.lora_up.weight",
-    "lora_unet_single_blocks_13_modulation_lin.alpha",
-    "lora_unet_single_blocks_13_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_13_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_14_linear1.alpha",
-    "lora_unet_single_blocks_14_linear1.lora_down.weight",
-    "lora_unet_single_blocks_14_linear1.lora_up.weight",
-    "lora_unet_single_blocks_14_linear2.alpha",
-    "lora_unet_single_blocks_14_linear2.lora_down.weight",
-    "lora_unet_single_blocks_14_linear2.lora_up.weight",
-    "lora_unet_single_blocks_14_modulation_lin.alpha",
-    "lora_unet_single_blocks_14_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_14_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_15_linear1.alpha",
-    "lora_unet_single_blocks_15_linear1.lora_down.weight",
-    "lora_unet_single_blocks_15_linear1.lora_up.weight",
-    "lora_unet_single_blocks_15_linear2.alpha",
-    "lora_unet_single_blocks_15_linear2.lora_down.weight",
-    "lora_unet_single_blocks_15_linear2.lora_up.weight",
-    "lora_unet_single_blocks_15_modulation_lin.alpha",
-    "lora_unet_single_blocks_15_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_15_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_16_linear1.alpha",
-    "lora_unet_single_blocks_16_linear1.lora_down.weight",
-    "lora_unet_single_blocks_16_linear1.lora_up.weight",
-    "lora_unet_single_blocks_16_linear2.alpha",
-    "lora_unet_single_blocks_16_linear2.lora_down.weight",
-    "lora_unet_single_blocks_16_linear2.lora_up.weight",
-    "lora_unet_single_blocks_16_modulation_lin.alpha",
-    "lora_unet_single_blocks_16_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_16_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_17_linear1.alpha",
-    "lora_unet_single_blocks_17_linear1.lora_down.weight",
-    "lora_unet_single_blocks_17_linear1.lora_up.weight",
-    "lora_unet_single_blocks_17_linear2.alpha",
-    "lora_unet_single_blocks_17_linear2.lora_down.weight",
-    "lora_unet_single_blocks_17_linear2.lora_up.weight",
-    "lora_unet_single_blocks_17_modulation_lin.alpha",
-    "lora_unet_single_blocks_17_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_17_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_18_linear1.alpha",
-    "lora_unet_single_blocks_18_linear1.lora_down.weight",
-    "lora_unet_single_blocks_18_linear1.lora_up.weight",
-    "lora_unet_single_blocks_18_linear2.alpha",
-    "lora_unet_single_blocks_18_linear2.lora_down.weight",
-    "lora_unet_single_blocks_18_linear2.lora_up.weight",
-    "lora_unet_single_blocks_18_modulation_lin.alpha",
-    "lora_unet_single_blocks_18_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_18_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_19_linear1.alpha",
-    "lora_unet_single_blocks_19_linear1.lora_down.weight",
-    "lora_unet_single_blocks_19_linear1.lora_up.weight",
-    "lora_unet_single_blocks_19_linear2.alpha",
-    "lora_unet_single_blocks_19_linear2.lora_down.weight",
-    "lora_unet_single_blocks_19_linear2.lora_up.weight",
-    "lora_unet_single_blocks_19_modulation_lin.alpha",
-    "lora_unet_single_blocks_19_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_19_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_1_linear1.alpha",
-    "lora_unet_single_blocks_1_linear1.lora_down.weight",
-    "lora_unet_single_blocks_1_linear1.lora_up.weight",
-    "lora_unet_single_blocks_1_linear2.alpha",
-    "lora_unet_single_blocks_1_linear2.lora_down.weight",
-    "lora_unet_single_blocks_1_linear2.lora_up.weight",
-    "lora_unet_single_blocks_1_modulation_lin.alpha",
-    "lora_unet_single_blocks_1_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_1_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_20_linear1.alpha",
-    "lora_unet_single_blocks_20_linear1.lora_down.weight",
-    "lora_unet_single_blocks_20_linear1.lora_up.weight",
-    "lora_unet_single_blocks_20_linear2.alpha",
-    "lora_unet_single_blocks_20_linear2.lora_down.weight",
-    "lora_unet_single_blocks_20_linear2.lora_up.weight",
-    "lora_unet_single_blocks_20_modulation_lin.alpha",
-    "lora_unet_single_blocks_20_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_20_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_21_linear1.alpha",
-    "lora_unet_single_blocks_21_linear1.lora_down.weight",
-    "lora_unet_single_blocks_21_linear1.lora_up.weight",
-    "lora_unet_single_blocks_21_linear2.alpha",
-    "lora_unet_single_blocks_21_linear2.lora_down.weight",
-    "lora_unet_single_blocks_21_linear2.lora_up.weight",
-    "lora_unet_single_blocks_21_modulation_lin.alpha",
-    "lora_unet_single_blocks_21_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_21_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_22_linear1.alpha",
-    "lora_unet_single_blocks_22_linear1.lora_down.weight",
-    "lora_unet_single_blocks_22_linear1.lora_up.weight",
-    "lora_unet_single_blocks_22_linear2.alpha",
-    "lora_unet_single_blocks_22_linear2.lora_down.weight",
-    "lora_unet_single_blocks_22_linear2.lora_up.weight",
-    "lora_unet_single_blocks_22_modulation_lin.alpha",
-    "lora_unet_single_blocks_22_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_22_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_23_linear1.alpha",
-    "lora_unet_single_blocks_23_linear1.lora_down.weight",
-    "lora_unet_single_blocks_23_linear1.lora_up.weight",
-    "lora_unet_single_blocks_23_linear2.alpha",
-    "lora_unet_single_blocks_23_linear2.lora_down.weight",
-    "lora_unet_single_blocks_23_linear2.lora_up.weight",
-    "lora_unet_single_blocks_23_modulation_lin.alpha",
-    "lora_unet_single_blocks_23_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_23_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_24_linear1.alpha",
-    "lora_unet_single_blocks_24_linear1.lora_down.weight",
-    "lora_unet_single_blocks_24_linear1.lora_up.weight",
-    "lora_unet_single_blocks_24_linear2.alpha",
-    "lora_unet_single_blocks_24_linear2.lora_down.weight",
-    "lora_unet_single_blocks_24_linear2.lora_up.weight",
-    "lora_unet_single_blocks_24_modulation_lin.alpha",
-    "lora_unet_single_blocks_24_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_24_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_25_linear1.alpha",
-    "lora_unet_single_blocks_25_linear1.lora_down.weight",
-    "lora_unet_single_blocks_25_linear1.lora_up.weight",
-    "lora_unet_single_blocks_25_linear2.alpha",
-    "lora_unet_single_blocks_25_linear2.lora_down.weight",
-    "lora_unet_single_blocks_25_linear2.lora_up.weight",
-    "lora_unet_single_blocks_25_modulation_lin.alpha",
-    "lora_unet_single_blocks_25_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_25_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_26_linear1.alpha",
-    "lora_unet_single_blocks_26_linear1.lora_down.weight",
-    "lora_unet_single_blocks_26_linear1.lora_up.weight",
-    "lora_unet_single_blocks_26_linear2.alpha",
-    "lora_unet_single_blocks_26_linear2.lora_down.weight",
-    "lora_unet_single_blocks_26_linear2.lora_up.weight",
-    "lora_unet_single_blocks_26_modulation_lin.alpha",
-    "lora_unet_single_blocks_26_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_26_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_27_linear1.alpha",
-    "lora_unet_single_blocks_27_linear1.lora_down.weight",
-    "lora_unet_single_blocks_27_linear1.lora_up.weight",
-    "lora_unet_single_blocks_27_linear2.alpha",
-    "lora_unet_single_blocks_27_linear2.lora_down.weight",
-    "lora_unet_single_blocks_27_linear2.lora_up.weight",
-    "lora_unet_single_blocks_27_modulation_lin.alpha",
-    "lora_unet_single_blocks_27_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_27_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_28_linear1.alpha",
-    "lora_unet_single_blocks_28_linear1.lora_down.weight",
-    "lora_unet_single_blocks_28_linear1.lora_up.weight",
-    "lora_unet_single_blocks_28_linear2.alpha",
-    "lora_unet_single_blocks_28_linear2.lora_down.weight",
-    "lora_unet_single_blocks_28_linear2.lora_up.weight",
-    "lora_unet_single_blocks_28_modulation_lin.alpha",
-    "lora_unet_single_blocks_28_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_28_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_29_linear1.alpha",
-    "lora_unet_single_blocks_29_linear1.lora_down.weight",
-    "lora_unet_single_blocks_29_linear1.lora_up.weight",
-    "lora_unet_single_blocks_29_linear2.alpha",
-    "lora_unet_single_blocks_29_linear2.lora_down.weight",
-    "lora_unet_single_blocks_29_linear2.lora_up.weight",
-    "lora_unet_single_blocks_29_modulation_lin.alpha",
-    "lora_unet_single_blocks_29_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_29_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_2_linear1.alpha",
-    "lora_unet_single_blocks_2_linear1.lora_down.weight",
-    "lora_unet_single_blocks_2_linear1.lora_up.weight",
-    "lora_unet_single_blocks_2_linear2.alpha",
-    "lora_unet_single_blocks_2_linear2.lora_down.weight",
-    "lora_unet_single_blocks_2_linear2.lora_up.weight",
-    "lora_unet_single_blocks_2_modulation_lin.alpha",
-    "lora_unet_single_blocks_2_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_2_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_30_linear1.alpha",
-    "lora_unet_single_blocks_30_linear1.lora_down.weight",
-    "lora_unet_single_blocks_30_linear1.lora_up.weight",
-    "lora_unet_single_blocks_30_linear2.alpha",
-    "lora_unet_single_blocks_30_linear2.lora_down.weight",
-    "lora_unet_single_blocks_30_linear2.lora_up.weight",
-    "lora_unet_single_blocks_30_modulation_lin.alpha",
-    "lora_unet_single_blocks_30_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_30_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_31_linear1.alpha",
-    "lora_unet_single_blocks_31_linear1.lora_down.weight",
-    "lora_unet_single_blocks_31_linear1.lora_up.weight",
-    "lora_unet_single_blocks_31_linear2.alpha",
-    "lora_unet_single_blocks_31_linear2.lora_down.weight",
-    "lora_unet_single_blocks_31_linear2.lora_up.weight",
-    "lora_unet_single_blocks_31_modulation_lin.alpha",
-    "lora_unet_single_blocks_31_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_31_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_32_linear1.alpha",
-    "lora_unet_single_blocks_32_linear1.lora_down.weight",
-    "lora_unet_single_blocks_32_linear1.lora_up.weight",
-    "lora_unet_single_blocks_32_linear2.alpha",
-    "lora_unet_single_blocks_32_linear2.lora_down.weight",
-    "lora_unet_single_blocks_32_linear2.lora_up.weight",
-    "lora_unet_single_blocks_32_modulation_lin.alpha",
-    "lora_unet_single_blocks_32_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_32_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_33_linear1.alpha",
-    "lora_unet_single_blocks_33_linear1.lora_down.weight",
-    "lora_unet_single_blocks_33_linear1.lora_up.weight",
-    "lora_unet_single_blocks_33_linear2.alpha",
-    "lora_unet_single_blocks_33_linear2.lora_down.weight",
-    "lora_unet_single_blocks_33_linear2.lora_up.weight",
-    "lora_unet_single_blocks_33_modulation_lin.alpha",
-    "lora_unet_single_blocks_33_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_33_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_34_linear1.alpha",
-    "lora_unet_single_blocks_34_linear1.lora_down.weight",
-    "lora_unet_single_blocks_34_linear1.lora_up.weight",
-    "lora_unet_single_blocks_34_linear2.alpha",
-    "lora_unet_single_blocks_34_linear2.lora_down.weight",
-    "lora_unet_single_blocks_34_linear2.lora_up.weight",
-    "lora_unet_single_blocks_34_modulation_lin.alpha",
-    "lora_unet_single_blocks_34_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_34_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_35_linear1.alpha",
-    "lora_unet_single_blocks_35_linear1.lora_down.weight",
-    "lora_unet_single_blocks_35_linear1.lora_up.weight",
-    "lora_unet_single_blocks_35_linear2.alpha",
-    "lora_unet_single_blocks_35_linear2.lora_down.weight",
-    "lora_unet_single_blocks_35_linear2.lora_up.weight",
-    "lora_unet_single_blocks_35_modulation_lin.alpha",
-    "lora_unet_single_blocks_35_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_35_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_36_linear1.alpha",
-    "lora_unet_single_blocks_36_linear1.lora_down.weight",
-    "lora_unet_single_blocks_36_linear1.lora_up.weight",
-    "lora_unet_single_blocks_36_linear2.alpha",
-    "lora_unet_single_blocks_36_linear2.lora_down.weight",
-    "lora_unet_single_blocks_36_linear2.lora_up.weight",
-    "lora_unet_single_blocks_36_modulation_lin.alpha",
-    "lora_unet_single_blocks_36_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_36_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_37_linear1.alpha",
-    "lora_unet_single_blocks_37_linear1.lora_down.weight",
-    "lora_unet_single_blocks_37_linear1.lora_up.weight",
-    "lora_unet_single_blocks_37_linear2.alpha",
-    "lora_unet_single_blocks_37_linear2.lora_down.weight",
-    "lora_unet_single_blocks_37_linear2.lora_up.weight",
-    "lora_unet_single_blocks_37_modulation_lin.alpha",
-    "lora_unet_single_blocks_37_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_37_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_3_linear1.alpha",
-    "lora_unet_single_blocks_3_linear1.lora_down.weight",
-    "lora_unet_single_blocks_3_linear1.lora_up.weight",
-    "lora_unet_single_blocks_3_linear2.alpha",
-    "lora_unet_single_blocks_3_linear2.lora_down.weight",
-    "lora_unet_single_blocks_3_linear2.lora_up.weight",
-    "lora_unet_single_blocks_3_modulation_lin.alpha",
-    "lora_unet_single_blocks_3_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_3_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_4_linear1.alpha",
-    "lora_unet_single_blocks_4_linear1.lora_down.weight",
-    "lora_unet_single_blocks_4_linear1.lora_up.weight",
-    "lora_unet_single_blocks_4_linear2.alpha",
-    "lora_unet_single_blocks_4_linear2.lora_down.weight",
-    "lora_unet_single_blocks_4_linear2.lora_up.weight",
-    "lora_unet_single_blocks_4_modulation_lin.alpha",
-    "lora_unet_single_blocks_4_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_4_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_5_linear1.alpha",
-    "lora_unet_single_blocks_5_linear1.lora_down.weight",
-    "lora_unet_single_blocks_5_linear1.lora_up.weight",
-    "lora_unet_single_blocks_5_linear2.alpha",
-    "lora_unet_single_blocks_5_linear2.lora_down.weight",
-    "lora_unet_single_blocks_5_linear2.lora_up.weight",
-    "lora_unet_single_blocks_5_modulation_lin.alpha",
-    "lora_unet_single_blocks_5_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_5_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_6_linear1.alpha",
-    "lora_unet_single_blocks_6_linear1.lora_down.weight",
-    "lora_unet_single_blocks_6_linear1.lora_up.weight",
-    "lora_unet_single_blocks_6_linear2.alpha",
-    "lora_unet_single_blocks_6_linear2.lora_down.weight",
-    "lora_unet_single_blocks_6_linear2.lora_up.weight",
-    "lora_unet_single_blocks_6_modulation_lin.alpha",
-    "lora_unet_single_blocks_6_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_6_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_7_linear1.alpha",
-    "lora_unet_single_blocks_7_linear1.lora_down.weight",
-    "lora_unet_single_blocks_7_linear1.lora_up.weight",
-    "lora_unet_single_blocks_7_linear2.alpha",
-    "lora_unet_single_blocks_7_linear2.lora_down.weight",
-    "lora_unet_single_blocks_7_linear2.lora_up.weight",
-    "lora_unet_single_blocks_7_modulation_lin.alpha",
-    "lora_unet_single_blocks_7_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_7_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_8_linear1.alpha",
-    "lora_unet_single_blocks_8_linear1.lora_down.weight",
-    "lora_unet_single_blocks_8_linear1.lora_up.weight",
-    "lora_unet_single_blocks_8_linear2.alpha",
-    "lora_unet_single_blocks_8_linear2.lora_down.weight",
-    "lora_unet_single_blocks_8_linear2.lora_up.weight",
-    "lora_unet_single_blocks_8_modulation_lin.alpha",
-    "lora_unet_single_blocks_8_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_8_modulation_lin.lora_up.weight",
-    "lora_unet_single_blocks_9_linear1.alpha",
-    "lora_unet_single_blocks_9_linear1.lora_down.weight",
-    "lora_unet_single_blocks_9_linear1.lora_up.weight",
-    "lora_unet_single_blocks_9_linear2.alpha",
-    "lora_unet_single_blocks_9_linear2.lora_down.weight",
-    "lora_unet_single_blocks_9_linear2.lora_up.weight",
-    "lora_unet_single_blocks_9_modulation_lin.alpha",
-    "lora_unet_single_blocks_9_modulation_lin.lora_down.weight",
-    "lora_unet_single_blocks_9_modulation_lin.lora_up.weight",
-]

tests/backend/peft/conversions/test_flux_lora_conversion_utils.py

@@ -1,97 +0,0 @@
-import pytest
-import torch
-
-from invokeai.backend.flux.model import Flux
-from invokeai.backend.flux.util import params
-from invokeai.backend.peft.conversions.flux_kohya_lora_conversion_utils import (
-    convert_flux_kohya_state_dict_to_invoke_format,
-    is_state_dict_likely_in_flux_kohya_format,
-    lora_model_from_flux_kohya_state_dict,
-)
-from tests.backend.peft.conversions.lora_state_dicts.flux_lora_kohya_format import state_dict_keys
-
-
-def test_is_state_dict_likely_in_flux_kohya_format_true():
-    """Test that is_state_dict_likely_in_flux_kohya_format() can identify a state dict in the Kohya FLUX LoRA format."""
-    # Construct a state dict that is in the Kohya FLUX LoRA format.
-    state_dict: dict[str, torch.Tensor] = {}
-    for k in state_dict_keys:
-        state_dict[k] = torch.empty(1)
-    assert is_state_dict_likely_in_flux_kohya_format(state_dict)
-
-
-def test_is_state_dict_likely_in_flux_kohya_format_false():
-    """Test that is_state_dict_likely_in_flux_kohya_format() returns False for a state dict that is not in the Kohya FLUX LoRA format."""
-    state_dict: dict[str, torch.Tensor] = {
-        "unexpected_key.lora_up.weight": torch.empty(1),
-    }
-    assert not is_state_dict_likely_in_flux_kohya_format(state_dict)
-
-
-def test_convert_flux_kohya_state_dict_to_invoke_format():
-    # Construct state_dict from state_dict_keys.
-    state_dict: dict[str, torch.Tensor] = {}
-    for k in state_dict_keys:
-        state_dict[k] = torch.empty(1)
-
-    converted_state_dict = convert_flux_kohya_state_dict_to_invoke_format(state_dict)
-
-    # Extract the prefixes from the converted state dict (i.e. without the .lora_up.weight, .lora_down.weight, and
-    # .alpha suffixes).
-    converted_key_prefixes: list[str] = []
-    for k in converted_state_dict.keys():
-        k = k.replace(".lora_up.weight", "")
-        k = k.replace(".lora_down.weight", "")
-        k = k.replace(".alpha", "")
-        converted_key_prefixes.append(k)
-
-    # Initialize a FLUX model on the meta device.
-    with torch.device("meta"):
-        model = Flux(params["flux-dev"])
-    model_keys = set(model.state_dict().keys())
-
-    # Assert that the converted state dict matches the keys in the actual model.
-    for converted_key_prefix in converted_key_prefixes:
-        found_match = False
-        for model_key in model_keys:
-            if model_key.startswith(converted_key_prefix):
-                found_match = True
-                break
-        if not found_match:
-            raise AssertionError(f"Could not find a match for the converted key prefix: {converted_key_prefix}")
-
-
-def test_convert_flux_kohya_state_dict_to_invoke_format_error():
-    """Test that an error is raised by convert_flux_kohya_state_dict_to_invoke_format() if the input state_dict contains
-    unexpected keys.
-    """
-    state_dict = {
-        "unexpected_key.lora_up.weight": torch.empty(1),
-    }
-
-    with pytest.raises(ValueError):
-        convert_flux_kohya_state_dict_to_invoke_format(state_dict)
-
-
-def test_lora_model_from_flux_kohya_state_dict():
-    """Test that a LoRAModelRaw can be created from a state dict in the Kohya FLUX LoRA format."""
-    # Construct state_dict from state_dict_keys.
-    state_dict: dict[str, torch.Tensor] = {}
-    for k in state_dict_keys:
-        state_dict[k] = torch.empty(1)
-
-    lora_model = lora_model_from_flux_kohya_state_dict(state_dict)
-
-    # Prepare expected layer keys.
-    expected_layer_keys: set[str] = set()
-    for k in state_dict_keys:
-        k = k.replace("lora_unet_", "")
-        k = k.replace(".lora_up.weight", "")
-        k = k.replace(".lora_down.weight", "")
-        k = k.replace(".alpha", "")
-        expected_layer_keys.add(k)
-
-    # Assert that the lora_model has the expected layers.
-    lora_model_keys = set(lora_model.layers.keys())
-    lora_model_keys = {k.replace(".", "_") for k in lora_model_keys}
-    assert lora_model_keys == expected_layer_keys