experiment: fix types

This provides useful context for subsequent logs during queue item execution.

.github/workflows/typegen-checks.yml

@@ -0,0 +1,85 @@
+# Runs typegen schema quality checks.
+# Frontend types should match the server.
+#
+# Checks for changes to files before running the checks.
+# If always_run is true, always runs the checks.
+
+name: 'typegen checks'
+
+on:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+    types:
+      - 'ready_for_review'
+      - 'opened'
+      - 'synchronize'
+  merge_group:
+  workflow_dispatch:
+    inputs:
+      always_run:
+        description: 'Always run the checks'
+        required: true
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      always_run:
+        description: 'Always run the checks'
+        required: true
+        type: boolean
+        default: true
+
+jobs:
+  typegen-checks:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15 # expected run time: <5 min
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: check for changed files
+        if: ${{ inputs.always_run != true }}
+        id: changed-files
+        uses: tj-actions/changed-files@v42
+        with:
+          files_yaml: |
+            src:
+              - 'pyproject.toml'
+              - 'invokeai/**'
+
+      - name: setup python
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: install python dependencies
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        run: pip3 install --use-pep517 --editable="."
+
+      - name: install frontend dependencies
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        uses: ./.github/actions/install-frontend-deps
+
+      - name: copy schema
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        run: cp invokeai/frontend/web/src/services/api/schema.ts invokeai/frontend/web/src/services/api/schema_orig.ts
+        shell: bash
+
+      - name: generate schema
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        run: make frontend-typegen
+        shell: bash
+
+      - name: compare files
+        if: ${{ steps.changed-files.outputs.src_any_changed == 'true' || inputs.always_run == true }}
+        run: |
+          if ! diff invokeai/frontend/web/src/services/api/schema.ts invokeai/frontend/web/src/services/api/schema_orig.ts; then
+            echo "Files are different!";
+            exit 1;
+          fi
+        shell: bash

docker/Dockerfile

@@ -2,29 +2,42 @@
 
 ## Builder stage
 
-FROM library/ubuntu:23.04 AS builder
+FROM library/ubuntu:24.04 AS builder
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     apt update && apt-get install -y \
-        git \
-        python3-venv \
-        python3-pip \
-        build-essential
+        build-essential \
+        git
 
-ENV INVOKEAI_SRC=/opt/invokeai
-ENV VIRTUAL_ENV=/opt/venv/invokeai
+# Install `uv` for package management
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
 
+ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV INVOKEAI_SRC=/opt/invokeai
+ENV PYTHON_VERSION=3.11
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
+
 ARG GPU_DRIVER=cuda
 ARG TARGETPLATFORM="linux/amd64"
 # unused but available
 ARG BUILDPLATFORM
 
-WORKDIR ${INVOKEAI_SRC}
+# Switch to the `ubuntu` user to work around dependency issues with uv-installed python
+RUN mkdir -p ${VIRTUAL_ENV} && \
+    mkdir -p ${INVOKEAI_SRC} && \
+    chmod -R a+w /opt
+USER ubuntu
 
+# Install python and create the venv
+RUN uv python install ${PYTHON_VERSION} && \
+    uv venv --relocatable --prompt "invoke" --python ${PYTHON_VERSION} ${VIRTUAL_ENV}
+
+WORKDIR ${INVOKEAI_SRC}
 COPY invokeai ./invokeai
 COPY pyproject.toml ./
 
@@ -32,25 +45,18 @@ COPY pyproject.toml ./
 # the local working copy can be bind-mounted into the image
 # at path defined by ${INVOKEAI_SRC}
 # NOTE: there are no pytorch builds for arm64 + cuda, only cpu
-# x86_64/CUDA is default
-RUN --mount=type=cache,target=/root/.cache/pip \
-    python3 -m venv ${VIRTUAL_ENV} &&\
+# x86_64/CUDA is the default
+RUN --mount=type=cache,target=/home/ubuntu/.cache/uv,uid=1000,gid=1000 \
     if [ "$TARGETPLATFORM" = "linux/arm64" ] || [ "$GPU_DRIVER" = "cpu" ]; then \
         extra_index_url_arg="--extra-index-url https://download.pytorch.org/whl/cpu"; \
     elif [ "$GPU_DRIVER" = "rocm" ]; then \
         extra_index_url_arg="--extra-index-url https://download.pytorch.org/whl/rocm6.1"; \
     else \
         extra_index_url_arg="--extra-index-url https://download.pytorch.org/whl/cu124"; \
-    fi &&\
+    fi && \
+    uv pip install --python ${PYTHON_VERSION} $extra_index_url_arg -e "."
 
-    # xformers + triton fails to install on arm64
-    if [ "$GPU_DRIVER" = "cuda" ] && [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-        pip install $extra_index_url_arg -e ".[xformers]"; \
-    else \
-        pip install $extra_index_url_arg -e "."; \
-    fi
-
-# #### Build the Web UI ------------------------------------
+#### Build the Web UI ------------------------------------
 
 FROM node:20-slim AS web-builder
 ENV PNPM_HOME="/pnpm"
@@ -66,7 +72,7 @@ RUN npx vite build
 
 #### Runtime stage ---------------------------------------
 
-FROM library/ubuntu:23.04 AS runtime
+FROM library/ubuntu:24.04 AS runtime
 
 ARG DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED=1
@@ -83,17 +89,16 @@ RUN apt update && apt install -y --no-install-recommends \
         gosu \
         magic-wormhole \
         libglib2.0-0 \
-        libgl1-mesa-glx \
-        python3-venv \
-        python3-pip \
+        libgl1 \
+        libglx-mesa0 \
         build-essential \
         libopencv-dev \
         libstdc++-10-dev &&\
     apt-get clean && apt-get autoclean
 
-
 ENV INVOKEAI_SRC=/opt/invokeai
-ENV VIRTUAL_ENV=/opt/venv/invokeai
+ENV VIRTUAL_ENV=/opt/venv
+ENV PYTHON_VERSION=3.11
 ENV INVOKEAI_ROOT=/invokeai
 ENV INVOKEAI_HOST=0.0.0.0
 ENV INVOKEAI_PORT=9090
@@ -101,6 +106,14 @@ ENV PATH="$VIRTUAL_ENV/bin:$INVOKEAI_SRC:$PATH"
 ENV CONTAINER_UID=${CONTAINER_UID:-1000}
 ENV CONTAINER_GID=${CONTAINER_GID:-1000}
 
+# Install `uv` for package management
+# and install python for the ubuntu user (expected to exist on ubuntu >=24.x)
+# this is too tiny to optimize with multi-stage builds, but maybe we'll come back to it
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
+USER ubuntu
+RUN uv python install ${PYTHON_VERSION}
+USER root
+
 # --link requires buldkit w/ dockerfile syntax 1.4
 COPY --link --from=builder ${INVOKEAI_SRC} ${INVOKEAI_SRC}
 COPY --link --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
@@ -115,7 +128,7 @@ WORKDIR ${INVOKEAI_SRC}
 
 # build patchmatch
 RUN cd /usr/lib/$(uname -p)-linux-gnu/pkgconfig/ && ln -sf opencv4.pc opencv.pc
-RUN python3 -c "from patchmatch import patch_match"
+RUN python -c "from patchmatch import patch_match"
 
 RUN mkdir -p ${INVOKEAI_ROOT} && chown -R ${CONTAINER_UID}:${CONTAINER_GID} ${INVOKEAI_ROOT}
 

docker/docker-entrypoint.sh

@@ -16,6 +16,9 @@ set -e -o pipefail
 
 USER_ID=${CONTAINER_UID:-1000}
 USER=ubuntu
+# if the user does not exist, create it. It is expected to be present on ubuntu >=24.x
+_=$(id ${USER} 2>&1) || useradd -u ${USER_ID} ${USER}
+# ensure the UID is correct
 usermod -u ${USER_ID} ${USER} 1>/dev/null
 
 ### Set the $PUBLIC_KEY env var to enable SSH access.
@@ -36,6 +39,8 @@ fi
 mkdir -p "${INVOKEAI_ROOT}"
 chown --recursive ${USER} "${INVOKEAI_ROOT}" || true
 cd "${INVOKEAI_ROOT}"
+export HF_HOME=${HF_HOME:-$INVOKEAI_ROOT/.cache/huggingface}
+export MPLCONFIGDIR=${MPLCONFIGDIR:-$INVOKEAI_ROOT/.matplotlib}
 
 # Run the CMD as the Container User (not root).
 exec gosu ${USER} "$@"

invokeai/app/api_app.py

@@ -59,11 +59,32 @@ logger.info(f"Using torch device: {torch_device_name}")
 
 loop = asyncio.new_event_loop()
 
+# We may change the port if the default is in use, this global variable is used to store the port so that we can log
+# the correct port when the server starts in the lifespan handler.
+port = app_config.port
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # Add startup event to load dependencies
     ApiDependencies.initialize(config=app_config, event_handler_id=event_handler_id, loop=loop, logger=logger)
+
+    # Log the server address when it starts - in case the network log level is not high enough to see the startup log
+    proto = "https" if app_config.ssl_certfile else "http"
+    msg = f"Invoke running on {proto}://{app_config.host}:{port} (Press CTRL+C to quit)"
+
+    # Logging this way ignores the logger's log level and _always_ logs the message
+    record = logger.makeRecord(
+        name=logger.name,
+        level=logging.INFO,
+        fn="",
+        lno=0,
+        msg=msg,
+        args=(),
+        exc_info=None,
+    )
+    logger.handle(record)
+
     yield
     # Shut down threads
     ApiDependencies.shutdown()
@@ -206,6 +227,7 @@ def invoke_api() -> None:
         else:
             jurigged.watch(logger=InvokeAILogger.get_logger(name="jurigged").info)
 
+    global port
     port = find_port(app_config.port)
     if port != app_config.port:
         logger.warn(f"Port {app_config.port} in use, using port {port}")
@@ -217,18 +239,17 @@ def invoke_api() -> None:
         host=app_config.host,
         port=port,
         loop="asyncio",
-        log_level=app_config.log_level,
+        log_level=app_config.log_level_network,
         ssl_certfile=app_config.ssl_certfile,
         ssl_keyfile=app_config.ssl_keyfile,
     )
     server = uvicorn.Server(config)
 
     # replace uvicorn's loggers with InvokeAI's for consistent appearance
-    for logname in ["uvicorn.access", "uvicorn"]:
-        log = InvokeAILogger.get_logger(logname)
-        log.handlers.clear()
-        for ch in logger.handlers:
-            log.addHandler(ch)
+    uvicorn_logger = InvokeAILogger.get_logger("uvicorn")
+    uvicorn_logger.handlers.clear()
+    for hdlr in logger.handlers:
+        uvicorn_logger.addHandler(hdlr)
 
     loop.run_until_complete(server.serve())
 

invokeai/app/invocations/__init__.py

@@ -15,6 +15,11 @@ custom_nodes_readme_path = str(custom_nodes_path / "README.md")
 shutil.copy(Path(__file__).parent / "custom_nodes/init.py", custom_nodes_init_path)
 shutil.copy(Path(__file__).parent / "custom_nodes/README.md", custom_nodes_readme_path)
 
+# set the same permissions as the destination directory, in case our source is read-only,
+# so that the files are user-writable
+for p in custom_nodes_path.glob("**/*"):
+    p.chmod(custom_nodes_path.stat().st_mode)
+
 # Import custom nodes, see https://docs.python.org/3/library/importlib.html#importing-programmatically
 spec = spec_from_file_location("custom_nodes", custom_nodes_init_path)
 if spec is None or spec.loader is None:

invokeai/app/invocations/compel.py

@@ -19,9 +19,9 @@ from invokeai.app.invocations.model import CLIPField
 from invokeai.app.invocations.primitives import ConditioningOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.util.ti_utils import generate_ti_list
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
 from invokeai.backend.model_patcher import ModelPatcher
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
     BasicConditioningInfo,
     ConditioningFieldData,
@@ -66,10 +66,10 @@ class CompelInvocation(BaseInvocation):
         tokenizer_info = context.models.load(self.clip.tokenizer)
         text_encoder_info = context.models.load(self.clip.text_encoder)
 
-        def _lora_loader() -> Iterator[Tuple[LoRAModelRaw, float]]:
+        def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
             for lora in self.clip.loras:
                 lora_info = context.models.load(lora.lora)
-                assert isinstance(lora_info.model, LoRAModelRaw)
+                assert isinstance(lora_info.model, ModelPatchRaw)
                 yield (lora_info.model, lora.weight)
                 del lora_info
             return
@@ -82,7 +82,7 @@ class CompelInvocation(BaseInvocation):
             # apply all patches while the model is on the target device
             text_encoder_info.model_on_device() as (cached_weights, text_encoder),
             tokenizer_info as tokenizer,
-            LoRAPatcher.apply_lora_patches(
+            LayerPatcher.apply_model_patches(
                 model=text_encoder,
                 patches=_lora_loader(),
                 prefix="lora_te_",
@@ -162,11 +162,11 @@ class SDXLPromptInvocationBase:
                 c_pooled = None
             return c, c_pooled
 
-        def _lora_loader() -> Iterator[Tuple[LoRAModelRaw, float]]:
+        def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
             for lora in clip_field.loras:
                 lora_info = context.models.load(lora.lora)
                 lora_model = lora_info.model
-                assert isinstance(lora_model, LoRAModelRaw)
+                assert isinstance(lora_model, ModelPatchRaw)
                 yield (lora_model, lora.weight)
                 del lora_info
             return
@@ -179,7 +179,7 @@ class SDXLPromptInvocationBase:
             # apply all patches while the model is on the target device
             text_encoder_info.model_on_device() as (cached_weights, text_encoder),
             tokenizer_info as tokenizer,
-            LoRAPatcher.apply_lora_patches(
+            LayerPatcher.apply_model_patches(
                 text_encoder,
                 patches=_lora_loader(),
                 prefix=lora_prefix,

invokeai/app/invocations/create_denoise_mask.py

@@ -6,7 +6,6 @@ from PIL import Image
 from torchvision.transforms.functional import resize as tv_resize
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
-from invokeai.app.invocations.constants import DEFAULT_PRECISION
 from invokeai.app.invocations.fields import FieldDescriptions, ImageField, Input, InputField
 from invokeai.app.invocations.image_to_latents import ImageToLatentsInvocation
 from invokeai.app.invocations.model import VAEField
@@ -29,11 +28,7 @@ class CreateDenoiseMaskInvocation(BaseInvocation):
     image: Optional[ImageField] = InputField(default=None, description="Image which will be masked", ui_order=1)
     mask: ImageField = InputField(description="The mask to use when pasting", ui_order=2)
     tiled: bool = InputField(default=False, description=FieldDescriptions.tiled, ui_order=3)
-    fp32: bool = InputField(
-        default=DEFAULT_PRECISION == torch.float32,
-        description=FieldDescriptions.fp32,
-        ui_order=4,
-    )
+    fp32: bool = InputField(default=False, description=FieldDescriptions.fp32, ui_order=4)
 
     def prep_mask_tensor(self, mask_image: Image.Image) -> torch.Tensor:
         if mask_image.mode != "L":

invokeai/app/invocations/create_gradient_mask.py

@@ -7,7 +7,6 @@ from PIL import Image, ImageFilter
 from torchvision.transforms.functional import resize as tv_resize
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output
-from invokeai.app.invocations.constants import DEFAULT_PRECISION
 from invokeai.app.invocations.fields import (
     DenoiseMaskField,
     FieldDescriptions,
@@ -76,11 +75,7 @@ class CreateGradientMaskInvocation(BaseInvocation):
         ui_order=7,
     )
     tiled: bool = InputField(default=False, description=FieldDescriptions.tiled, ui_order=8)
-    fp32: bool = InputField(
-        default=DEFAULT_PRECISION == torch.float32,
-        description=FieldDescriptions.fp32,
-        ui_order=9,
-    )
+    fp32: bool = InputField(default=False, description=FieldDescriptions.fp32, ui_order=9)
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> GradientMaskOutput:

invokeai/app/invocations/denoise_latents.py

@@ -37,10 +37,10 @@ from invokeai.app.invocations.t2i_adapter import T2IAdapterField
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.util.controlnet_utils import prepare_control_image
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
 from invokeai.backend.model_manager import BaseModelType, ModelVariantType
 from invokeai.backend.model_patcher import ModelPatcher
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion import PipelineIntermediateState
 from invokeai.backend.stable_diffusion.denoise_context import DenoiseContext, DenoiseInputs
 from invokeai.backend.stable_diffusion.diffusers_pipeline import (
@@ -987,10 +987,10 @@ class DenoiseLatentsInvocation(BaseInvocation):
         def step_callback(state: PipelineIntermediateState) -> None:
             context.util.sd_step_callback(state, unet_config.base)
 
-        def _lora_loader() -> Iterator[Tuple[LoRAModelRaw, float]]:
+        def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
             for lora in self.unet.loras:
                 lora_info = context.models.load(lora.lora)
-                assert isinstance(lora_info.model, LoRAModelRaw)
+                assert isinstance(lora_info.model, ModelPatchRaw)
                 yield (lora_info.model, lora.weight)
                 del lora_info
             return
@@ -1003,7 +1003,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
             ModelPatcher.apply_freeu(unet, self.unet.freeu_config),
             SeamlessExt.static_patch_model(unet, self.unet.seamless_axes),  # FIXME
             # Apply the LoRA after unet has been moved to its target device for faster patching.
-            LoRAPatcher.apply_lora_patches(
+            LayerPatcher.apply_model_patches(
                 model=unet,
                 patches=_lora_loader(),
                 prefix="lora_unet_",

invokeai/app/invocations/fields.py

@@ -56,6 +56,7 @@ class UIType(str, Enum, metaclass=MetaEnum):
     CLIPLEmbedModel = "CLIPLEmbedModelField"
     CLIPGEmbedModel = "CLIPGEmbedModelField"
     SpandrelImageToImageModel = "SpandrelImageToImageModelField"
+    ControlLoRAModel = "ControlLoRAModelField"
     # endregion
 
     # region Misc Field Types
@@ -143,6 +144,7 @@ class FieldDescriptions:
     controlnet_model = "ControlNet model to load"
     vae_model = "VAE model to load"
     lora_model = "LoRA model to load"
+    control_lora_model = "Control LoRA model to load"
     main_model = "Main model (UNet, VAE, CLIP) to load"
     flux_model = "Flux model (Transformer) to load"
     sd3_model = "SD3 model (MMDiTX) to load"

invokeai/app/invocations/flux_control_lora_loader.py

@@ -0,0 +1,49 @@
+from invokeai.app.invocations.baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    Classification,
+    invocation,
+    invocation_output,
+)
+from invokeai.app.invocations.fields import FieldDescriptions, ImageField, InputField, OutputField, UIType
+from invokeai.app.invocations.model import ControlLoRAField, ModelIdentifierField
+from invokeai.app.services.shared.invocation_context import InvocationContext
+
+
+@invocation_output("flux_control_lora_loader_output")
+class FluxControlLoRALoaderOutput(BaseInvocationOutput):
+    """Flux Control LoRA Loader Output"""
+
+    control_lora: ControlLoRAField = OutputField(
+        title="Flux Control LoRA", description="Control LoRAs to apply on model loading", default=None
+    )
+
+
+@invocation(
+    "flux_control_lora_loader",
+    title="Flux Control LoRA",
+    tags=["lora", "model", "flux"],
+    category="model",
+    version="1.1.0",
+    classification=Classification.Prototype,
+)
+class FluxControlLoRALoaderInvocation(BaseInvocation):
+    """LoRA model and Image to use with FLUX transformer generation."""
+
+    lora: ModelIdentifierField = InputField(
+        description=FieldDescriptions.control_lora_model, title="Control LoRA", ui_type=UIType.ControlLoRAModel
+    )
+    image: ImageField = InputField(description="The image to encode.")
+    weight: float = InputField(description="The weight of the LoRA.", default=1.0)
+
+    def invoke(self, context: InvocationContext) -> FluxControlLoRALoaderOutput:
+        if not context.models.exists(self.lora.key):
+            raise ValueError(f"Unknown lora: {self.lora.key}!")
+
+        return FluxControlLoRALoaderOutput(
+            control_lora=ControlLoRAField(
+                lora=self.lora,
+                img=self.image,
+                weight=self.weight,
+            )
+        )

invokeai/app/invocations/flux_denoise.py

@@ -1,10 +1,12 @@
 from contextlib import ExitStack
-from typing import Callable, Iterator, Optional, Tuple
+from typing import Callable, Iterator, Optional, Tuple, Union
 
+import einops
 import numpy as np
 import numpy.typing as npt
 import torch
 import torchvision.transforms as tv_transforms
+from PIL import Image
 from torchvision.transforms.functional import resize as tv_resize
 from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
 
@@ -21,8 +23,9 @@ from invokeai.app.invocations.fields import (
     WithMetadata,
 )
 from invokeai.app.invocations.flux_controlnet import FluxControlNetField
+from invokeai.app.invocations.flux_vae_encode import FluxVaeEncodeInvocation
 from invokeai.app.invocations.ip_adapter import IPAdapterField
-from invokeai.app.invocations.model import TransformerField, VAEField
+from invokeai.app.invocations.model import ControlLoRAField, LoRAField, TransformerField, VAEField
 from invokeai.app.invocations.primitives import LatentsOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.flux.controlnet.instantx_controlnet_flux import InstantXControlNetFlux
@@ -44,10 +47,10 @@ from invokeai.backend.flux.sampling_utils import (
     unpack,
 )
 from invokeai.backend.flux.text_conditioning import FluxTextConditioning
-from invokeai.backend.lora.conversions.flux_lora_constants import FLUX_LORA_TRANSFORMER_PREFIX
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
 from invokeai.backend.model_manager.config import ModelFormat
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_TRANSFORMER_PREFIX
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import FLUXConditioningInfo
 from invokeai.backend.util.devices import TorchDevice
@@ -89,6 +92,9 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         input=Input.Connection,
         title="Transformer",
     )
+    control_lora: Optional[ControlLoRAField] = InputField(
+        description=FieldDescriptions.control_lora_model, input=Input.Connection, title="Control LoRA", default=None
+    )
     positive_text_conditioning: FluxConditioningField | list[FluxConditioningField] = InputField(
         description=FieldDescriptions.positive_cond, input=Input.Connection
     )
@@ -194,7 +200,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         )
 
         transformer_info = context.models.load(self.transformer.transformer)
-        is_schnell = "schnell" in transformer_info.config.config_path
+        is_schnell = "schnell" in getattr(transformer_info.config, "config_path", "")
 
         # Calculate the timestep schedule.
         timesteps = get_schedule(
@@ -234,6 +240,12 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         if len(timesteps) <= 1:
             return x
 
+        if is_schnell and self.control_lora:
+            raise ValueError("Control LoRAs cannot be used with FLUX Schnell")
+
+        # Prepare the extra image conditioning tensor if a FLUX structural control image is provided.
+        img_cond = self._prep_structural_control_img_cond(context)
+
         inpaint_mask = self._prep_inpaint_mask(context, x)
 
         img_ids = generate_img_ids(h=latent_h, w=latent_w, batch_size=b, device=x.device, dtype=x.dtype)
@@ -241,6 +253,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         # Pack all latent tensors.
         init_latents = pack(init_latents) if init_latents is not None else None
         inpaint_mask = pack(inpaint_mask) if inpaint_mask is not None else None
+        img_cond = pack(img_cond) if img_cond is not None else None
         noise = pack(noise)
         x = pack(x)
 
@@ -296,7 +309,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
             if config.format in [ModelFormat.Checkpoint]:
                 # The model is non-quantized, so we can apply the LoRA weights directly into the model.
                 exit_stack.enter_context(
-                    LoRAPatcher.apply_lora_patches(
+                    LayerPatcher.apply_model_patches(
                         model=transformer,
                         patches=self._lora_iterator(context),
                         prefix=FLUX_LORA_TRANSFORMER_PREFIX,
@@ -311,7 +324,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
                 # The model is quantized, so apply the LoRA weights as sidecar layers. This results in slower inference,
                 # than directly patching the weights, but is agnostic to the quantization format.
                 exit_stack.enter_context(
-                    LoRAPatcher.apply_lora_sidecar_patches(
+                    LayerPatcher.apply_model_sidecar_patches(
                         model=transformer,
                         patches=self._lora_iterator(context),
                         prefix=FLUX_LORA_TRANSFORMER_PREFIX,
@@ -345,6 +358,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
                 controlnet_extensions=controlnet_extensions,
                 pos_ip_adapter_extensions=pos_ip_adapter_extensions,
                 neg_ip_adapter_extensions=neg_ip_adapter_extensions,
+                img_cond=img_cond,
             )
 
         x = unpack(x.float(), self.height, self.width)
@@ -575,6 +589,29 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
 
         return controlnet_extensions
 
+    def _prep_structural_control_img_cond(self, context: InvocationContext) -> torch.Tensor | None:
+        if self.control_lora is None:
+            return None
+
+        if not self.controlnet_vae:
+            raise ValueError("controlnet_vae must be set when using a FLUX Control LoRA.")
+
+        # Load the conditioning image and resize it to the target image size.
+        cond_img = context.images.get_pil(self.control_lora.img.image_name)
+        cond_img = cond_img.convert("RGB")
+        cond_img = cond_img.resize((self.width, self.height), Image.Resampling.BICUBIC)
+        cond_img = np.array(cond_img)
+
+        # Normalize the conditioning image to the range [-1, 1].
+        # This normalization is based on the original implementations here:
+        # https://github.com/black-forest-labs/flux/blob/805da8571a0b49b6d4043950bd266a65328c243b/src/flux/modules/image_embedders.py#L34
+        # https://github.com/black-forest-labs/flux/blob/805da8571a0b49b6d4043950bd266a65328c243b/src/flux/modules/image_embedders.py#L60
+        img_cond = torch.from_numpy(cond_img).float() / 127.5 - 1.0
+        img_cond = einops.rearrange(img_cond, "h w c -> 1 c h w")
+
+        vae_info = context.models.load(self.controlnet_vae.vae)
+        return FluxVaeEncodeInvocation.vae_encode(vae_info=vae_info, image_tensor=img_cond)
+
     def _normalize_ip_adapter_fields(self) -> list[IPAdapterField]:
         if self.ip_adapter is None:
             return []
@@ -681,10 +718,15 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
 
         return pos_ip_adapter_extensions, neg_ip_adapter_extensions
 
-    def _lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[LoRAModelRaw, float]]:
-        for lora in self.transformer.loras:
+    def _lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[ModelPatchRaw, float]]:
+        loras: list[Union[LoRAField, ControlLoRAField]] = [*self.transformer.loras]
+        if self.control_lora:
+            # Note: Since FLUX structural control LoRAs modify the shape of some weights, it is important that they are
+            # applied last.
+            loras.append(self.control_lora)
+        for lora in loras:
             lora_info = context.models.load(lora.lora)
-            assert isinstance(lora_info.model, LoRAModelRaw)
+            assert isinstance(lora_info.model, ModelPatchRaw)
             yield (lora_info.model, lora.weight)
             del lora_info
 

invokeai/app/invocations/flux_text_encoder.py

@@ -17,10 +17,10 @@ from invokeai.app.invocations.model import CLIPField, T5EncoderField
 from invokeai.app.invocations.primitives import FluxConditioningOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.flux.modules.conditioner import HFEncoder
-from invokeai.backend.lora.conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
 from invokeai.backend.model_manager.config import ModelFormat
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningFieldData, FLUXConditioningInfo
 
 
@@ -111,7 +111,7 @@ class FluxTextEncoderInvocation(BaseInvocation):
             if clip_text_encoder_config.format in [ModelFormat.Diffusers]:
                 # The model is non-quantized, so we can apply the LoRA weights directly into the model.
                 exit_stack.enter_context(
-                    LoRAPatcher.apply_lora_patches(
+                    LayerPatcher.apply_model_patches(
                         model=clip_text_encoder,
                         patches=self._clip_lora_iterator(context),
                         prefix=FLUX_LORA_CLIP_PREFIX,
@@ -130,9 +130,9 @@ class FluxTextEncoderInvocation(BaseInvocation):
         assert isinstance(pooled_prompt_embeds, torch.Tensor)
         return pooled_prompt_embeds
 
-    def _clip_lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[LoRAModelRaw, float]]:
+    def _clip_lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[ModelPatchRaw, float]]:
         for lora in self.clip.loras:
             lora_info = context.models.load(lora.lora)
-            assert isinstance(lora_info.model, LoRAModelRaw)
+            assert isinstance(lora_info.model, ModelPatchRaw)
             yield (lora_info.model, lora.weight)
             del lora_info

invokeai/app/invocations/image.py

@@ -1055,3 +1055,19 @@ class CanvasV2MaskAndCropInvocation(BaseInvocation, WithMetadata, WithBoard):
             image_dto = context.images.save(image=generated_image)
 
         return ImageOutput.build(image_dto)
+
+
+@invocation(
+    "test_typegen_invocation",
+    title="Test Typegen Invocation",
+    tags=["image"],
+    category="image",
+    version="1.2.2",
+)
+class TestTypegenInvocation(BaseInvocation):
+    """Test typegen ci"""
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = Image.new("RGBA", (512, 512), (255, 0, 0, 255))
+        image_dto = context.images.save(image=image)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/image_to_latents.py

@@ -13,7 +13,7 @@ from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
 from diffusers.models.autoencoders.autoencoder_tiny import AutoencoderTiny
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
-from invokeai.app.invocations.constants import DEFAULT_PRECISION, LATENT_SCALE_FACTOR
+from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
 from invokeai.app.invocations.fields import (
     FieldDescriptions,
     ImageField,
@@ -49,7 +49,7 @@ class ImageToLatentsInvocation(BaseInvocation):
     # NOTE: tile_size = 0 is a special value. We use this rather than `int | None`, because the workflow UI does not
     # offer a way to directly set None values.
     tile_size: int = InputField(default=0, multiple_of=8, description=FieldDescriptions.vae_tile_size)
-    fp32: bool = InputField(default=DEFAULT_PRECISION == torch.float32, description=FieldDescriptions.fp32)
+    fp32: bool = InputField(default=False, description=FieldDescriptions.fp32)
 
     @staticmethod
     def vae_encode(

invokeai/app/invocations/latents_to_image.py

@@ -12,7 +12,7 @@ from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
 from diffusers.models.autoencoders.autoencoder_tiny import AutoencoderTiny
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
-from invokeai.app.invocations.constants import DEFAULT_PRECISION, LATENT_SCALE_FACTOR
+from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
 from invokeai.app.invocations.fields import (
     FieldDescriptions,
     Input,
@@ -51,7 +51,7 @@ class LatentsToImageInvocation(BaseInvocation, WithMetadata, WithBoard):
     # NOTE: tile_size = 0 is a special value. We use this rather than `int | None`, because the workflow UI does not
     # offer a way to directly set None values.
     tile_size: int = InputField(default=0, multiple_of=8, description=FieldDescriptions.vae_tile_size)
-    fp32: bool = InputField(default=DEFAULT_PRECISION == torch.float32, description=FieldDescriptions.fp32)
+    fp32: bool = InputField(default=False, description=FieldDescriptions.fp32)
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> ImageOutput:

invokeai/app/invocations/model.py

@@ -10,7 +10,7 @@ from invokeai.app.invocations.baseinvocation import (
     invocation,
     invocation_output,
 )
-from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, OutputField, UIType
+from invokeai.app.invocations.fields import FieldDescriptions, ImageField, Input, InputField, OutputField, UIType
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.app.shared.models import FreeUConfig
 from invokeai.backend.model_manager.config import (
@@ -65,11 +65,6 @@ class CLIPField(BaseModel):
     loras: List[LoRAField] = Field(description="LoRAs to apply on model loading")
 
 
-class TransformerField(BaseModel):
-    transformer: ModelIdentifierField = Field(description="Info to load Transformer submodel")
-    loras: List[LoRAField] = Field(description="LoRAs to apply on model loading")
-
-
 class T5EncoderField(BaseModel):
     tokenizer: ModelIdentifierField = Field(description="Info to load tokenizer submodel")
     text_encoder: ModelIdentifierField = Field(description="Info to load text_encoder submodel")
@@ -80,6 +75,15 @@ class VAEField(BaseModel):
     seamless_axes: List[str] = Field(default_factory=list, description='Axes("x" and "y") to which apply seamless')
 
 
+class ControlLoRAField(LoRAField):
+    img: ImageField = Field(description="Image to use in structural conditioning")
+
+
+class TransformerField(BaseModel):
+    transformer: ModelIdentifierField = Field(description="Info to load Transformer submodel")
+    loras: List[LoRAField] = Field(description="LoRAs to apply on model loading")
+
+
 @invocation_output("unet_output")
 class UNetOutput(BaseInvocationOutput):
     """Base class for invocations that output a UNet field."""

invokeai/app/invocations/sd3_text_encoder.py

@@ -16,10 +16,10 @@ from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField
 from invokeai.app.invocations.model import CLIPField, T5EncoderField
 from invokeai.app.invocations.primitives import SD3ConditioningOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.backend.lora.conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
 from invokeai.backend.model_manager.config import ModelFormat
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningFieldData, SD3ConditioningInfo
 
 # The SD3 T5 Max Sequence Length set based on the default in diffusers.
@@ -150,7 +150,7 @@ class Sd3TextEncoderInvocation(BaseInvocation):
             if clip_text_encoder_config.format in [ModelFormat.Diffusers]:
                 # The model is non-quantized, so we can apply the LoRA weights directly into the model.
                 exit_stack.enter_context(
-                    LoRAPatcher.apply_lora_patches(
+                    LayerPatcher.apply_model_patches(
                         model=clip_text_encoder,
                         patches=self._clip_lora_iterator(context, clip_model),
                         prefix=FLUX_LORA_CLIP_PREFIX,
@@ -193,9 +193,9 @@ class Sd3TextEncoderInvocation(BaseInvocation):
 
     def _clip_lora_iterator(
         self, context: InvocationContext, clip_model: CLIPField
-    ) -> Iterator[Tuple[LoRAModelRaw, float]]:
+    ) -> Iterator[Tuple[ModelPatchRaw, float]]:
         for lora in clip_model.loras:
             lora_info = context.models.load(lora.lora)
-            assert isinstance(lora_info.model, LoRAModelRaw)
+            assert isinstance(lora_info.model, ModelPatchRaw)
             yield (lora_info.model, lora.weight)
             del lora_info

invokeai/app/invocations/tiled_multi_diffusion_denoise_latents.py

@@ -22,8 +22,8 @@ from invokeai.app.invocations.fields import (
 from invokeai.app.invocations.model import UNetField
 from invokeai.app.invocations.primitives import LatentsOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.diffusers_pipeline import ControlNetData, PipelineIntermediateState
 from invokeai.backend.stable_diffusion.multi_diffusion_pipeline import (
     MultiDiffusionPipeline,
@@ -194,10 +194,10 @@ class TiledMultiDiffusionDenoiseLatents(BaseInvocation):
             context.util.sd_step_callback(state, unet_config.base)
 
         # Prepare an iterator that yields the UNet's LoRA models and their weights.
-        def _lora_loader() -> Iterator[Tuple[LoRAModelRaw, float]]:
+        def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
             for lora in self.unet.loras:
                 lora_info = context.models.load(lora.lora)
-                assert isinstance(lora_info.model, LoRAModelRaw)
+                assert isinstance(lora_info.model, ModelPatchRaw)
                 yield (lora_info.model, lora.weight)
                 del lora_info
 
@@ -207,7 +207,7 @@ class TiledMultiDiffusionDenoiseLatents(BaseInvocation):
         with (
             ExitStack() as exit_stack,
             unet_info as unet,
-            LoRAPatcher.apply_lora_patches(model=unet, patches=_lora_loader(), prefix="lora_unet_"),
+            LayerPatcher.apply_model_patches(model=unet, patches=_lora_loader(), prefix="lora_unet_"),
         ):
             assert isinstance(unet, UNet2DConditionModel)
             latents = latents.to(device=unet.device, dtype=unet.dtype)

invokeai/app/services/config/config_default.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import copy
+import filecmp
 import locale
 import os
 import re
@@ -96,6 +97,7 @@ class InvokeAIAppConfig(BaseSettings):
         log_format: Log format. Use "plain" for text-only, "color" for colorized output, "legacy" for 2.3-style logging and "syslog" for syslog-style.<br>Valid values: `plain`, `color`, `syslog`, `legacy`
         log_level: Emit logging messages at this level or higher.<br>Valid values: `debug`, `info`, `warning`, `error`, `critical`
         log_sql: Log SQL queries. `log_level` must be `debug` for this to do anything. Extremely verbose.
+        log_level_network: Log level for network-related messages. 'info' and 'debug' are very verbose.<br>Valid values: `debug`, `info`, `warning`, `error`, `critical`
         use_memory_db: Use in-memory database. Useful for development.
         dev_reload: Automatically reload when Python sources are changed. Does not reload node definitions.
         profile_graphs: Enable graph profiling using `cProfile`.
@@ -162,6 +164,7 @@ class InvokeAIAppConfig(BaseSettings):
     log_format:              LOG_FORMAT = Field(default="color",            description='Log format. Use "plain" for text-only, "color" for colorized output, "legacy" for 2.3-style logging and "syslog" for syslog-style.')
     log_level:                LOG_LEVEL = Field(default="info",             description="Emit logging messages at this level or higher.")
     log_sql:                       bool = Field(default=False,              description="Log SQL queries. `log_level` must be `debug` for this to do anything. Extremely verbose.")
+    log_level_network:        LOG_LEVEL = Field(default='warning',          description="Log level for network-related messages. 'info' and 'debug' are very verbose.")
 
     # Development
     use_memory_db:                 bool = Field(default=False,              description="Use in-memory database. Useful for development.")
@@ -525,9 +528,35 @@ def get_config() -> InvokeAIAppConfig:
     ]
     example_config.write_file(config.config_file_path.with_suffix(".example.yaml"), as_example=True)
 
-    # Copy all legacy configs - We know `__path__[0]` is correct here
+    # Copy all legacy configs only if needed
+    # We know `__path__[0]` is correct here
     configs_src = Path(model_configs.__path__[0])  # pyright: ignore [reportUnknownMemberType, reportUnknownArgumentType, reportAttributeAccessIssue]
-    shutil.copytree(configs_src, config.legacy_conf_path, dirs_exist_ok=True)
+    dest_path = config.legacy_conf_path
+
+    # Create destination (we don't need to check for existence)
+    dest_path.mkdir(parents=True, exist_ok=True)
+
+    # Compare directories recursively
+    comparison = filecmp.dircmp(configs_src, dest_path)
+    need_copy = any(
+        [
+            comparison.left_only,  # Files exist only in source
+            comparison.diff_files,  # Files that differ
+            comparison.common_funny,  # Files that couldn't be compared
+        ]
+    )
+
+    if need_copy:
+        # Get permissions from destination directory
+        dest_mode = dest_path.stat().st_mode
+
+        # Copy directory tree
+        shutil.copytree(configs_src, dest_path, dirs_exist_ok=True)
+
+        # Set permissions on copied files to match destination directory
+        dest_path.chmod(dest_mode)
+        for p in dest_path.glob("**/*"):
+            p.chmod(dest_mode)
 
     if config.config_file_path.exists():
         config_from_file = load_and_migrate_config(config.config_file_path)

invokeai/app/services/model_install/model_install_default.py

@@ -438,9 +438,10 @@ class ModelInstallService(ModelInstallServiceBase):
         variants = "|".join(ModelRepoVariant.__members__.values())
         hf_repoid_re = f"^([^/:]+/[^/:]+)(?::({variants})?(?::/?([^:]+))?)?$"
         source_obj: Optional[StringLikeSource] = None
+        source_stripped = source.strip('"')
 
-        if Path(source).exists():  # A local file or directory
-            source_obj = LocalModelSource(path=Path(source))
+        if Path(source_stripped).exists():  # A local file or directory
+            source_obj = LocalModelSource(path=Path(source_stripped))
         elif match := re.match(hf_repoid_re, source):
             source_obj = HFModelSource(
                 repo_id=match.group(1),

invokeai/app/services/session_processor/session_processor_default.py

@@ -439,7 +439,9 @@ class DefaultSessionProcessor(SessionProcessorBase):
                         poll_now_event.wait(self._polling_interval)
                         continue
 
-                    self._invoker.services.logger.debug(f"Executing queue item {self._queue_item.item_id}")
+                    self._invoker.services.logger.info(
+                        f"Executing queue item {self._queue_item.item_id}, session {self._queue_item.session_id}"
+                    )
                     cancel_event.clear()
 
                     # Run the graph

invokeai/app/services/shared/sqlite_migrator/migrations/migration_11.py

@@ -35,7 +35,7 @@ class Migration11Callback:
 
     def _remove_convert_cache(self) -> None:
         """Rename models/.cache to models/.convert_cache."""
-        self._logger.info("Removing .cache directory. Converted models will now be cached in .convert_cache.")
+        self._logger.info("Removing models/.cache directory. Converted models will now be cached in .convert_cache.")
         legacy_convert_path = self._app_config.root_path / "models" / ".cache"
         shutil.rmtree(legacy_convert_path, ignore_errors=True)
 

invokeai/backend/flux/denoise.py

@@ -30,6 +30,8 @@ def denoise(
     controlnet_extensions: list[XLabsControlNetExtension | InstantXControlNetExtension],
     pos_ip_adapter_extensions: list[XLabsIPAdapterExtension],
     neg_ip_adapter_extensions: list[XLabsIPAdapterExtension],
+    # extra img tokens
+    img_cond: torch.Tensor | None,
 ):
     # step 0 is the initial state
     total_steps = len(timesteps) - 1
@@ -69,9 +71,9 @@ def denoise(
         # controlnet_residuals datastructure is efficient in that it likely contains multiple references to the same
         # tensors. Calculating the sum materializes each tensor into its own instance.
         merged_controlnet_residuals = sum_controlnet_flux_outputs(controlnet_residuals)
-
+        pred_img = torch.cat((img, img_cond), dim=-1) if img_cond is not None else img
         pred = model(
-            img=img,
+            img=pred_img,
             img_ids=img_ids,
             txt=pos_regional_prompting_extension.regional_text_conditioning.t5_embeddings,
             txt_ids=pos_regional_prompting_extension.regional_text_conditioning.t5_txt_ids,

invokeai/backend/flux/model.py

@@ -1,6 +1,7 @@
 # Initially pulled from https://github.com/black-forest-labs/flux
 
 from dataclasses import dataclass
+from typing import Optional
 
 import torch
 from torch import Tensor, nn
@@ -35,6 +36,7 @@ class FluxParams:
     theta: int
     qkv_bias: bool
     guidance_embed: bool
+    out_channels: Optional[int] = None
 
 
 class Flux(nn.Module):
@@ -47,7 +49,7 @@ class Flux(nn.Module):
 
         self.params = params
         self.in_channels = params.in_channels
-        self.out_channels = self.in_channels
+        self.out_channels = params.out_channels or self.in_channels
         if params.hidden_size % params.num_heads != 0:
             raise ValueError(f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}")
         pe_dim = params.hidden_size // params.num_heads

invokeai/backend/lora/layers/any_lora_layer.py

@@ -1,11 +0,0 @@
-from typing import Union
-
-from invokeai.backend.lora.layers.concatenated_lora_layer import ConcatenatedLoRALayer
-from invokeai.backend.lora.layers.full_layer import FullLayer
-from invokeai.backend.lora.layers.ia3_layer import IA3Layer
-from invokeai.backend.lora.layers.loha_layer import LoHALayer
-from invokeai.backend.lora.layers.lokr_layer import LoKRLayer
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-from invokeai.backend.lora.layers.norm_layer import NormLayer
-
-AnyLoRALayer = Union[LoRALayer, LoHALayer, LoKRLayer, FullLayer, IA3Layer, NormLayer, ConcatenatedLoRALayer]

invokeai/backend/lora/sidecar_layers/concatenated_lora/concatenated_lora_linear_sidecar_layer.py

@@ -1,34 +0,0 @@
-import torch
-
-from invokeai.backend.lora.layers.concatenated_lora_layer import ConcatenatedLoRALayer
-
-
-class ConcatenatedLoRALinearSidecarLayer(torch.nn.Module):
-    def __init__(
-        self,
-        concatenated_lora_layer: ConcatenatedLoRALayer,
-        weight: float,
-    ):
-        super().__init__()
-
-        self._concatenated_lora_layer = concatenated_lora_layer
-        self._weight = weight
-
-    def forward(self, input: torch.Tensor) -> torch.Tensor:
-        x_chunks: list[torch.Tensor] = []
-        for lora_layer in self._concatenated_lora_layer.lora_layers:
-            x_chunk = torch.nn.functional.linear(input, lora_layer.down)
-            if lora_layer.mid is not None:
-                x_chunk = torch.nn.functional.linear(x_chunk, lora_layer.mid)
-            x_chunk = torch.nn.functional.linear(x_chunk, lora_layer.up, bias=lora_layer.bias)
-            x_chunk *= self._weight * lora_layer.scale()
-            x_chunks.append(x_chunk)
-
-        # TODO(ryand): Generalize to support concat_axis != 0.
-        assert self._concatenated_lora_layer.concat_axis == 0
-        x = torch.cat(x_chunks, dim=-1)
-        return x
-
-    def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None):
-        self._concatenated_lora_layer.to(device=device, dtype=dtype)
-        return self

invokeai/backend/lora/sidecar_layers/lora/lora_linear_sidecar_layer.py

@@ -1,27 +0,0 @@
-import torch
-
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-
-
-class LoRALinearSidecarLayer(torch.nn.Module):
-    def __init__(
-        self,
-        lora_layer: LoRALayer,
-        weight: float,
-    ):
-        super().__init__()
-
-        self._lora_layer = lora_layer
-        self._weight = weight
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = torch.nn.functional.linear(x, self._lora_layer.down)
-        if self._lora_layer.mid is not None:
-            x = torch.nn.functional.linear(x, self._lora_layer.mid)
-        x = torch.nn.functional.linear(x, self._lora_layer.up, bias=self._lora_layer.bias)
-        x *= self._weight * self._lora_layer.scale()
-        return x
-
-    def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None):
-        self._lora_layer.to(device=device, dtype=dtype)
-        return self

invokeai/backend/lora/sidecar_layers/lora_sidecar_module.py

@@ -1,24 +0,0 @@
-import torch
-
-
-class LoRASidecarModule(torch.nn.Module):
-    """A LoRA sidecar module that wraps an original module and adds LoRA layers to it."""
-
-    def __init__(self, orig_module: torch.nn.Module, lora_layers: list[torch.nn.Module]):
-        super().__init__()
-        self.orig_module = orig_module
-        self._lora_layers = lora_layers
-
-    def add_lora_layer(self, lora_layer: torch.nn.Module):
-        self._lora_layers.append(lora_layer)
-
-    def forward(self, input: torch.Tensor) -> torch.Tensor:
-        x = self.orig_module(input)
-        for lora_layer in self._lora_layers:
-            x += lora_layer(input)
-        return x
-
-    def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None):
-        self._orig_module.to(device=device, dtype=dtype)
-        for lora_layer in self._lora_layers:
-            lora_layer.to(device=device, dtype=dtype)

invokeai/backend/model_manager/config.py

@@ -67,6 +67,7 @@ class ModelType(str, Enum):
     Main = "main"
     VAE = "vae"
     LoRA = "lora"
+    ControlLoRa = "control_lora"
     ControlNet = "controlnet"  # used by model_probe
     TextualInversion = "embedding"
     IPAdapter = "ip_adapter"
@@ -273,6 +274,36 @@ class LoRALyCORISConfig(LoRAConfigBase):
         return Tag(f"{ModelType.LoRA.value}.{ModelFormat.LyCORIS.value}")
 
 
+class ControlAdapterConfigBase(BaseModel):
+    default_settings: Optional[ControlAdapterDefaultSettings] = Field(
+        description="Default settings for this model", default=None
+    )
+
+
+class ControlLoRALyCORISConfig(ModelConfigBase, ControlAdapterConfigBase):
+    """Model config for Control LoRA models."""
+
+    type: Literal[ModelType.ControlLoRa] = ModelType.ControlLoRa
+    trigger_phrases: Optional[set[str]] = Field(description="Set of trigger phrases for this model", default=None)
+    format: Literal[ModelFormat.LyCORIS] = ModelFormat.LyCORIS
+
+    @staticmethod
+    def get_tag() -> Tag:
+        return Tag(f"{ModelType.ControlLoRa.value}.{ModelFormat.LyCORIS.value}")
+
+
+class ControlLoRADiffusersConfig(ModelConfigBase, ControlAdapterConfigBase):
+    """Model config for Control LoRA models."""
+
+    type: Literal[ModelType.ControlLoRa] = ModelType.ControlLoRa
+    trigger_phrases: Optional[set[str]] = Field(description="Set of trigger phrases for this model", default=None)
+    format: Literal[ModelFormat.Diffusers] = ModelFormat.Diffusers
+
+    @staticmethod
+    def get_tag() -> Tag:
+        return Tag(f"{ModelType.ControlLoRa.value}.{ModelFormat.Diffusers.value}")
+
+
 class LoRADiffusersConfig(LoRAConfigBase):
     """Model config for LoRA/Diffusers models."""
 
@@ -304,12 +335,6 @@ class VAEDiffusersConfig(ModelConfigBase):
         return Tag(f"{ModelType.VAE.value}.{ModelFormat.Diffusers.value}")
 
 
-class ControlAdapterConfigBase(BaseModel):
-    default_settings: Optional[ControlAdapterDefaultSettings] = Field(
-        description="Default settings for this model", default=None
-    )
-
-
 class ControlNetDiffusersConfig(DiffusersConfigBase, ControlAdapterConfigBase):
     """Model config for ControlNet models (diffusers version)."""
 
@@ -535,6 +560,8 @@ AnyModelConfig = Annotated[
         Annotated[ControlNetDiffusersConfig, ControlNetDiffusersConfig.get_tag()],
         Annotated[ControlNetCheckpointConfig, ControlNetCheckpointConfig.get_tag()],
         Annotated[LoRALyCORISConfig, LoRALyCORISConfig.get_tag()],
+        Annotated[ControlLoRALyCORISConfig, ControlLoRALyCORISConfig.get_tag()],
+        Annotated[ControlLoRADiffusersConfig, ControlLoRADiffusersConfig.get_tag()],
         Annotated[LoRADiffusersConfig, LoRADiffusersConfig.get_tag()],
         Annotated[T5EncoderConfig, T5EncoderConfig.get_tag()],
         Annotated[T5EncoderBnbQuantizedLlmInt8bConfig, T5EncoderBnbQuantizedLlmInt8bConfig.get_tag()],

invokeai/backend/model_manager/load/model_loaders/lora.py

@@ -9,14 +9,6 @@ import torch
 from safetensors.torch import load_file
 
 from invokeai.app.services.config import InvokeAIAppConfig
-from invokeai.backend.lora.conversions.flux_diffusers_lora_conversion_utils import (
-    lora_model_from_flux_diffusers_state_dict,
-)
-from invokeai.backend.lora.conversions.flux_kohya_lora_conversion_utils import (
-    lora_model_from_flux_kohya_state_dict,
-)
-from invokeai.backend.lora.conversions.sd_lora_conversion_utils import lora_model_from_sd_state_dict
-from invokeai.backend.lora.conversions.sdxl_lora_conversion_utils import convert_sdxl_keys_to_diffusers_format
 from invokeai.backend.model_manager import (
     AnyModel,
     AnyModelConfig,
@@ -28,10 +20,25 @@ from invokeai.backend.model_manager import (
 from invokeai.backend.model_manager.load.load_default import ModelLoader
 from invokeai.backend.model_manager.load.model_cache.model_cache_base import ModelCacheBase
 from invokeai.backend.model_manager.load.model_loader_registry import ModelLoaderRegistry
+from invokeai.backend.patches.lora_conversions.flux_control_lora_utils import (
+    is_state_dict_likely_flux_control,
+    lora_model_from_flux_control_state_dict,
+)
+from invokeai.backend.patches.lora_conversions.flux_diffusers_lora_conversion_utils import (
+    lora_model_from_flux_diffusers_state_dict,
+)
+from invokeai.backend.patches.lora_conversions.flux_kohya_lora_conversion_utils import (
+    is_state_dict_likely_in_flux_kohya_format,
+    lora_model_from_flux_kohya_state_dict,
+)
+from invokeai.backend.patches.lora_conversions.sd_lora_conversion_utils import lora_model_from_sd_state_dict
+from invokeai.backend.patches.lora_conversions.sdxl_lora_conversion_utils import convert_sdxl_keys_to_diffusers_format
 
 
 @ModelLoaderRegistry.register(base=BaseModelType.Any, type=ModelType.LoRA, format=ModelFormat.Diffusers)
 @ModelLoaderRegistry.register(base=BaseModelType.Any, type=ModelType.LoRA, format=ModelFormat.LyCORIS)
+@ModelLoaderRegistry.register(base=BaseModelType.Flux, type=ModelType.ControlLoRa, format=ModelFormat.LyCORIS)
+@ModelLoaderRegistry.register(base=BaseModelType.Flux, type=ModelType.ControlLoRa, format=ModelFormat.Diffusers)
 class LoRALoader(ModelLoader):
     """Class to load LoRA models."""
 
@@ -75,7 +82,10 @@ class LoRALoader(ModelLoader):
                 # https://github.com/huggingface/diffusers/blob/main/examples/dreambooth/train_dreambooth_lora_flux.py#L1194
                 model = lora_model_from_flux_diffusers_state_dict(state_dict=state_dict, alpha=None)
             elif config.format == ModelFormat.LyCORIS:
-                model = lora_model_from_flux_kohya_state_dict(state_dict=state_dict)
+                if is_state_dict_likely_in_flux_kohya_format(state_dict=state_dict):
+                    model = lora_model_from_flux_kohya_state_dict(state_dict=state_dict)
+                elif is_state_dict_likely_flux_control(state_dict=state_dict):
+                    model = lora_model_from_flux_control_state_dict(state_dict=state_dict)
             else:
                 raise ValueError(f"LoRA model is in unsupported FLUX format: {config.format}")
         elif self._model_base in [BaseModelType.StableDiffusion1, BaseModelType.StableDiffusion2]:

invokeai/backend/model_manager/load/model_util.py

@@ -15,9 +15,9 @@ from invokeai.backend.image_util.depth_anything.depth_anything_pipeline import D
 from invokeai.backend.image_util.grounding_dino.grounding_dino_pipeline import GroundingDinoPipeline
 from invokeai.backend.image_util.segment_anything.segment_anything_pipeline import SegmentAnythingPipeline
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
 from invokeai.backend.model_manager.config import AnyModel
 from invokeai.backend.onnx.onnx_runtime import IAIOnnxRuntimeModel
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 from invokeai.backend.spandrel_image_to_image_model import SpandrelImageToImageModel
 from invokeai.backend.textual_inversion import TextualInversionModelRaw
 from invokeai.backend.util.calc_tensor_size import calc_tensor_size
@@ -43,7 +43,7 @@ def calc_model_size_by_data(logger: logging.Logger, model: AnyModel) -> int:
         (
             TextualInversionModelRaw,
             IPAdapter,
-            LoRAModelRaw,
+            ModelPatchRaw,
             SpandrelImageToImageModel,
             GroundingDinoPipeline,
             SegmentAnythingPipeline,

invokeai/backend/model_manager/probe.py

@@ -15,10 +15,6 @@ from invokeai.backend.flux.controlnet.state_dict_utils import (
     is_state_dict_xlabs_controlnet,
 )
 from invokeai.backend.flux.ip_adapter.state_dict_utils import is_state_dict_xlabs_ip_adapter
-from invokeai.backend.lora.conversions.flux_diffusers_lora_conversion_utils import (
-    is_state_dict_likely_in_flux_diffusers_format,
-)
-from invokeai.backend.lora.conversions.flux_kohya_lora_conversion_utils import is_state_dict_likely_in_flux_kohya_format
 from invokeai.backend.model_hash.model_hash import HASHING_ALGORITHMS, ModelHash
 from invokeai.backend.model_manager.config import (
     AnyModelConfig,
@@ -43,6 +39,13 @@ from invokeai.backend.model_manager.util.model_util import (
     lora_token_vector_length,
     read_checkpoint_meta,
 )
+from invokeai.backend.patches.lora_conversions.flux_control_lora_utils import is_state_dict_likely_flux_control
+from invokeai.backend.patches.lora_conversions.flux_diffusers_lora_conversion_utils import (
+    is_state_dict_likely_in_flux_diffusers_format,
+)
+from invokeai.backend.patches.lora_conversions.flux_kohya_lora_conversion_utils import (
+    is_state_dict_likely_in_flux_kohya_format,
+)
 from invokeai.backend.quantization.gguf.ggml_tensor import GGMLTensor
 from invokeai.backend.quantization.gguf.loaders import gguf_sd_loader
 from invokeai.backend.spandrel_image_to_image_model import SpandrelImageToImageModel
@@ -199,8 +202,8 @@ class ModelProbe(object):
         fields["default_settings"] = fields.get("default_settings")
 
         if not fields["default_settings"]:
-            if fields["type"] in {ModelType.ControlNet, ModelType.T2IAdapter}:
-                fields["default_settings"] = get_default_settings_controlnet_t2i_adapter(fields["name"])
+            if fields["type"] in {ModelType.ControlNet, ModelType.T2IAdapter, ModelType.ControlLoRa}:
+                fields["default_settings"] = get_default_settings_control_adapters(fields["name"])
             elif fields["type"] is ModelType.Main:
                 fields["default_settings"] = get_default_settings_main(fields["base"])
 
@@ -258,6 +261,9 @@ class ModelProbe(object):
         ckpt = checkpoint if checkpoint else read_checkpoint_meta(model_path, scan=True)
         ckpt = ckpt.get("state_dict", ckpt)
 
+        if isinstance(ckpt, dict) and is_state_dict_likely_flux_control(ckpt):
+            return ModelType.ControlLoRa
+
         for key in [str(k) for k in ckpt.keys()]:
             if key.startswith(
                 (
@@ -497,7 +503,7 @@ MODEL_NAME_TO_PREPROCESSOR = {
 }
 
 
-def get_default_settings_controlnet_t2i_adapter(model_name: str) -> Optional[ControlAdapterDefaultSettings]:
+def get_default_settings_control_adapters(model_name: str) -> Optional[ControlAdapterDefaultSettings]:
     for k, v in MODEL_NAME_TO_PREPROCESSOR.items():
         model_name_lower = model_name.lower()
         if k in model_name_lower:
@@ -624,8 +630,10 @@ class LoRACheckpointProbe(CheckpointProbeBase):
         return ModelFormat.LyCORIS
 
     def get_base_type(self) -> BaseModelType:
-        if is_state_dict_likely_in_flux_kohya_format(self.checkpoint) or is_state_dict_likely_in_flux_diffusers_format(
-            self.checkpoint
+        if (
+            is_state_dict_likely_in_flux_kohya_format(self.checkpoint)
+            or is_state_dict_likely_in_flux_diffusers_format(self.checkpoint)
+            or is_state_dict_likely_flux_control(self.checkpoint)
         ):
             return BaseModelType.Flux
 
@@ -1034,6 +1042,7 @@ class T2IAdapterFolderProbe(FolderProbeBase):
 ModelProbe.register_probe("diffusers", ModelType.Main, PipelineFolderProbe)
 ModelProbe.register_probe("diffusers", ModelType.VAE, VaeFolderProbe)
 ModelProbe.register_probe("diffusers", ModelType.LoRA, LoRAFolderProbe)
+ModelProbe.register_probe("diffusers", ModelType.ControlLoRa, LoRAFolderProbe)
 ModelProbe.register_probe("diffusers", ModelType.TextualInversion, TextualInversionFolderProbe)
 ModelProbe.register_probe("diffusers", ModelType.T5Encoder, T5EncoderFolderProbe)
 ModelProbe.register_probe("diffusers", ModelType.ControlNet, ControlNetFolderProbe)
@@ -1046,6 +1055,7 @@ ModelProbe.register_probe("diffusers", ModelType.SpandrelImageToImage, SpandrelI
 ModelProbe.register_probe("checkpoint", ModelType.Main, PipelineCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.VAE, VaeCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.LoRA, LoRACheckpointProbe)
+ModelProbe.register_probe("checkpoint", ModelType.ControlLoRa, LoRACheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.TextualInversion, TextualInversionCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.ControlNet, ControlNetCheckpointProbe)
 ModelProbe.register_probe("checkpoint", ModelType.IPAdapter, IPAdapterCheckpointProbe)

invokeai/backend/model_manager/starter_models.py

@@ -488,6 +488,22 @@ union_cnet_flux = StarterModel(
     type=ModelType.ControlNet,
 )
 # endregion
+# region Control LoRA
+flux_canny_control_lora = StarterModel(
+    name="Hard Edge Detection (Canny)",
+    base=BaseModelType.Flux,
+    source="black-forest-labs/FLUX.1-Canny-dev-lora::flux1-canny-dev-lora.safetensors",
+    description="Uses detected edges in the image to control composition.",
+    type=ModelType.ControlLoRa,
+)
+flux_depth_control_lora = StarterModel(
+    name="Depth Map",
+    base=BaseModelType.Flux,
+    source="black-forest-labs/FLUX.1-Depth-dev-lora::flux1-depth-dev-lora.safetensors",
+    description="Uses depth information in the image to control the depth in the generation.",
+    type=ModelType.ControlLoRa,
+)
+# endregion
 # region T2I Adapter
 t2i_canny_sd1 = StarterModel(
     name="Hard Edge Detection (canny)",
@@ -630,6 +646,8 @@ STARTER_MODELS: list[StarterModel] = [
     tile_sdxl,
     union_cnet_sdxl,
     union_cnet_flux,
+    flux_canny_control_lora,
+    flux_depth_control_lora,
     t2i_canny_sd1,
     t2i_sketch_sd1,
     t2i_depth_sd1,
@@ -688,6 +706,8 @@ flux_bundle: list[StarterModel] = [
     clip_l_encoder,
     union_cnet_flux,
     ip_adapter_flux,
+    flux_canny_control_lora,
+    flux_depth_control_lora,
 ]
 
 STARTER_BUNDLES: dict[str, list[StarterModel]] = {

invokeai/backend/model_patcher.py

@@ -5,17 +5,14 @@ from __future__ import annotations
 
 import pickle
 from contextlib import contextmanager
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union
+from typing import Any, Iterator, List, Optional, Tuple, Type, Union
 
-import numpy as np
 import torch
 from diffusers import UNet2DConditionModel
 from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
 
 from invokeai.app.shared.models import FreeUConfig
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
 from invokeai.backend.model_manager.load.optimizations import skip_torch_weight_init
-from invokeai.backend.onnx.onnx_runtime import IAIOnnxRuntimeModel
 from invokeai.backend.textual_inversion import TextualInversionManager, TextualInversionModelRaw
 
 
@@ -176,180 +173,3 @@ class ModelPatcher:
             assert hasattr(unet, "disable_freeu")  # mypy doesn't pick up this attribute?
             if did_apply_freeu:
                 unet.disable_freeu()
-
-
-class ONNXModelPatcher:
-    # based on
-    # https://github.com/ssube/onnx-web/blob/ca2e436f0623e18b4cfe8a0363fcfcf10508acf7/api/onnx_web/convert/diffusion/lora.py#L323
-    @classmethod
-    @contextmanager
-    def apply_lora(
-        cls,
-        model: IAIOnnxRuntimeModel,
-        loras: List[Tuple[LoRAModelRaw, float]],
-        prefix: str,
-    ) -> None:
-        from invokeai.backend.models.base import IAIOnnxRuntimeModel
-
-        if not isinstance(model, IAIOnnxRuntimeModel):
-            raise Exception("Only IAIOnnxRuntimeModel models supported")
-
-        orig_weights = {}
-
-        try:
-            blended_loras: Dict[str, torch.Tensor] = {}
-
-            for lora, lora_weight in loras:
-                for layer_key, layer in lora.layers.items():
-                    if not layer_key.startswith(prefix):
-                        continue
-
-                    layer.to(dtype=torch.float32)
-                    layer_key = layer_key.replace(prefix, "")
-                    # TODO: rewrite to pass original tensor weight(required by ia3)
-                    layer_weight = layer.get_weight(None).detach().cpu().numpy() * lora_weight
-                    if layer_key in blended_loras:
-                        blended_loras[layer_key] += layer_weight
-                    else:
-                        blended_loras[layer_key] = layer_weight
-
-            node_names = {}
-            for node in model.nodes.values():
-                node_names[node.name.replace("/", "_").replace(".", "_").lstrip("_")] = node.name
-
-            for layer_key, lora_weight in blended_loras.items():
-                conv_key = layer_key + "_Conv"
-                gemm_key = layer_key + "_Gemm"
-                matmul_key = layer_key + "_MatMul"
-
-                if conv_key in node_names or gemm_key in node_names:
-                    if conv_key in node_names:
-                        conv_node = model.nodes[node_names[conv_key]]
-                    else:
-                        conv_node = model.nodes[node_names[gemm_key]]
-
-                    weight_name = [n for n in conv_node.input if ".weight" in n][0]
-                    orig_weight = model.tensors[weight_name]
-
-                    if orig_weight.shape[-2:] == (1, 1):
-                        if lora_weight.shape[-2:] == (1, 1):
-                            new_weight = orig_weight.squeeze((3, 2)) + lora_weight.squeeze((3, 2))
-                        else:
-                            new_weight = orig_weight.squeeze((3, 2)) + lora_weight
-
-                        new_weight = np.expand_dims(new_weight, (2, 3))
-                    else:
-                        if orig_weight.shape != lora_weight.shape:
-                            new_weight = orig_weight + lora_weight.reshape(orig_weight.shape)
-                        else:
-                            new_weight = orig_weight + lora_weight
-
-                    orig_weights[weight_name] = orig_weight
-                    model.tensors[weight_name] = new_weight.astype(orig_weight.dtype)
-
-                elif matmul_key in node_names:
-                    weight_node = model.nodes[node_names[matmul_key]]
-                    matmul_name = [n for n in weight_node.input if "MatMul" in n][0]
-
-                    orig_weight = model.tensors[matmul_name]
-                    new_weight = orig_weight + lora_weight.transpose()
-
-                    orig_weights[matmul_name] = orig_weight
-                    model.tensors[matmul_name] = new_weight.astype(orig_weight.dtype)
-
-                else:
-                    # warn? err?
-                    pass
-
-            yield
-
-        finally:
-            # restore original weights
-            for name, orig_weight in orig_weights.items():
-                model.tensors[name] = orig_weight
-
-    @classmethod
-    @contextmanager
-    def apply_ti(
-        cls,
-        tokenizer: CLIPTokenizer,
-        text_encoder: IAIOnnxRuntimeModel,
-        ti_list: List[Tuple[str, Any]],
-    ) -> Iterator[Tuple[CLIPTokenizer, TextualInversionManager]]:
-        from invokeai.backend.models.base import IAIOnnxRuntimeModel
-
-        if not isinstance(text_encoder, IAIOnnxRuntimeModel):
-            raise Exception("Only IAIOnnxRuntimeModel models supported")
-
-        orig_embeddings = None
-
-        try:
-            # HACK: The CLIPTokenizer API does not include a way to remove tokens after calling add_tokens(...). As a
-            # workaround, we create a full copy of `tokenizer` so that its original behavior can be restored after
-            # exiting this `apply_ti(...)` context manager.
-            #
-            # In a previous implementation, the deep copy was obtained with `ti_tokenizer = copy.deepcopy(tokenizer)`,
-            # but a pickle roundtrip was found to be much faster (1 sec vs. 0.05 secs).
-            ti_tokenizer = pickle.loads(pickle.dumps(tokenizer))
-            ti_manager = TextualInversionManager(ti_tokenizer)
-
-            def _get_trigger(ti_name: str, index: int) -> str:
-                trigger = ti_name
-                if index > 0:
-                    trigger += f"-!pad-{i}"
-                return f"<{trigger}>"
-
-            # modify text_encoder
-            orig_embeddings = text_encoder.tensors["text_model.embeddings.token_embedding.weight"]
-
-            # modify tokenizer
-            new_tokens_added = 0
-            for ti_name, ti in ti_list:
-                if ti.embedding_2 is not None:
-                    ti_embedding = (
-                        ti.embedding_2 if ti.embedding_2.shape[1] == orig_embeddings.shape[0] else ti.embedding
-                    )
-                else:
-                    ti_embedding = ti.embedding
-
-                for i in range(ti_embedding.shape[0]):
-                    new_tokens_added += ti_tokenizer.add_tokens(_get_trigger(ti_name, i))
-
-            embeddings = np.concatenate(
-                (np.copy(orig_embeddings), np.zeros((new_tokens_added, orig_embeddings.shape[1]))),
-                axis=0,
-            )
-
-            for ti_name, _ in ti_list:
-                ti_tokens = []
-                for i in range(ti_embedding.shape[0]):
-                    embedding = ti_embedding[i].detach().numpy()
-                    trigger = _get_trigger(ti_name, i)
-
-                    token_id = ti_tokenizer.convert_tokens_to_ids(trigger)
-                    if token_id == ti_tokenizer.unk_token_id:
-                        raise RuntimeError(f"Unable to find token id for token '{trigger}'")
-
-                    if embeddings[token_id].shape != embedding.shape:
-                        raise ValueError(
-                            f"Cannot load embedding for {trigger}. It was trained on a model with token dimension"
-                            f" {embedding.shape[0]}, but the current model has token dimension"
-                            f" {embeddings[token_id].shape[0]}."
-                        )
-
-                    embeddings[token_id] = embedding
-                    ti_tokens.append(token_id)
-
-                if len(ti_tokens) > 1:
-                    ti_manager.pad_tokens[ti_tokens[0]] = ti_tokens[1:]
-
-            text_encoder.tensors["text_model.embeddings.token_embedding.weight"] = embeddings.astype(
-                orig_embeddings.dtype
-            )
-
-            yield ti_tokenizer, ti_manager
-
-        finally:
-            # restore
-            if orig_embeddings is not None:
-                text_encoder.tensors["text_model.embeddings.token_embedding.weight"] = orig_embeddings

invokeai/backend/patches/layers/base_layer_patch.py

@@ -0,0 +1,22 @@
+from abc import ABC, abstractmethod
+
+import torch
+
+
+class BaseLayerPatch(ABC):
+    @abstractmethod
+    def get_parameters(self, orig_module: torch.nn.Module, weight: float) -> dict[str, torch.Tensor]:
+        """Get the parameter residual updates that should be applied to the original parameters. Parameters omitted
+        from the returned dict are not updated.
+        """
+        ...
+
+    @abstractmethod
+    def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None):
+        """Move all internal tensors to the specified device and dtype."""
+        ...
+
+    @abstractmethod
+    def calc_size(self) -> int:
+        """Calculate the total size of all internal tensors in bytes."""
+        ...

invokeai/backend/patches/layers/concatenated_lora_layer.py

@@ -2,8 +2,8 @@ from typing import Optional, Sequence
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 
 
 class ConcatenatedLoRALayer(LoRALayerBase):
@@ -20,7 +20,7 @@ class ConcatenatedLoRALayer(LoRALayerBase):
         self.lora_layers = lora_layers
         self.concat_axis = concat_axis
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         return None
 
     def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:

invokeai/backend/patches/layers/flux_control_lora_layer.py

@@ -0,0 +1,19 @@
+import torch
+
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+
+
+class FluxControlLoRALayer(LoRALayer):
+    """A special case of LoRALayer for use with FLUX Control LoRAs that pads the target parameter with zeros if the
+    shapes don't match.
+    """
+
+    def get_parameters(self, orig_module: torch.nn.Module, weight: float) -> dict[str, torch.Tensor]:
+        """This overrides the base class behavior to skip the reshaping step."""
+        scale = self.scale()
+        params = {"weight": self.get_weight(orig_module.weight) * (weight * scale)}
+        bias = self.get_bias(orig_module.bias)
+        if bias is not None:
+            params["bias"] = bias * (weight * scale)
+
+        return params

invokeai/backend/patches/layers/full_layer.py

@@ -2,7 +2,7 @@ from typing import Dict, Optional
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 from invokeai.backend.util.calc_tensor_size import calc_tensor_size
 
 
@@ -20,7 +20,7 @@ class FullLayer(LoRALayerBase):
         cls.warn_on_unhandled_keys(values=values, handled_keys={"diff", "diff_b"})
         return layer
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         return None
 
     def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:

invokeai/backend/patches/layers/ia3_layer.py

@@ -2,7 +2,7 @@ from typing import Dict, Optional
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 
 
 class IA3Layer(LoRALayerBase):
@@ -16,7 +16,7 @@ class IA3Layer(LoRALayerBase):
         self.weight = weight
         self.on_input = on_input
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         return None
 
     @classmethod

invokeai/backend/patches/layers/loha_layer.py

@@ -2,7 +2,7 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 from invokeai.backend.util.calc_tensor_size import calc_tensors_size
 
 
@@ -32,7 +32,7 @@ class LoHALayer(LoRALayerBase):
         self.t2 = t2
         assert (self.t1 is None) == (self.t2 is None)
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         return self.w1_b.shape[0]
 
     @classmethod

invokeai/backend/patches/layers/lokr_layer.py

@@ -2,7 +2,7 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 from invokeai.backend.util.calc_tensor_size import calc_tensors_size
 
 
@@ -39,7 +39,7 @@ class LoKRLayer(LoRALayerBase):
         assert (self.w2 is None) != (self.w2_a is None)
         assert (self.w2_a is None) == (self.w2_b is None)
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         if self.w1_b is not None:
             return self.w1_b.shape[0]
         elif self.w2_b is not None:

invokeai/backend/patches/layers/lora_layer.py

@@ -2,7 +2,7 @@ from typing import Dict, Optional
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 from invokeai.backend.util.calc_tensor_size import calc_tensors_size
 
 
@@ -55,7 +55,7 @@ class LoRALayer(LoRALayerBase):
 
         return layer
 
-    def rank(self) -> int:
+    def _rank(self) -> int:
         return self.down.shape[0]
 
     def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:

invokeai/backend/patches/layers/lora_layer_base.py

@@ -1,12 +1,13 @@
-from typing import Dict, Optional, Set
+from typing import Optional
 
 import torch
 
 import invokeai.backend.util.logging as logger
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
 from invokeai.backend.util.calc_tensor_size import calc_tensors_size
 
 
-class LoRALayerBase:
+class LoRALayerBase(BaseLayerPatch):
     """Base class for all LoRA-like patching layers."""
 
     # Note: It is tempting to make this a torch.nn.Module sub-class and make all tensors 'torch.nn.Parameter's. Then we
@@ -23,6 +24,7 @@ class LoRALayerBase:
     def _parse_bias(
         cls, bias_indices: torch.Tensor | None, bias_values: torch.Tensor | None, bias_size: torch.Tensor | None
     ) -> torch.Tensor | None:
+        """Helper function to parse a bias tensor from a state dict in LyCORIS format."""
         assert (bias_indices is None) == (bias_values is None) == (bias_size is None)
 
         bias = None
@@ -37,11 +39,14 @@ class LoRALayerBase:
     ) -> float | None:
         return alpha.item() if alpha is not None else None
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
+        """Return the rank of the LoRA-like layer. Or None if the layer does not have a rank. This value is used to
+        calculate the scale.
+        """
         raise NotImplementedError()
 
     def scale(self) -> float:
-        rank = self.rank()
+        rank = self._rank()
         if self._alpha is None or rank is None:
             return 1.0
         return self._alpha / rank
@@ -52,15 +57,23 @@ class LoRALayerBase:
     def get_bias(self, orig_bias: torch.Tensor) -> Optional[torch.Tensor]:
         return self.bias
 
-    def get_parameters(self, orig_module: torch.nn.Module) -> Dict[str, torch.Tensor]:
-        params = {"weight": self.get_weight(orig_module.weight)}
+    def get_parameters(self, orig_module: torch.nn.Module, weight: float) -> dict[str, torch.Tensor]:
+        scale = self.scale()
+        params = {"weight": self.get_weight(orig_module.weight) * (weight * scale)}
         bias = self.get_bias(orig_module.bias)
         if bias is not None:
-            params["bias"] = bias
+            params["bias"] = bias * (weight * scale)
+
+        # Reshape all params to match the original module's shape.
+        for param_name, param_weight in params.items():
+            orig_param = orig_module.get_parameter(param_name)
+            if param_weight.shape != orig_param.shape:
+                params[param_name] = param_weight.reshape(orig_param.shape)
+
         return params
 
     @classmethod
-    def warn_on_unhandled_keys(cls, values: Dict[str, torch.Tensor], handled_keys: Set[str]):
+    def warn_on_unhandled_keys(cls, values: dict[str, torch.Tensor], handled_keys: set[str]):
         """Log a warning if values contains unhandled keys."""
         unknown_keys = set(values.keys()) - handled_keys
         if unknown_keys:

invokeai/backend/patches/layers/norm_layer.py

@@ -2,7 +2,7 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.layers.lora_layer_base import LoRALayerBase
+from invokeai.backend.patches.layers.lora_layer_base import LoRALayerBase
 from invokeai.backend.util.calc_tensor_size import calc_tensor_size
 
 
@@ -20,7 +20,7 @@ class NormLayer(LoRALayerBase):
         cls.warn_on_unhandled_keys(values, {"w_norm", "b_norm"})
         return layer
 
-    def rank(self) -> int | None:
+    def _rank(self) -> int | None:
         return None
 
     def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:

invokeai/backend/patches/layers/set_parameter_layer.py

@@ -0,0 +1,27 @@
+import torch
+
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.util.calc_tensor_size import calc_tensor_size
+
+
+class SetParameterLayer(BaseLayerPatch):
+    """A layer that sets a single parameter to a new target value.
+    (The diff between the target value and current value is calculated internally.)
+    """
+
+    def __init__(self, param_name: str, weight: torch.Tensor):
+        super().__init__()
+        self.weight = weight
+        self.param_name = param_name
+
+    def get_parameters(self, orig_module: torch.nn.Module, weight: float) -> dict[str, torch.Tensor]:
+        # Note: We intentionally ignore the weight parameter here. This matches the behavior in the official FLUX
+        # Control LoRA implementation.
+        diff = self.weight - orig_module.get_parameter(self.param_name)
+        return {self.param_name: diff}
+
+    def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None):
+        self.weight = self.weight.to(device=device, dtype=dtype)
+
+    def calc_size(self) -> int:
+        return calc_tensor_size(self.weight)

invokeai/backend/patches/layers/utils.py

@@ -2,16 +2,16 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.lora.layers.full_layer import FullLayer
-from invokeai.backend.lora.layers.ia3_layer import IA3Layer
-from invokeai.backend.lora.layers.loha_layer import LoHALayer
-from invokeai.backend.lora.layers.lokr_layer import LoKRLayer
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-from invokeai.backend.lora.layers.norm_layer import NormLayer
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.full_layer import FullLayer
+from invokeai.backend.patches.layers.ia3_layer import IA3Layer
+from invokeai.backend.patches.layers.loha_layer import LoHALayer
+from invokeai.backend.patches.layers.lokr_layer import LoKRLayer
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+from invokeai.backend.patches.layers.norm_layer import NormLayer
 
 
-def any_lora_layer_from_state_dict(state_dict: Dict[str, torch.Tensor]) -> AnyLoRALayer:
+def any_lora_layer_from_state_dict(state_dict: Dict[str, torch.Tensor]) -> BaseLayerPatch:
     # Detect layers according to LyCORIS detection logic(`weight_list_det`)
     # https://github.com/KohakuBlueleaf/LyCORIS/tree/8ad8000efb79e2b879054da8c9356e6143591bad/lycoris/modules
 

invokeai/backend/patches/lora_conversions/flux_control_lora_utils.py

@@ -0,0 +1,84 @@
+import re
+from typing import Any, Dict
+
+import torch
+
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.flux_control_lora_layer import FluxControlLoRALayer
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+from invokeai.backend.patches.layers.set_parameter_layer import SetParameterLayer
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_TRANSFORMER_PREFIX
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+
+# A regex pattern that matches all of the keys in the Flux Dev/Canny LoRA format.
+# Example keys:
+#   guidance_in.in_layer.lora_B.bias
+#   single_blocks.0.linear1.lora_A.weight
+#   double_blocks.0.img_attn.norm.key_norm.scale
+FLUX_CONTROL_TRANSFORMER_KEY_REGEX = r"(\w+\.)+(lora_A\.weight|lora_B\.weight|lora_B\.bias|scale)"
+
+
+def is_state_dict_likely_flux_control(state_dict: Dict[str, Any]) -> bool:
+    """Checks if the provided state dict is likely in the FLUX Control LoRA format.
+
+    This is intended to be a high-precision detector, but it is not guaranteed to have perfect precision. (A
+    perfect-precision detector would require checking all keys against a whitelist and verifying tensor shapes.)
+    """
+
+    all_keys_match = all(re.match(FLUX_CONTROL_TRANSFORMER_KEY_REGEX, str(k)) for k in state_dict.keys())
+
+    # Check the shape of the img_in weight, because this layer shape is modified by FLUX control LoRAs.
+    lora_a_weight = state_dict.get("img_in.lora_A.weight", None)
+    lora_b_bias = state_dict.get("img_in.lora_B.bias", None)
+    lora_b_weight = state_dict.get("img_in.lora_B.weight", None)
+
+    return (
+        all_keys_match
+        and lora_a_weight is not None
+        and lora_b_bias is not None
+        and lora_b_weight is not None
+        and lora_a_weight.shape[1] == 128
+        and lora_b_weight.shape[0] == 3072
+        and lora_b_bias.shape[0] == 3072
+    )
+
+
+def lora_model_from_flux_control_state_dict(state_dict: Dict[str, torch.Tensor]) -> ModelPatchRaw:
+    # Group keys by layer.
+    grouped_state_dict: dict[str, dict[str, torch.Tensor]] = {}
+    for key, value in state_dict.items():
+        key_props = key.split(".")
+        layer_prop_size = -2 if any(prop in key for prop in ["lora_B", "lora_A"]) else -1
+        layer_name = ".".join(key_props[:layer_prop_size])
+        param_name = ".".join(key_props[layer_prop_size:])
+        if layer_name not in grouped_state_dict:
+            grouped_state_dict[layer_name] = {}
+        grouped_state_dict[layer_name][param_name] = value
+
+    # Create LoRA layers.
+    layers: dict[str, BaseLayerPatch] = {}
+    for layer_key, layer_state_dict in grouped_state_dict.items():
+        prefixed_key = f"{FLUX_LORA_TRANSFORMER_PREFIX}{layer_key}"
+        if layer_key == "img_in":
+            # img_in is a special case because it changes the shape of the original weight.
+            layers[prefixed_key] = FluxControlLoRALayer(
+                layer_state_dict["lora_B.weight"],
+                None,
+                layer_state_dict["lora_A.weight"],
+                None,
+                layer_state_dict["lora_B.bias"],
+            )
+        elif all(k in layer_state_dict for k in ["lora_A.weight", "lora_B.bias", "lora_B.weight"]):
+            layers[prefixed_key] = LoRALayer(
+                layer_state_dict["lora_B.weight"],
+                None,
+                layer_state_dict["lora_A.weight"],
+                None,
+                layer_state_dict["lora_B.bias"],
+            )
+        elif "scale" in layer_state_dict:
+            layers[prefixed_key] = SetParameterLayer("scale", layer_state_dict["scale"])
+        else:
+            raise ValueError(f"{layer_key} not expected")
+
+    return ModelPatchRaw(layers=layers)

invokeai/backend/patches/lora_conversions/flux_diffusers_lora_conversion_utils.py

@@ -2,11 +2,11 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.conversions.flux_lora_constants import FLUX_LORA_TRANSFORMER_PREFIX
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.lora.layers.concatenated_lora_layer import ConcatenatedLoRALayer
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.concatenated_lora_layer import ConcatenatedLoRALayer
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_TRANSFORMER_PREFIX
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 
 
 def is_state_dict_likely_in_flux_diffusers_format(state_dict: Dict[str, torch.Tensor]) -> bool:
@@ -30,7 +30,9 @@ def is_state_dict_likely_in_flux_diffusers_format(state_dict: Dict[str, torch.Te
     return all_keys_in_peft_format and all_expected_keys_present
 
 
-def lora_model_from_flux_diffusers_state_dict(state_dict: Dict[str, torch.Tensor], alpha: float | None) -> LoRAModelRaw:
+def lora_model_from_flux_diffusers_state_dict(
+    state_dict: Dict[str, torch.Tensor], alpha: float | None
+) -> ModelPatchRaw:
     """Loads a state dict in the Diffusers FLUX LoRA format into a LoRAModelRaw object.
 
     This function is based on:
@@ -49,7 +51,7 @@ def lora_model_from_flux_diffusers_state_dict(state_dict: Dict[str, torch.Tensor
     mlp_ratio = 4.0
     mlp_hidden_dim = int(hidden_size * mlp_ratio)
 
-    layers: dict[str, AnyLoRALayer] = {}
+    layers: dict[str, BaseLayerPatch] = {}
 
     def add_lora_layer_if_present(src_key: str, dst_key: str) -> None:
         if src_key in grouped_state_dict:
@@ -215,7 +217,7 @@ def lora_model_from_flux_diffusers_state_dict(state_dict: Dict[str, torch.Tensor
 
     layers_with_prefix = {f"{FLUX_LORA_TRANSFORMER_PREFIX}{k}": v for k, v in layers.items()}
 
-    return LoRAModelRaw(layers=layers_with_prefix)
+    return ModelPatchRaw(layers=layers_with_prefix)
 
 
 def _group_by_layer(state_dict: Dict[str, torch.Tensor]) -> dict[str, dict[str, torch.Tensor]]:

invokeai/backend/patches/lora_conversions/flux_kohya_lora_conversion_utils.py

@@ -3,10 +3,13 @@ from typing import Any, Dict, TypeVar
 
 import torch
 
-from invokeai.backend.lora.conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX, FLUX_LORA_TRANSFORMER_PREFIX
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.lora.layers.utils import any_lora_layer_from_state_dict
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.utils import any_lora_layer_from_state_dict
+from invokeai.backend.patches.lora_conversions.flux_lora_constants import (
+    FLUX_LORA_CLIP_PREFIX,
+    FLUX_LORA_TRANSFORMER_PREFIX,
+)
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 
 # A regex pattern that matches all of the transformer keys in the Kohya FLUX LoRA format.
 # Example keys:
@@ -36,7 +39,7 @@ def is_state_dict_likely_in_flux_kohya_format(state_dict: Dict[str, Any]) -> boo
     )
 
 
-def lora_model_from_flux_kohya_state_dict(state_dict: Dict[str, torch.Tensor]) -> LoRAModelRaw:
+def lora_model_from_flux_kohya_state_dict(state_dict: Dict[str, torch.Tensor]) -> ModelPatchRaw:
     # Group keys by layer.
     grouped_state_dict: dict[str, dict[str, torch.Tensor]] = {}
     for key, value in state_dict.items():
@@ -61,14 +64,14 @@ def lora_model_from_flux_kohya_state_dict(state_dict: Dict[str, torch.Tensor]) -
     clip_grouped_sd = _convert_flux_clip_kohya_state_dict_to_invoke_format(clip_grouped_sd)
 
     # Create LoRA layers.
-    layers: dict[str, AnyLoRALayer] = {}
+    layers: dict[str, BaseLayerPatch] = {}
     for layer_key, layer_state_dict in transformer_grouped_sd.items():
         layers[FLUX_LORA_TRANSFORMER_PREFIX + layer_key] = any_lora_layer_from_state_dict(layer_state_dict)
     for layer_key, layer_state_dict in clip_grouped_sd.items():
         layers[FLUX_LORA_CLIP_PREFIX + layer_key] = any_lora_layer_from_state_dict(layer_state_dict)
 
     # Create and return the LoRAModelRaw.
-    return LoRAModelRaw(layers=layers)
+    return ModelPatchRaw(layers=layers)
 
 
 T = TypeVar("T")

invokeai/backend/patches/lora_conversions/sd_lora_conversion_utils.py

@@ -2,19 +2,19 @@ from typing import Dict
 
 import torch
 
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.lora.layers.utils import any_lora_layer_from_state_dict
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.utils import any_lora_layer_from_state_dict
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 
 
-def lora_model_from_sd_state_dict(state_dict: Dict[str, torch.Tensor]) -> LoRAModelRaw:
+def lora_model_from_sd_state_dict(state_dict: Dict[str, torch.Tensor]) -> ModelPatchRaw:
     grouped_state_dict: dict[str, dict[str, torch.Tensor]] = _group_state(state_dict)
 
-    layers: dict[str, AnyLoRALayer] = {}
+    layers: dict[str, BaseLayerPatch] = {}
     for layer_key, values in grouped_state_dict.items():
         layers[layer_key] = any_lora_layer_from_state_dict(values)
 
-    return LoRAModelRaw(layers=layers)
+    return ModelPatchRaw(layers=layers)
 
 
 def _group_state(state_dict: Dict[str, torch.Tensor]) -> Dict[str, Dict[str, torch.Tensor]]:

invokeai/backend/patches/model_patch_raw.py

@@ -3,20 +3,17 @@ from typing import Mapping, Optional
 
 import torch
 
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
 from invokeai.backend.raw_model import RawModel
 
 
-class LoRAModelRaw(RawModel):  # (torch.nn.Module):
-    def __init__(self, layers: Mapping[str, AnyLoRALayer]):
+class ModelPatchRaw(RawModel):
+    def __init__(self, layers: Mapping[str, BaseLayerPatch]):
         self.layers = layers
 
     def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None:
-        for _key, layer in self.layers.items():
+        for layer in self.layers.values():
             layer.to(device=device, dtype=dtype)
 
     def calc_size(self) -> int:
-        model_size = 0
-        for _, layer in self.layers.items():
-            model_size += layer.calc_size()
-        return model_size
+        return sum(layer.calc_size() for layer in self.layers.values())

invokeai/backend/patches/model_patcher.py

@@ -3,26 +3,23 @@ from typing import Dict, Iterable, Optional, Tuple
 
 import torch
 
-from invokeai.backend.lora.layers.any_lora_layer import AnyLoRALayer
-from invokeai.backend.lora.layers.concatenated_lora_layer import ConcatenatedLoRALayer
-from invokeai.backend.lora.layers.lora_layer import LoRALayer
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.sidecar_layers.concatenated_lora.concatenated_lora_linear_sidecar_layer import (
-    ConcatenatedLoRALinearSidecarLayer,
-)
-from invokeai.backend.lora.sidecar_layers.lora.lora_linear_sidecar_layer import LoRALinearSidecarLayer
-from invokeai.backend.lora.sidecar_layers.lora_sidecar_module import LoRASidecarModule
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.flux_control_lora_layer import FluxControlLoRALayer
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.pad_with_zeros import pad_with_zeros
+from invokeai.backend.patches.sidecar_wrappers.base_sidecar_wrapper import BaseSidecarWrapper
+from invokeai.backend.patches.sidecar_wrappers.utils import wrap_module_with_sidecar_wrapper
 from invokeai.backend.util.devices import TorchDevice
 from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 
-class LoRAPatcher:
+class LayerPatcher:
     @staticmethod
     @torch.no_grad()
     @contextmanager
-    def apply_lora_patches(
+    def apply_model_patches(
         model: torch.nn.Module,
-        patches: Iterable[Tuple[LoRAModelRaw, float]],
+        patches: Iterable[Tuple[ModelPatchRaw, float]],
         prefix: str,
         cached_weights: Optional[Dict[str, torch.Tensor]] = None,
     ):
@@ -40,7 +37,7 @@ class LoRAPatcher:
         original_weights = OriginalWeightsStorage(cached_weights)
         try:
             for patch, patch_weight in patches:
-                LoRAPatcher.apply_lora_patch(
+                LayerPatcher.apply_model_patch(
                     model=model,
                     prefix=prefix,
                     patch=patch,
@@ -52,14 +49,15 @@ class LoRAPatcher:
             yield
         finally:
             for param_key, weight in original_weights.get_changed_weights():
-                model.get_parameter(param_key).copy_(weight)
+                cur_param = model.get_parameter(param_key)
+                cur_param.data = weight.to(dtype=cur_param.dtype, device=cur_param.device, copy=True)
 
     @staticmethod
     @torch.no_grad()
-    def apply_lora_patch(
+    def apply_model_patch(
         model: torch.nn.Module,
         prefix: str,
-        patch: LoRAModelRaw,
+        patch: ModelPatchRaw,
         patch_weight: float,
         original_weights: OriginalWeightsStorage,
     ):
@@ -87,46 +85,70 @@ class LoRAPatcher:
             if not layer_key.startswith(prefix):
                 continue
 
-            module_key, module = LoRAPatcher._get_submodule(
+            module_key, module = LayerPatcher._get_submodule(
                 model, layer_key[prefix_len:], layer_key_is_flattened=layer_keys_are_flattened
             )
 
-            # All of the LoRA weight calculations will be done on the same device as the module weight.
-            # (Performance will be best if this is a CUDA device.)
-            device = module.weight.device
-            dtype = module.weight.dtype
+            LayerPatcher._apply_model_layer_patch(
+                module_to_patch=module,
+                module_to_patch_key=module_key,
+                patch=layer,
+                patch_weight=patch_weight,
+                original_weights=original_weights,
+            )
 
-            layer_scale = layer.scale()
+    @staticmethod
+    @torch.no_grad()
+    def _apply_model_layer_patch(
+        module_to_patch: torch.nn.Module,
+        module_to_patch_key: str,
+        patch: BaseLayerPatch,
+        patch_weight: float,
+        original_weights: OriginalWeightsStorage,
+    ):
+        # All of the LoRA weight calculations will be done on the same device as the module weight.
+        # (Performance will be best if this is a CUDA device.)
+        first_param = next(module_to_patch.parameters())
+        device = first_param.device
+        dtype = first_param.dtype
 
-            # We intentionally move to the target device first, then cast. Experimentally, this was found to
-            # be significantly faster for 16-bit CPU tensors being moved to a CUDA device than doing the
-            # same thing in a single call to '.to(...)'.
-            layer.to(device=device)
-            layer.to(dtype=torch.float32)
+        # We intentionally move to the target device first, then cast. Experimentally, this was found to
+        # be significantly faster for 16-bit CPU tensors being moved to a CUDA device than doing the
+        # same thing in a single call to '.to(...)'.
+        patch.to(device=device)
+        patch.to(dtype=torch.float32)
 
-            # TODO(ryand): Using torch.autocast(...) over explicit casting may offer a speed benefit on CUDA
-            # devices here. Experimentally, it was found to be very slow on CPU. More investigation needed.
-            for param_name, lora_param_weight in layer.get_parameters(module).items():
-                param_key = module_key + "." + param_name
-                module_param = module.get_parameter(param_name)
+        # TODO(ryand): Using torch.autocast(...) over explicit casting may offer a speed benefit on CUDA
+        # devices here. Experimentally, it was found to be very slow on CPU. More investigation needed.
+        for param_name, param_weight in patch.get_parameters(module_to_patch, weight=patch_weight).items():
+            param_key = module_to_patch_key + "." + param_name
+            module_param = module_to_patch.get_parameter(param_name)
 
-                # Save original weight
-                original_weights.save(param_key, module_param)
+            # Save original weight
+            original_weights.save(param_key, module_param)
 
-                if module_param.shape != lora_param_weight.shape:
-                    lora_param_weight = lora_param_weight.reshape(module_param.shape)
+            # HACK(ryand): This condition is only necessary to handle layers in FLUX control LoRAs that change the
+            # shape of the original layer.
+            if module_param.nelement() != param_weight.nelement():
+                assert isinstance(patch, FluxControlLoRALayer)
+                expanded_weight = pad_with_zeros(module_param, param_weight.shape)
+                setattr(
+                    module_to_patch,
+                    param_name,
+                    torch.nn.Parameter(expanded_weight, requires_grad=module_param.requires_grad),
+                )
+                module_param = expanded_weight
 
-                lora_param_weight *= patch_weight * layer_scale
-                module_param += lora_param_weight.to(dtype=dtype)
+            module_param += param_weight.to(dtype=dtype)
 
-            layer.to(device=TorchDevice.CPU_DEVICE)
+        patch.to(device=TorchDevice.CPU_DEVICE)
 
     @staticmethod
     @torch.no_grad()
     @contextmanager
-    def apply_lora_sidecar_patches(
+    def apply_model_sidecar_patches(
         model: torch.nn.Module,
-        patches: Iterable[Tuple[LoRAModelRaw, float]],
+        patches: Iterable[Tuple[ModelPatchRaw, float]],
         prefix: str,
         dtype: torch.dtype,
     ):
@@ -147,7 +169,7 @@ class LoRAPatcher:
         original_modules: dict[str, torch.nn.Module] = {}
         try:
             for patch, patch_weight in patches:
-                LoRAPatcher._apply_lora_sidecar_patch(
+                LayerPatcher._apply_model_sidecar_patch(
                     model=model,
                     prefix=prefix,
                     patch=patch,
@@ -160,14 +182,14 @@ class LoRAPatcher:
             # Restore original modules.
             # Note: This logic assumes no nested modules in original_modules.
             for module_key, orig_module in original_modules.items():
-                module_parent_key, module_name = LoRAPatcher._split_parent_key(module_key)
+                module_parent_key, module_name = LayerPatcher._split_parent_key(module_key)
                 parent_module = model.get_submodule(module_parent_key)
-                LoRAPatcher._set_submodule(parent_module, module_name, orig_module)
+                LayerPatcher._set_submodule(parent_module, module_name, orig_module)
 
     @staticmethod
-    def _apply_lora_sidecar_patch(
+    def _apply_model_sidecar_patch(
         model: torch.nn.Module,
-        patch: LoRAModelRaw,
+        patch: ModelPatchRaw,
         patch_weight: float,
         prefix: str,
         original_modules: dict[str, torch.nn.Module],
@@ -190,32 +212,50 @@ class LoRAPatcher:
             if not layer_key.startswith(prefix):
                 continue
 
-            module_key, module = LoRAPatcher._get_submodule(
+            module_key, module = LayerPatcher._get_submodule(
                 model, layer_key[prefix_len:], layer_key_is_flattened=layer_keys_are_flattened
             )
 
-            # Initialize the LoRA sidecar layer.
-            lora_sidecar_layer = LoRAPatcher._initialize_lora_sidecar_layer(module, layer, patch_weight)
+            LayerPatcher._apply_model_layer_wrapper_patch(
+                model=model,
+                module_to_patch=module,
+                module_to_patch_key=module_key,
+                patch=layer,
+                patch_weight=patch_weight,
+                original_modules=original_modules,
+                dtype=dtype,
+            )
 
-            # Replace the original module with a LoRASidecarModule if it has not already been done.
-            if module_key in original_modules:
-                # The module has already been patched with a LoRASidecarModule. Append to it.
-                assert isinstance(module, LoRASidecarModule)
-                lora_sidecar_module = module
-            else:
-                # The module has not yet been patched with a LoRASidecarModule. Create one.
-                lora_sidecar_module = LoRASidecarModule(module, [])
-                original_modules[module_key] = module
-                module_parent_key, module_name = LoRAPatcher._split_parent_key(module_key)
-                module_parent = model.get_submodule(module_parent_key)
-                LoRAPatcher._set_submodule(module_parent, module_name, lora_sidecar_module)
+    @staticmethod
+    @torch.no_grad()
+    def _apply_model_layer_wrapper_patch(
+        model: torch.nn.Module,
+        module_to_patch: torch.nn.Module,
+        module_to_patch_key: str,
+        patch: BaseLayerPatch,
+        patch_weight: float,
+        original_modules: dict[str, torch.nn.Module],
+        dtype: torch.dtype,
+    ):
+        """Apply a single LoRA wrapper patch to a model."""
+        # Replace the original module with a BaseSidecarWrapper if it has not already been done.
+        if not isinstance(module_to_patch, BaseSidecarWrapper):
+            wrapped_module = wrap_module_with_sidecar_wrapper(orig_module=module_to_patch)
+            original_modules[module_to_patch_key] = module_to_patch
+            module_parent_key, module_name = LayerPatcher._split_parent_key(module_to_patch_key)
+            module_parent = model.get_submodule(module_parent_key)
+            LayerPatcher._set_submodule(module_parent, module_name, wrapped_module)
+        else:
+            assert module_to_patch_key in original_modules
+            wrapped_module = module_to_patch
 
-            # Move the LoRA sidecar layer to the same device/dtype as the orig module.
-            # TODO(ryand): Experiment with moving to the device first, then casting. This could be faster.
-            lora_sidecar_layer.to(device=lora_sidecar_module.orig_module.weight.device, dtype=dtype)
+        # Move the LoRA layer to the same device/dtype as the orig module.
+        first_param = next(module_to_patch.parameters())
+        device = first_param.device
+        patch.to(device=device, dtype=dtype)
 
-            # Add the LoRA sidecar layer to the LoRASidecarModule.
-            lora_sidecar_module.add_lora_layer(lora_sidecar_layer)
+        # Add the patch to the sidecar wrapper.
+        wrapped_module.add_patch(patch, patch_weight)
 
     @staticmethod
     def _split_parent_key(module_key: str) -> tuple[str, str]:
@@ -235,21 +275,6 @@ class LoRAPatcher:
         else:
             raise ValueError(f"Invalid module key: {module_key}")
 
-    @staticmethod
-    def _initialize_lora_sidecar_layer(orig_layer: torch.nn.Module, lora_layer: AnyLoRALayer, patch_weight: float):
-        # TODO(ryand): Add support for more original layer types and LoRA layer types.
-        if isinstance(orig_layer, torch.nn.Linear) or (
-            isinstance(orig_layer, LoRASidecarModule) and isinstance(orig_layer.orig_module, torch.nn.Linear)
-        ):
-            if isinstance(lora_layer, LoRALayer):
-                return LoRALinearSidecarLayer(lora_layer=lora_layer, weight=patch_weight)
-            elif isinstance(lora_layer, ConcatenatedLoRALayer):
-                return ConcatenatedLoRALinearSidecarLayer(concatenated_lora_layer=lora_layer, weight=patch_weight)
-            else:
-                raise ValueError(f"Unsupported Linear LoRA layer type: {type(lora_layer)}")
-        else:
-            raise ValueError(f"Unsupported layer type: {type(orig_layer)}")
-
     @staticmethod
     def _set_submodule(parent_module: torch.nn.Module, module_name: str, submodule: torch.nn.Module):
         try:

invokeai/backend/patches/pad_with_zeros.py

@@ -0,0 +1,9 @@
+import torch
+
+
+def pad_with_zeros(orig_weight: torch.Tensor, target_shape: torch.Size) -> torch.Tensor:
+    """Pad a weight tensor with zeros to match the target shape."""
+    expanded_weight = torch.zeros(target_shape, dtype=orig_weight.dtype, device=orig_weight.device)
+    slices = tuple(slice(0, dim) for dim in orig_weight.shape)
+    expanded_weight[slices] = orig_weight
+    return expanded_weight

invokeai/backend/patches/sidecar_wrappers/base_sidecar_wrapper.py

@@ -0,0 +1,54 @@
+import torch
+
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+
+
+class BaseSidecarWrapper(torch.nn.Module):
+    """A base class for sidecar wrappers.
+
+    A sidecar wrapper is a wrapper for an existing torch.nn.Module that applies a
+    list of patches as 'sidecar' patches. I.e. it applies the sidecar patches during forward inference without modifying
+    the original module.
+
+    Sidecar wrappers are typically used over regular patches when:
+    - The original module is quantized and so the weights can't be patched in the usual way.
+    - The original module is on the CPU and modifying the weights would require backing up the original weights and
+      doubling the CPU memory usage.
+    """
+
+    def __init__(
+        self, orig_module: torch.nn.Module, patches_and_weights: list[tuple[BaseLayerPatch, float]] | None = None
+    ):
+        super().__init__()
+        self._orig_module = orig_module
+        self._patches_and_weights = [] if patches_and_weights is None else patches_and_weights
+
+    @property
+    def orig_module(self) -> torch.nn.Module:
+        return self._orig_module
+
+    def add_patch(self, patch: BaseLayerPatch, patch_weight: float):
+        """Add a patch to the sidecar wrapper."""
+        self._patches_and_weights.append((patch, patch_weight))
+
+    def _aggregate_patch_parameters(
+        self, patches_and_weights: list[tuple[BaseLayerPatch, float]]
+    ) -> dict[str, torch.Tensor]:
+        """Helper function that aggregates the parameters from all patches into a single dict."""
+        params: dict[str, torch.Tensor] = {}
+
+        for patch, patch_weight in patches_and_weights:
+            # TODO(ryand): self._orig_module could be quantized. Depending on what the patch is doing with the original
+            # module, this might fail or return incorrect results.
+            layer_params = patch.get_parameters(self._orig_module, weight=patch_weight)
+
+            for param_name, param_weight in layer_params.items():
+                if param_name not in params:
+                    params[param_name] = param_weight
+                else:
+                    params[param_name] += param_weight
+
+        return params
+
+    def forward(self, *args, **kwargs):  # type: ignore
+        raise NotImplementedError()

invokeai/backend/patches/sidecar_wrappers/conv1d_sidecar_wrapper.py

@@ -0,0 +1,11 @@
+import torch
+
+from invokeai.backend.patches.sidecar_wrappers.base_sidecar_wrapper import BaseSidecarWrapper
+
+
+class Conv1dSidecarWrapper(BaseSidecarWrapper):
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        aggregated_param_residuals = self._aggregate_patch_parameters(self._patches_and_weights)
+        return self.orig_module(input) + torch.nn.functional.conv1d(
+            input, aggregated_param_residuals["weight"], aggregated_param_residuals.get("bias", None)
+        )

invokeai/backend/patches/sidecar_wrappers/conv2d_sidecar_wrapper.py

@@ -0,0 +1,11 @@
+import torch
+
+from invokeai.backend.patches.sidecar_wrappers.base_sidecar_wrapper import BaseSidecarWrapper
+
+
+class Conv2dSidecarWrapper(BaseSidecarWrapper):
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        aggregated_param_residuals = self._aggregate_patch_parameters(self._patches_and_weights)
+        return self.orig_module(input) + torch.nn.functional.conv1d(
+            input, aggregated_param_residuals["weight"], aggregated_param_residuals.get("bias", None)
+        )

invokeai/backend/patches/sidecar_wrappers/flux_rms_norm_sidecar_wrapper.py

@@ -0,0 +1,24 @@
+import torch
+
+from invokeai.backend.patches.layers.set_parameter_layer import SetParameterLayer
+from invokeai.backend.patches.sidecar_wrappers.base_sidecar_wrapper import BaseSidecarWrapper
+
+
+class FluxRMSNormSidecarWrapper(BaseSidecarWrapper):
+    """A sidecar wrapper for a FLUX RMSNorm layer.
+
+    This wrapper is a special case. It is added specifically to enable FLUX structural control LoRAs, which overwrite
+    the RMSNorm scale parameters.
+    """
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        # Given the narrow focus of this wrapper, we only support a very particular patch configuration:
+        assert len(self._patches_and_weights) == 1
+        patch, _patch_weight = self._patches_and_weights[0]
+        assert isinstance(patch, SetParameterLayer)
+        assert patch.param_name == "scale"
+
+        # Apply the patch.
+        # NOTE(ryand): Currently, we ignore the patch weight when running as a sidecar. It's not clear how this should
+        # be handled.
+        return torch.nn.functional.rms_norm(input, patch.weight.shape, patch.weight, eps=1e-6)

invokeai/backend/patches/sidecar_wrappers/linear_sidecar_wrapper.py

@@ -0,0 +1,66 @@
+import torch
+
+from invokeai.backend.patches.layers.base_layer_patch import BaseLayerPatch
+from invokeai.backend.patches.layers.concatenated_lora_layer import ConcatenatedLoRALayer
+from invokeai.backend.patches.layers.flux_control_lora_layer import FluxControlLoRALayer
+from invokeai.backend.patches.layers.lora_layer import LoRALayer
+from invokeai.backend.patches.sidecar_wrappers.base_sidecar_wrapper import BaseSidecarWrapper
+
+
+class LinearSidecarWrapper(BaseSidecarWrapper):
+    def _lora_forward(self, input: torch.Tensor, lora_layer: LoRALayer, lora_weight: float) -> torch.Tensor:
+        """An optimized implementation of the residual calculation for a Linear LoRALayer."""
+        x = torch.nn.functional.linear(input, lora_layer.down)
+        if lora_layer.mid is not None:
+            x = torch.nn.functional.linear(x, lora_layer.mid)
+        x = torch.nn.functional.linear(x, lora_layer.up, bias=lora_layer.bias)
+        x *= lora_weight * lora_layer.scale()
+        return x
+
+    def _concatenated_lora_forward(
+        self, input: torch.Tensor, concatenated_lora_layer: ConcatenatedLoRALayer, lora_weight: float
+    ) -> torch.Tensor:
+        """An optimized implementation of the residual calculation for a Linear ConcatenatedLoRALayer."""
+        x_chunks: list[torch.Tensor] = []
+        for lora_layer in concatenated_lora_layer.lora_layers:
+            x_chunk = torch.nn.functional.linear(input, lora_layer.down)
+            if lora_layer.mid is not None:
+                x_chunk = torch.nn.functional.linear(x_chunk, lora_layer.mid)
+            x_chunk = torch.nn.functional.linear(x_chunk, lora_layer.up, bias=lora_layer.bias)
+            x_chunk *= lora_weight * lora_layer.scale()
+            x_chunks.append(x_chunk)
+
+        # TODO(ryand): Generalize to support concat_axis != 0.
+        assert concatenated_lora_layer.concat_axis == 0
+        x = torch.cat(x_chunks, dim=-1)
+        return x
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        # First, apply the original linear layer.
+        # NOTE: We slice the input to match the original weight shape in order to work with FluxControlLoRAs, which
+        # change the linear layer's in_features.
+        orig_input = input
+        input = orig_input[..., : self.orig_module.in_features]
+        output = self.orig_module(input)
+
+        # Then, apply layers for which we have optimized implementations.
+        unprocessed_patches_and_weights: list[tuple[BaseLayerPatch, float]] = []
+        for patch, patch_weight in self._patches_and_weights:
+            if isinstance(patch, FluxControlLoRALayer):
+                # Note that we use the original input here, not the sliced input.
+                output += self._lora_forward(orig_input, patch, patch_weight)
+            elif isinstance(patch, LoRALayer):
+                output += self._lora_forward(input, patch, patch_weight)
+            elif isinstance(patch, ConcatenatedLoRALayer):
+                output += self._concatenated_lora_forward(input, patch, patch_weight)
+            else:
+                unprocessed_patches_and_weights.append((patch, patch_weight))
+
+        # Finally, apply any remaining patches.
+        if len(unprocessed_patches_and_weights) > 0:
+            aggregated_param_residuals = self._aggregate_patch_parameters(unprocessed_patches_and_weights)
+            output += torch.nn.functional.linear(
+                input, aggregated_param_residuals["weight"], aggregated_param_residuals.get("bias", None)
+            )
+
+        return output

invokeai/backend/patches/sidecar_wrappers/utils.py

@@ -0,0 +1,20 @@
+import torch
+
+from invokeai.backend.flux.modules.layers import RMSNorm
+from invokeai.backend.patches.sidecar_wrappers.conv1d_sidecar_wrapper import Conv1dSidecarWrapper
+from invokeai.backend.patches.sidecar_wrappers.conv2d_sidecar_wrapper import Conv2dSidecarWrapper
+from invokeai.backend.patches.sidecar_wrappers.flux_rms_norm_sidecar_wrapper import FluxRMSNormSidecarWrapper
+from invokeai.backend.patches.sidecar_wrappers.linear_sidecar_wrapper import LinearSidecarWrapper
+
+
+def wrap_module_with_sidecar_wrapper(orig_module: torch.nn.Module) -> torch.nn.Module:
+    if isinstance(orig_module, torch.nn.Linear):
+        return LinearSidecarWrapper(orig_module)
+    elif isinstance(orig_module, torch.nn.Conv1d):
+        return Conv1dSidecarWrapper(orig_module)
+    elif isinstance(orig_module, torch.nn.Conv2d):
+        return Conv2dSidecarWrapper(orig_module)
+    elif isinstance(orig_module, RMSNorm):
+        return FluxRMSNormSidecarWrapper(orig_module)
+    else:
+        raise ValueError(f"No sidecar wrapper found for module type: {type(orig_module)}")

invokeai/backend/quantization/gguf/ggml_tensor.py

@@ -52,6 +52,7 @@ GGML_TENSOR_OP_TABLE = {
     torch.ops.aten.t.default: dequantize_and_run,  # pyright: ignore
     torch.ops.aten.addmm.default: dequantize_and_run,  # pyright: ignore
     torch.ops.aten.mul.Tensor: dequantize_and_run,  # pyright: ignore
+    torch.ops.aten.add.Tensor: dequantize_and_run,  # pyright: ignore
 }
 
 if torch.backends.mps.is_available():

invokeai/backend/stable_diffusion/extensions/lora.py

@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING
 
 from diffusers import UNet2DConditionModel
 
-from invokeai.backend.lora.lora_model_raw import LoRAModelRaw
-from invokeai.backend.lora.lora_patcher import LoRAPatcher
+from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
+from invokeai.backend.patches.model_patcher import LayerPatcher
 from invokeai.backend.stable_diffusion.extensions.base import ExtensionBase
 
 if TYPE_CHECKING:
@@ -30,8 +30,8 @@ class LoRAExt(ExtensionBase):
     @contextmanager
     def patch_unet(self, unet: UNet2DConditionModel, original_weights: OriginalWeightsStorage):
         lora_model = self._node_context.models.load(self._model_id).model
-        assert isinstance(lora_model, LoRAModelRaw)
-        LoRAPatcher.apply_lora_patch(
+        assert isinstance(lora_model, ModelPatchRaw)
+        LayerPatcher.apply_model_patch(
             model=unet,
             prefix="lora_unet_",
             patch=lora_model,

invokeai/frontend/web/public/locales/de.json

@@ -98,7 +98,8 @@
         "close": "Schließen",
         "clipboard": "Zwischenablage",
         "generating": "Generieren",
-        "loadingModel": "Lade Modell"
+        "loadingModel": "Lade Modell",
+        "warnings": "Warnungen"
     },
     "gallery": {
         "galleryImageSize": "Bildgröße",
@@ -601,7 +602,9 @@
         "hfTokenLabel": "HuggingFace-Token (für einige Modelle erforderlich)",
         "hfTokenHelperText": "Für die Nutzung einiger Modelle ist ein HF-Token erforderlich. Klicken Sie hier, um Ihr Token zu erstellen oder zu erhalten.",
         "hfForbidden": "Sie haben keinen Zugriff auf dieses HF-Modell",
-        "hfTokenInvalid": "Ungültiges oder fehlendes HF-Token"
+        "hfTokenInvalid": "Ungültiges oder fehlendes HF-Token",
+        "restoreDefaultSettings": "Klicken, um die Standardeinstellungen des Modells zu verwenden.",
+        "usingDefaultSettings": "Die Standardeinstellungen des Modells werden verwendet"
     },
     "parameters": {
         "images": "Bilder",
@@ -645,8 +648,19 @@
             "fluxModelIncompatibleScaledBboxWidth": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), Skalierte Bbox-Breite ist {{width}}",
             "fluxModelIncompatibleScaledBboxHeight": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), Skalierte Bbox-Höhe ist {{height}}",
             "fluxModelIncompatibleBboxWidth": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), Bbox-Breite ist {{width}}",
-            "fluxModelIncompatibleBboxHeight": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), Bbox-Höhe ist {{height}}"
-        }
+            "fluxModelIncompatibleBboxHeight": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), Bbox-Höhe ist {{height}}",
+            "noNodesInGraph": "Keine Knoten im Graphen",
+            "canvasIsTransforming": "Leinwand ist beschäftigt (wird transformiert)",
+            "canvasIsRasterizing": "Leinwand ist beschäftigt (wird gerastert)",
+            "canvasIsCompositing": "Leinwand ist beschäftigt (wird zusammengesetzt)",
+            "canvasIsFiltering": "Leinwand ist beschäftigt (wird gefiltert)",
+            "canvasIsSelectingObject": "Leinwand ist beschäftigt (wird Objekt ausgewählt)",
+            "noPrompts": "Keine Eingabeaufforderungen generiert"
+        },
+        "seed": "Seed",
+        "patchmatchDownScaleSize": "Herunterskalieren",
+        "seamlessXAxis": "Nahtlose X Achse",
+        "seamlessYAxis": "Nahtlose Y Achse"
     },
     "settings": {
         "displayInProgress": "Zwischenbilder anzeigen",
@@ -1101,6 +1115,18 @@
         },
         "paramHrf": {
             "heading": "High Resolution Fix aktivieren"
+        },
+        "seamlessTilingYAxis": {
+            "heading": "Nahtlose Kachelung Y Achse",
+            "paragraphs": [
+                "Nahtloses Kacheln eines Bildes entlang der vertikalen Achse."
+            ]
+        },
+        "seamlessTilingXAxis": {
+            "paragraphs": [
+                "Nahtloses Kacheln eines Bildes entlang der horizontalen Achse."
+            ],
+            "heading": "Nahtlose Kachelung X Achse"
         }
     },
     "invocationCache": {

invokeai/frontend/web/public/locales/en.json

@@ -809,6 +809,7 @@
         "starterBundleHelpText": "Easily install all models needed to get started with a base model, including a main model, controlnets, IP adapters, and more. Selecting a bundle will skip any models that you already have installed.",
         "starterModels": "Starter Models",
         "starterModelsInModelManager": "Starter Models can be found in Model Manager",
+        "controlLora": "Control LoRA",
         "syncModels": "Sync Models",
         "textualInversions": "Textual Inversions",
         "triggerPhrases": "Trigger Phrases",
@@ -1032,6 +1033,7 @@
             "fluxModelIncompatibleBboxHeight": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), bbox height is {{height}}",
             "fluxModelIncompatibleScaledBboxWidth": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), scaled bbox width is {{width}}",
             "fluxModelIncompatibleScaledBboxHeight": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), scaled bbox height is {{height}}",
+            "fluxModelMultipleControlLoRAs": "Can only use 1 Control LoRA at a time",
             "canvasIsFiltering": "Canvas is busy (filtering)",
             "canvasIsTransforming": "Canvas is busy (transforming)",
             "canvasIsRasterizing": "Canvas is busy (rasterizing)",
@@ -2110,6 +2112,7 @@
         },
         "logNamespaces": {
             "logNamespaces": "Log Namespaces",
+            "dnd": "Drag and Drop",
             "gallery": "Gallery",
             "models": "Models",
             "config": "Config",
@@ -2133,8 +2136,7 @@
     "whatsNew": {
         "whatsNewInInvoke": "What's New in Invoke",
         "items": [
-            "<StrongComponent>FLUX Regional Guidance (beta)</StrongComponent>: Our beta release of FLUX Regional Guidance is live for regional prompt control.",
-            "<StrongComponent>Various UX Improvements</StrongComponent>: A number of small UX and Quality of Life improvements throughout the app."
+            "<StrongComponent>Flux Control Layers</StrongComponent>: New control models for edge detection and depth mapping are now supported for Flux dev models."
         ],
         "readReleaseNotes": "Read Release Notes",
         "watchRecentReleaseVideos": "Watch Recent Release Videos",

invokeai/frontend/web/public/locales/es.json

@@ -507,8 +507,7 @@
         "watchUiUpdatesOverview": "Descripción general de las actualizaciones de la interfaz de usuario de Watch",
         "whatsNewInInvoke": "Novedades en Invoke",
         "items": [
-            "<StrongComponent>SD 3.5</StrongComponent>: compatibilidad con SD 3.5 Medium y Large.",
-            "<StrongComponent>Lienzo</StrongComponent>: Se ha simplificado el procesamiento de la capa de control y se ha mejorado la configuración predeterminada del control."
+            "<StrongComponent>SD 3.5</StrongComponent>: compatibilidad con SD 3.5 Medium y Large."
         ]
     },
     "invocationCache": {

invokeai/frontend/web/public/locales/fr.json

@@ -96,7 +96,9 @@
         "negativePrompt": "Prompt Négatif",
         "ok": "Ok",
         "close": "Fermer",
-        "clipboard": "Presse-papier"
+        "clipboard": "Presse-papier",
+        "loadingModel": "Chargement du modèle",
+        "generating": "En Génération"
     },
     "gallery": {
         "galleryImageSize": "Taille de l'image",
@@ -287,7 +289,20 @@
         "noDefaultSettings": "Aucun paramètre par défaut configuré pour ce modèle. Visitez le Gestionnaire de Modèles pour ajouter des paramètres par défaut.",
         "usingDefaultSettings": "Utilisation des paramètres par défaut du modèle",
         "defaultSettingsOutOfSync": "Certain paramètres ne correspondent pas aux valeurs par défaut du modèle :",
-        "restoreDefaultSettings": "Cliquez pour utiliser les paramètres par défaut du modèle."
+        "restoreDefaultSettings": "Cliquez pour utiliser les paramètres par défaut du modèle.",
+        "hfForbiddenErrorMessage": "Nous vous recommandons de visiter la page du modèle sur HuggingFace.com. Le propriétaire peut exiger l'acceptation des conditions pour pouvoir télécharger.",
+        "hfTokenRequired": "Vous essayez de télécharger un modèle qui nécessite un token HuggingFace valide.",
+        "clipLEmbed": "CLIP-L Embed",
+        "hfTokenSaved": "Token HF enregistré",
+        "hfTokenUnableToVerifyErrorMessage": "Impossible de vérifier le token HuggingFace. Cela est probablement dû à une erreur réseau. Veuillez réessayer plus tard.",
+        "clipGEmbed": "CLIP-G Embed",
+        "hfTokenUnableToVerify": "Impossible de vérifier le token HF",
+        "hfTokenInvalidErrorMessage": "Token HuggingFace invalide ou manquant.",
+        "hfTokenLabel": "Token HuggingFace (Requis pour certains modèles)",
+        "hfTokenHelperText": "Un token HF est requis pour utiliser certains modèles. Cliquez ici pour créer ou obtenir votre token.",
+        "hfTokenInvalid": "Token HF invalide ou manquant",
+        "hfForbidden": "Vous n'avez pas accès à ce modèle HF.",
+        "hfTokenInvalidErrorMessage2": "Mettre à jour dans le "
     },
     "parameters": {
         "images": "Images",
@@ -336,7 +351,11 @@
             "fluxModelIncompatibleBboxWidth": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), la largeur de la bounding box est {{width}}",
             "noT5EncoderModelSelected": "Aucun modèle T5 Encoder sélectionné pour la génération FLUX",
             "fluxModelIncompatibleScaledBboxWidth": "$t(parameters.invoke.fluxRequiresDimensionsToBeMultipleOf16), la largeur de la bounding box mise à l'échelle est {{width}}",
-            "canvasIsCompositing": "La toile est en train de composer"
+            "canvasIsCompositing": "La toile est en train de composer",
+            "collectionEmpty": "{{nodeLabel}} -> {{fieldLabel}} collection vide",
+            "collectionTooFewItems": "{{nodeLabel}} -> {{fieldLabel}} : trop peu d'éléments, minimum {{minItems}}",
+            "collectionTooManyItems": "{{nodeLabel}} -> {{fieldLabel}} : trop d'éléments, maximum {{maxItems}}",
+            "canvasIsSelectingObject": "La toile est occupée (sélection d'objet)"
         },
         "negativePromptPlaceholder": "Prompt Négatif",
         "positivePromptPlaceholder": "Prompt Positif",
@@ -377,7 +396,9 @@
         "sendToUpscale": "Envoyer à Agrandir",
         "guidance": "Guidage",
         "postProcessing": "Post-traitement (Maj + U)",
-        "processImage": "Traiter l'image"
+        "processImage": "Traiter l'image",
+        "disabledNoRasterContent": "Désactivé (Aucun contenu raster)",
+        "recallMetadata": "Rappeler les métadonnées"
     },
     "settings": {
         "models": "Modèles",
@@ -415,7 +436,8 @@
         "confirmOnNewSession": "Confirmer lors d'une nouvelle session",
         "modelDescriptionsDisabledDesc": "Les descriptions des modèles dans les menus déroulants ont été désactivées. Activez-les dans les paramètres.",
         "enableModelDescriptions": "Activer les descriptions de modèle dans les menus déroulants",
-        "modelDescriptionsDisabled": "Descriptions de modèle dans les menus déroulants désactivés"
+        "modelDescriptionsDisabled": "Descriptions de modèle dans les menus déroulants désactivés",
+        "showDetailedInvocationProgress": "Afficher les détails de progression"
     },
     "toast": {
         "uploadFailed": "Importation échouée",
@@ -634,7 +656,8 @@
         "iterations_one": "Itération",
         "iterations_many": "Itérations",
         "iterations_other": "Itérations",
-        "back": "fin"
+        "back": "fin",
+        "batchSize": "Taille de lot"
     },
     "prompt": {
         "noMatchingTriggers": "Pas de déclancheurs correspondants",
@@ -1152,7 +1175,8 @@
             "heading": "Force de débruitage",
             "paragraphs": [
                 "Intensité du bruit ajouté à l'image d'entrée.",
-                "0 produira une image identique, tandis que 1 produira une image complètement différente."
+                "0 produira une image identique, tandis que 1 produira une image complètement différente.",
+                "Lorsque aucune couche raster avec du contenu visible n'est présente, ce paramètre est ignoré."
             ]
         },
         "lora": {
@@ -1447,7 +1471,9 @@
         "parsingFailed": "L'analyse a échoué",
         "recallParameter": "Rappeler {{label}}",
         "canvasV2Metadata": "Toile",
-        "guidance": "Guide"
+        "guidance": "Guide",
+        "seamlessXAxis": "Axe X sans bords",
+        "seamlessYAxis": "Axe Y sans bords"
     },
     "sdxl": {
         "freePromptStyle": "Écriture de Prompt manuelle",
@@ -1668,7 +1694,13 @@
         "delete": "Supprimer"
     },
     "whatsNew": {
-        "whatsNewInInvoke": "Quoi de neuf dans Invoke"
+        "whatsNewInInvoke": "Quoi de neuf dans Invoke",
+        "watchRecentReleaseVideos": "Regarder les vidéos des dernières versions",
+        "items": [
+            "<StrongComponent>FLUX Guidage Régional (bêta)</StrongComponent> : Notre version bêta de FLUX Guidage Régional est en ligne pour le contrôle des prompt régionaux."
+        ],
+        "readReleaseNotes": "Notes de version",
+        "watchUiUpdatesOverview": "Aperçu des mises à jour de l'interface utilisateur"
     },
     "ui": {
         "tabs": {
@@ -1776,7 +1808,10 @@
             },
             "process": "Traiter",
             "apply": "Appliquer",
-            "cancel": "Annuler"
+            "cancel": "Annuler",
+            "advanced": "Avancé",
+            "processingLayerWith": "Calque de traitement avec le filtre {{type}}.",
+            "forMoreControl": "Pour plus de contrôle, cliquez sur Avancé ci-dessous."
         },
         "canvasContextMenu": {
             "saveToGalleryGroup": "Enregistrer dans la galerie",
@@ -2029,7 +2064,17 @@
         "convertInpaintMaskTo": "Convertir $t(controlLayers.inpaintMask) vers",
         "copyControlLayerTo": "Copier $t(controlLayers.controlLayer) vers",
         "newInpaintMask": "Nouveau $t(controlLayers.inpaintMask)",
-        "newRasterLayer": "Nouveau $t(controlLayers.rasterLayer)"
+        "newRasterLayer": "Nouveau $t(controlLayers.rasterLayer)",
+        "mergingLayers": "Fusionner les couches",
+        "resetCanvasLayers": "Réinitialiser les couches de la toile",
+        "resetGenerationSettings": "Réinitialiser les paramètres de génération",
+        "mergeDown": "Fusionner",
+        "controlLayerEmptyState": "<UploadButton>Télécharger une image</UploadButton>, faites glisser une image depuis la <GalleryButton>galerie</GalleryButton> sur ce calque, ou dessinez sur la toile pour commencer.",
+        "asRasterLayer": "En tant que $t(controlLayers.rasterLayer)",
+        "asRasterLayerResize": "En tant que $t(controlLayers.rasterLayer) (Redimensionner)",
+        "asControlLayer": "En tant que $t(controlLayers.controlLayer)",
+        "asControlLayerResize": "En $t(controlLayers.controlLayer) (Redimensionner)",
+        "newSession": "Nouvelle session"
     },
     "upscaling": {
         "exceedsMaxSizeDetails": "La limite maximale d'agrandissement est de {{maxUpscaleDimension}}x{{maxUpscaleDimension}} pixels. Veuillez essayer une image plus petite ou réduire votre sélection d'échelle.",
@@ -2047,7 +2092,9 @@
         "postProcessingModel": "Modèle de post-traitement",
         "missingUpscaleModel": "Modèle d'agrandissement manquant",
         "missingUpscaleInitialImage": "Image initiale manquante pour l'agrandissement",
-        "missingTileControlNetModel": "Aucun modèle ControlNet valide installé"
+        "missingTileControlNetModel": "Aucun modèle ControlNet valide installé",
+        "incompatibleBaseModelDesc": "L'upscaling est pris en charge uniquement pour les modèles d'architecture SD1.5 et SDXL. Changez le modèle principal pour activer l'upscaling.",
+        "incompatibleBaseModel": "Modèle principal non pris en charge pour l'upscaling"
     },
     "stylePresets": {
         "deleteTemplate": "Supprimer le template",
@@ -2133,5 +2180,62 @@
         "inviteTeammates": "Inviter des collègues",
         "professionalUpsell": "Disponible dans l'édition professionnelle d'Invoke. Cliquez ici ou visitez invoke.com/pricing pour plus de détails.",
         "professional": "Professionnel"
+    },
+    "supportVideos": {
+        "watch": "Regarder",
+        "videos": {
+            "upscaling": {
+                "description": "Comment améliorer la résolution des images avec les outils d'Invoke pour les agrandir.",
+                "title": "Upscaling"
+            },
+            "howDoIGenerateAndSaveToTheGallery": {
+                "description": "Étapes pour générer et enregistrer des images dans la galerie.",
+                "title": "Comment générer et enregistrer dans la galerie ?"
+            },
+            "usingControlLayersAndReferenceGuides": {
+                "title": "Utilisation des couche de contrôle et des guides de référence",
+                "description": "Apprenez à guider la création de vos images avec des couche de contrôle et des images de référence."
+            },
+            "exploringAIModelsAndConceptAdapters": {
+                "description": "Plongez dans les modèles d'IA et découvrez comment utiliser les adaptateurs de concepts pour un contrôle créatif.",
+                "title": "Exploration des modèles d'IA et des adaptateurs de concepts"
+            },
+            "howDoIUseControlNetsAndControlLayers": {
+                "title": "Comment utiliser les réseaux de contrôle et les couches de contrôle ?",
+                "description": "Apprenez à appliquer des couches de contrôle et des ControlNets à vos images."
+            },
+            "creatingAndComposingOnInvokesControlCanvas": {
+                "description": "Apprenez à composer des images en utilisant le canvas de contrôle d'Invoke.",
+                "title": "Créer et composer sur le canvas de contrôle d'Invoke"
+            },
+            "howDoIEditOnTheCanvas": {
+                "title": "Comment puis-je modifier sur la toile ?",
+                "description": "Guide pour éditer des images directement sur la toile."
+            },
+            "howDoIDoImageToImageTransformation": {
+                "title": "Comment effectuer une transformation d'image à image ?",
+                "description": "Tutoriel sur la réalisation de transformations d'image à image dans Invoke."
+            },
+            "howDoIUseGlobalIPAdaptersAndReferenceImages": {
+                "title": "Comment utiliser les IP Adapters globaux et les images de référence ?",
+                "description": "Introduction à l'ajout d'images de référence et IP Adapters globaux."
+            },
+            "howDoIUseInpaintMasks": {
+                "title": "Comment utiliser les masques d'inpainting ?"
+            },
+            "creatingYourFirstImage": {
+                "title": "Créer votre première image",
+                "description": "Introduction à la création d'une image à partir de zéro en utilisant les outils d'Invoke."
+            },
+            "understandingImageToImageAndDenoising": {
+                "title": "Comprendre l'Image-à-Image et le Débruitage",
+                "description": "Aperçu des transformations d'image à image et du débruitage dans Invoke."
+            }
+        },
+        "gettingStarted": "Commencer",
+        "studioSessionsDesc1": "Consultez le <StudioSessionsPlaylistLink /> pour des approfondissements sur Invoke.",
+        "studioSessionsDesc2": "Rejoignez notre <DiscordLink /> pour participer aux sessions en direct et poser vos questions. Les sessions sont ajoutée dans la playlist la semaine suivante.",
+        "supportVideos": "Vidéos d'assistance",
+        "controlCanvas": "Contrôler la toile"
     }
 }

invokeai/frontend/web/public/locales/it.json

@@ -2171,8 +2171,7 @@
         "watchRecentReleaseVideos": "Guarda i video su questa versione",
         "watchUiUpdatesOverview": "Guarda le novità dell'interfaccia",
         "items": [
-            "<StrongComponent>FLUX Regional Guidance (beta)</StrongComponent>: la nostra versione beta di FLUX Regional Guidance è attiva per il controllo dei prompt regionali.",
-            "<StrongComponent>Vari miglioramenti dell'esperienza utente</StrongComponent>: numerosi piccoli miglioramenti dell'esperienza utente e della qualità della vita in tutta l'app."
+            "<StrongComponent>FLUX Regional Guidance (beta)</StrongComponent>: la nostra versione beta di FLUX Regional Guidance è attiva per il controllo dei prompt regionali."
         ]
     },
     "system": {
@@ -2196,7 +2195,8 @@
             "events": "Eventi",
             "system": "Sistema",
             "metadata": "Metadati",
-            "logNamespaces": "Elementi del registro"
+            "logNamespaces": "Elementi del registro",
+            "dnd": "Trascina e rilascia"
         },
         "enableLogging": "Abilita la registrazione"
     },

invokeai/frontend/web/public/locales/vi.json

@@ -46,7 +46,7 @@
         "menuItemAutoAdd": "Tự động thêm cho Bảng này",
         "move": "Di Chuyển",
         "topMessage": "Bảng này chứa ảnh được dùng với những tính năng sau:",
-        "uncategorized": "Chưa Phân Loại",
+        "uncategorized": "Chưa Sắp Xếp",
         "archived": "Được Lưu Trữ",
         "loading": "Đang Tải...",
         "selectBoard": "Chọn Bảng",
@@ -91,7 +91,7 @@
         "sideBySide": "Cạnh Nhau",
         "alwaysShowImageSizeBadge": "Luôn Hiển Thị Kích Thước Ảnh",
         "autoAssignBoardOnClick": "Tự Động Gán Vào Bảng Khi Nhấp Chuột",
-        "jump": "Nhảy Vào",
+        "jump": "Nhảy Đến",
         "go": "Đi",
         "autoSwitchNewImages": "Tự Động Đổi Sang Hình Ảnh Mới",
         "featuresWillReset": "Nếu bạn xoá hình ảnh này, những tính năng đó sẽ lập tức được khởi động lại.",
@@ -133,7 +133,7 @@
         "alpha": "Alpha",
         "edit": "Sửa",
         "nodes": "Workflow",
-        "format": "định dạng",
+        "format": "Định Dạng",
         "delete": "Xoá",
         "details": "Chi Tiết",
         "imageFailedToLoad": "Không Thể Tải Hình Ảnh",
@@ -164,7 +164,7 @@
         "discordLabel": "Discord",
         "back": "Trở Về",
         "advanced": "Nâng Cao",
-        "batch": "Quản Lý Hàng Loạt",
+        "batch": "Quản Lý Lô",
         "modelManager": "Quản Lý Model",
         "dontShowMeThese": "Không hiển thị thứ này",
         "ok": "OK",
@@ -196,7 +196,7 @@
         "areYouSure": "Bạn chắc chứ?",
         "ai": "ai",
         "aboutDesc": "Sử dụng Invoke cho công việc? Xem thử:",
-        "aboutHeading": "Sở Hữu Khả Năng Sáng Tạo Cho Riêng Mình",
+        "aboutHeading": "Quyền Năng Sáng Tạo Của Riêng",
         "enabled": "Đã Bật",
         "close": "Đóng",
         "data": "Dữ Liệu",
@@ -232,24 +232,24 @@
         "enqueueing": "Xếp Vào Hàng Hàng Loạt",
         "prompts_other": "Lệnh",
         "iterations_other": "Lặp Lại",
-        "total": "Tổng Cộng",
+        "total": "Tổng",
         "pruneFailed": "Có Vấn Đề Khi Cắt Bớt Mục Khỏi Hàng",
         "clearSucceeded": "Hàng Đã Được Dọn Sạch",
         "cancel": "Huỷ Bỏ",
         "clearQueueAlertDialog2": "Bạn chắc chắn muốn dọn sạch hàng không?",
         "queueEmpty": "Hàng Trống",
         "queueBack": "Thêm Vào Hàng",
-        "batchFieldValues": "Giá Trị Vùng Hàng Loạt",
+        "batchFieldValues": "Giá Trị Vùng Theo Lô",
         "openQueue": "Mở Queue",
         "pause": "Dừng Lại",
         "pauseFailed": "Có Vấn Đề Khi Dừng Lại Bộ Xử Lý",
-        "batchQueued": "Hàng Loạt Đã Vào hàng",
-        "batchFailedToQueue": "Lỗi Khi Xếp Hàng Loạt Vào Hàng",
+        "batchQueued": "Lô Đã Vào Hàng",
+        "batchFailedToQueue": "Lỗi Khi Xếp Lô Vào Hàng",
         "next": "Tiếp Theo",
-        "in_progress": "Đang Tiến Hành",
+        "in_progress": "Đang Chạy",
         "failed": "Thất Bại",
         "canceled": "Bị Huỷ",
-        "cancelBatchFailed": "Có Vấn Đề Khi Huỷ Bỏ Hàng Loạt",
+        "cancelBatchFailed": "Có Vấn Đề Khi Huỷ Bỏ Lô",
         "workflows": "Workflow (Luồng làm việc)",
         "canvas": "Canvas (Vùng ảnh)",
         "upscaling": "Upscale (Nâng Cấp Chất Lượng Hình Ảnh)",
@@ -272,19 +272,19 @@
         "resumeTooltip": "Tiếp Tục Bộ Xử Lý",
         "clearFailed": "Có Vấn Đề Khi Dọn Dẹp Hàng",
         "generations_other": "Máy Tạo Sinh",
-        "cancelBatch": "Huỷ Bỏ Hàng Loạt",
+        "cancelBatch": "Huỷ Bỏ Lô",
         "status": "Trạng Thái",
         "pending": "Đang Chờ",
         "gallery": "Thư Viện",
         "front": "trước",
-        "batch": "Hàng Loạt",
+        "batch": "Lô",
         "origin": "Nguồn Gốc",
         "destination": "Điểm Đến",
         "other": "Khác",
         "graphFailedToQueue": "Lỗi Khi Xếp Đồ Thị Vào Hàng",
         "notReady": "Không Thể Xếp Hàng",
         "cancelItem": "Huỷ Bỏ Mục",
-        "cancelBatchSucceeded": "Mục Hàng Loạt Đã Huỷ Bỏ",
+        "cancelBatchSucceeded": "Lô Đã Huỷ Bỏ",
         "current": "Hiện Tại",
         "time": "Thời Gian",
         "completed": "Hoàn Tất",
@@ -294,7 +294,7 @@
         "completedIn": "Hoàn tất trong",
         "graphQueued": "Đồ Thị Đã Vào Hàng",
         "batchQueuedDesc_other": "Thêm {{count}} phiên vào {{direction}} của hàng",
-        "batchSize": "Kích Thước Vùng Hàng Loạt"
+        "batchSize": "Kích Thước Lô"
     },
     "hotkeys": {
         "canvas": {
@@ -457,8 +457,8 @@
                 "title": "Gợi Lại Tất Cả Metadata"
             },
             "recallSeed": {
-                "title": "Gợi Lại Tham Số Hạt Giống",
-                "desc": "Gợi lại tham số hạt giống của ảnh hiện tại."
+                "title": "Gợi Lại Hạt Giống",
+                "desc": "Gợi lại hạt giống của ảnh hiện tại."
             },
             "useSize": {
                 "title": "Dùng Kích Thước",
@@ -490,7 +490,7 @@
                 "title": "Đổi Ảnh So Sánh"
             },
             "remix": {
-                "desc": "Gợi lại tất cả metadata cho tham số hạt giống của ảnh hiện tại.",
+                "desc": "Gợi lại tất cả metadata cho hạt giống của ảnh hiện tại.",
                 "title": "Phối Lại"
             },
             "runPostprocessing": {
@@ -633,7 +633,7 @@
         "modelManager": "Quản Lý Model",
         "name": "Tên",
         "noModelSelected": "Không Có Model Được Chọn",
-        "installQueue": "Tải Xuống Danh Sách Đợi",
+        "installQueue": "Danh Sách Tải Xuống",
         "modelDeleteFailed": "Xoá model thất bại",
         "inplaceInstallDesc": "Tải xuống model mà không sao chép toàn bộ tài nguyên. Khi sử dụng model, nó được sẽ tải từ vị trí được đặt. Nếu bị tắt, toàn bộ tài nguyên của model sẽ được sao chép vào thư mục quản lý model của Invoke trong quá trình tải xuống.",
         "modelType": "Loại Model",
@@ -705,7 +705,7 @@
         "spandrelImageToImage": "Hình Ảnh Sang Hình Ảnh (Spandrel)",
         "starterBundles": "Quà Tân Thủ",
         "vae": "VAE",
-        "urlOrLocalPath": "URL Hoặc Đường Dẫn Trên Máy Chủ",
+        "urlOrLocalPath": "URL / Đường Dẫn",
         "triggerPhrases": "Từ Ngữ Kích Hoạt",
         "variant": "Biến Thể",
         "urlOrLocalPathHelper": "Url cần chỉ vào một tệp duy nhất. Còn đường dẫn trên máy chủ có thể chỉ vào một tệp hoặc một thư mục cho chỉ một model diffusers.",
@@ -723,7 +723,7 @@
         "starterModels": "Model Khởi Đầu",
         "typePhraseHere": "Thêm từ ngữ ở đây",
         "upcastAttention": "Upcast Attention",
-        "vaePrecision": "VAE Precision",
+        "vaePrecision": "Độ Chuẩn VAE",
         "installingBundle": "Đang Tải Nguyên Bộ",
         "installingModel": "Đang Tải Model",
         "installingXModels_other": "Đang tải {{count}} model",
@@ -751,18 +751,18 @@
         "parameterSet": "Dữ liệu tham số {{parameter}}",
         "positivePrompt": "Lệnh Tích Cực",
         "recallParameter": "Gợi Nhớ {{label}}",
-        "seed": "Tham Số Hạt Giống",
+        "seed": "Hạt Giống",
         "negativePrompt": "Lệnh Tiêu Cực",
         "noImageDetails": "Không tìm thấy chí tiết ảnh",
         "strength": "Mức độ mạnh từ ảnh sang ảnh",
         "Threshold": "Ngưỡng Nhiễu",
         "width": "Chiều Rộng",
-        "steps": "Tham Số Bước",
+        "steps": "Số Bước",
         "vae": "VAE",
         "workflow": "Workflow",
         "seamlessXAxis": "Trục X Liền Mạch",
         "seamlessYAxis": "Trục Y Liền Mạch",
-        "cfgScale": "Thước Đo CFG",
+        "cfgScale": "Thang CFG",
         "allPrompts": "Tất Cả Lệnh",
         "generationMode": "Chế Độ Tạo Sinh",
         "height": "Chiều Dài",
@@ -776,7 +776,7 @@
     },
     "accordions": {
         "generation": {
-            "title": "Generation (Máy Tạo Sinh)"
+            "title": "Generation (Tạo Sinh)"
         },
         "image": {
             "title": "Hình Ảnh"
@@ -786,7 +786,7 @@
             "options": "Lựa Chọn $t(accordions.advanced.title)"
         },
         "compositing": {
-            "coherenceTab": "Coherence Pass (Lớp Kết Hợp)",
+            "coherenceTab": "Coherence Pass (Liên Kết)",
             "title": "Kết Hợp",
             "infillTab": "Infill (Lấp Đầy)"
         },
@@ -795,21 +795,21 @@
         }
     },
     "invocationCache": {
-        "disableSucceeded": "Bộ Nhớ Đệm Kích Hoạt Đã Tắt",
-        "disableFailed": "Có Vấn Đề Khi Tắt Bộ Nhớ Đệm Kích Hoạt",
-        "hits": "Truy Cập Bộ Nhớ Đệm",
-        "maxCacheSize": "Kích Thước Tối Đa Bộ Nhớ Đệm",
-        "cacheSize": "Kích Thước Bộ Nhớ Đệm",
-        "enableFailed": "Có Vấn Đề Khi Bật Bộ Nhớ Đệm Kích Hoạt",
+        "disableSucceeded": "Bộ Nhớ Đệm Đã Tắt",
+        "disableFailed": "Có Vấn Đề Khi Tắt Bộ Nhớ Đệm",
+        "hits": "Số Lần Trúng",
+        "maxCacheSize": "Tối Đa",
+        "cacheSize": "Tổng Cache",
+        "enableFailed": "Có Vấn Đề Khi Bật Bộ Nhớ Đệm",
         "disable": "Tắt",
-        "invocationCache": "Bộ Nhớ Đệm Kích Hoạt",
-        "clearSucceeded": "Bộ Nhớ Đệm Kích Hoạt Đã Được Dọn",
-        "enableSucceeded": "Bộ Nhớ Đệm Kích Hoạt Đã Bật",
+        "invocationCache": "Bộ Nhớ Đệm",
+        "clearSucceeded": "Bộ Nhớ Đệm Đã Được Dọn",
+        "enableSucceeded": "Bộ Nhớ Đệm Đã Bật",
         "useCache": "Dùng Bộ Nhớ Đệm",
         "enable": "Bật",
-        "misses": "Không Truy Cập Bộ Nhớ Đệm",
+        "misses": "Số Lần Trật",
         "clear": "Dọn Dẹp",
-        "clearFailed": "Có Vấn Đề Khi Dọn Dẹp Bộ Nhớ Đệm Kích Hoạt"
+        "clearFailed": "Có Vấn Đề Khi Dọn Dẹp Bộ Nhớ Đệm"
     },
     "hrf": {
         "metadata": {
@@ -964,9 +964,9 @@
     },
     "popovers": {
         "paramCFGRescaleMultiplier": {
-            "heading": "CFG Rescale Multiplier",
+            "heading": "Hệ Số Nhân Thang CFG",
             "paragraphs": [
-                "Hệ số nhân điều chỉnh cho hướng dẫn CFG, dùng cho model được huấn luyện bằng zero-terminal SNR (ztsnr).",
+                "Hệ số nhân điều chỉnh để hướng dẫn cho CFG, dùng cho model được huấn luyện bằng zero-terminal SNR (ztsnr).",
                 "Giá trị khuyến cáo là 0.7 cho những model này."
             ]
         },
@@ -978,10 +978,10 @@
             ]
         },
         "paramCFGScale": {
-            "heading": "Thước Đo CFG",
+            "heading": "Thang CFG",
             "paragraphs": [
                 "Điều khiển mức độ lệnh tác động lên quá trình tạo sinh.",
-                "Giá trị của Thước đo CFG quá cao có thể tạo độ bão hoà quá mức và khiến ảnh tạo sinh bị méo mó. "
+                "Giá trị của Thang CFG quá cao có thể tạo độ bão hoà quá mức và khiến ảnh tạo sinh bị méo mó. "
             ]
         },
         "paramScheduler": {
@@ -992,13 +992,13 @@
             ]
         },
         "compositingCoherencePass": {
-            "heading": "Coherence Pass (Lớp Kết Hợp)",
+            "heading": "Coherence Pass (Liên Kết)",
             "paragraphs": [
                 "Bước thứ hai trong quá trình khử nhiễu để hợp nhất với ảnh inpaint/outpaint."
             ]
         },
         "refinerNegativeAestheticScore": {
-            "heading": "Điểm Tiêu Cực Cho Tiêu Chuẩn",
+            "heading": "Điểm Khác Tiêu Chuẩn",
             "paragraphs": [
                 "Trọng lượng để tạo sinh ảnh giống với ảnh có điểm tiêu chuẩn thấp, dựa vào dữ liệu huấn luyện."
             ]
@@ -1006,26 +1006,26 @@
         "refinerCfgScale": {
             "paragraphs": [
                 "Điều khiển mức độ lệnh tác động lên quá trình tạo sinh.",
-                "Giống với thước đo CFG để tạo sinh."
+                "Giống với thang CFG để tạo sinh."
             ],
-            "heading": "Thước Đo CFG"
+            "heading": "Thang CFG"
         },
         "refinerSteps": {
-            "heading": "Tham Số Bước",
+            "heading": "Số Bước",
             "paragraphs": [
                 "Số bước diễn ra trong khi tinh chế các phần nhỏ của quá trình tạo sinh.",
-                "Giống với tham số bước để tạo sinh."
+                "Giống với số bước để tạo sinh."
             ]
         },
         "paramSteps": {
-            "heading": "Tham Số Bước",
+            "heading": "Số Bước",
             "paragraphs": [
                 "Số bước dùng để biểu diễn trong mỗi lần tạo sinh.",
                 "Số bước càng cao thường sẽ tạo ra ảnh tốt hơn nhưng ngốn nhiều thời gian hơn."
             ]
         },
         "paramWidth": {
-            "heading": "Chiều Rộng",
+            "heading": "Rộng",
             "paragraphs": [
                 "Chiều rộng của ảnh tạo sinh. Phải là bội số của 8."
             ]
@@ -1052,14 +1052,14 @@
             "paragraphs": [
                 "Trọng lượng để tạo sinh ảnh giống với ảnh có điểm tiêu chuẩn cao, dựa vào dữ liệu huấn luyện."
             ],
-            "heading": "Điểm Tích Cực Cho Tiêu Chuẩn"
+            "heading": "Điểm Giống Tiêu Chuẩn"
         },
         "paramVAEPrecision": {
             "paragraphs": [
                 "Độ chính xác dùng trong khi mã hoá và giải mã VAE.",
                 "Chính xác một nửa/Fp16 sẽ hiệu quả hơn, đổi lại cho những thay đổi nhỏ với ảnh."
             ],
-            "heading": "VAE Precision"
+            "heading": "Độ Chuẩn VAE"
         },
         "fluxDevLicense": {
             "heading": "Giấy Phép Phi Thương Mại",
@@ -1078,14 +1078,14 @@
             "paragraphs": [
                 "Chiều dài của ảnh tạo sinh. Phải là bội số của 8."
             ],
-            "heading": "Chiều Dài"
+            "heading": "Dài"
         },
         "paramRatio": {
             "paragraphs": [
                 "Tỉ lệ khung hình của kích thước của ảnh được tạo ra.",
                 "Kích thước ảnh (theo số lượng pixel) tương đương với 512x512 được khuyến nghị cho model SD1.5 và kích thước tương đương với 1024x1024 được khuyến nghị cho model SDXL."
             ],
-            "heading": "Tỉ Lệ Khung Hình"
+            "heading": "Tỉ Lệ"
         },
         "seamlessTilingYAxis": {
             "paragraphs": [
@@ -1140,19 +1140,19 @@
         },
         "compositingCoherenceMinDenoise": {
             "paragraphs": [
-                "Sức mạnh khử nhiễu nhỏ nhất cho chế độ kết hợp",
-                "Sức mạnh khử nhiễu nhỏ nhất cho vùng kết hợp khi inpaint/outpaint"
+                "Độ khử nhiễu nhỏ nhất cho chế độ liên kết",
+                "Sức mạnh khử nhiễu nhỏ nhất cho vùng liên kết khi inpaint/outpaint"
             ],
-            "heading": "Độ Khử Nhiễu Tối Thiểu"
+            "heading": "Min Khử Nhiễu"
         },
         "compositingCoherenceEdgeSize": {
             "paragraphs": [
-                "Kích thước cạnh cho lớp kết hợp."
+                "Kích cỡ cạnh dùng cho coherence pass."
             ],
-            "heading": "Kích Thước Cạnh"
+            "heading": "Kích Cỡ Cạnh"
         },
         "compositingMaskBlur": {
-            "heading": "Mask Blur",
+            "heading": "Độ Mờ Vùng",
             "paragraphs": [
                 "Độ mờ của phần được phủ."
             ]
@@ -1180,7 +1180,7 @@
         "noiseUseCPU": {
             "paragraphs": [
                 "Điều chỉnh độ nhiễu được tạo ra trên CPU hay GPU.",
-                "Với Độ nhiễu CPU được bật, một tham số hạt giống cụ thể sẽ tạo ra hình ảnh giống nhau trên mọi máy.",
+                "Với Độ nhiễu CPU được bật, một hạt giống cụ thể sẽ tạo ra hình ảnh giống nhau trên mọi máy.",
                 "Không có tác động nào đến hiệu suất khi bật Độ nhiễu CPU."
             ],
             "heading": "Dùng Độ Nhiễu CPU"
@@ -1210,7 +1210,7 @@
                 "• Bước Bắt Đầu (%): Chỉ định lúc bắt đầu áp dụng chỉ dẫn từ layer này trong quá trình tạo sinh.",
                 "• Bước Kết Thúc (%): Chỉ định lúc dừng áp dụng chỉ dẫn của layer này và trở về chỉ dẫn chung từ model và các thiết lập khác."
             ],
-            "heading": "Phần Trăm Tham Số Bước Khi Bắt Đầu/Kết Thúc"
+            "heading": "Phần Trăm Số Bước Khi Bắt Đầu/Kết Thúc"
         },
         "scale": {
             "heading": "Tỉ Lệ",
@@ -1232,12 +1232,12 @@
         },
         "dynamicPromptsSeedBehaviour": {
             "paragraphs": [
-                "Điều khiển cách tham số hạt giống được dùng khi tạo sinh từ lệnh.",
-                "Cứ mỗi lần lặp, một tham số hạt giống mới sẽ được dùng. Dùng nó để khám phá những biến thể từ lệnh trên mỗi tham số hạt giống.",
-                "Ví dụ, nếu bạn có 5 lệnh, mỗi ảnh sẽ dùng cùng tham số hạt giống.",
-                "Một tham số hạt giống mới sẽ được dùng cho từng ảnh. Nó tạo ra nhiều biến thể."
+                "Điều khiển cách hạt giống được dùng khi tạo sinh từ lệnh.",
+                "Cứ mỗi lần lặp, một hạt giống mới sẽ được dùng. Dùng nó để khám phá những biến thể từ lệnh trên mỗi hạt giống.",
+                "Ví dụ, nếu bạn có 5 lệnh, mỗi ảnh sẽ dùng cùng hạt giống.",
+                "Một hạt giống mới sẽ được dùng cho từng ảnh. Nó tạo ra nhiều biến thể."
             ],
-            "heading": "Hành Động Cho Tham Số Hạt Giống"
+            "heading": "Hành Vi Của Hạt Giống"
         },
         "paramGuidance": {
             "heading": "Hướng Dẫn",
@@ -1266,10 +1266,10 @@
         },
         "paramAspect": {
             "paragraphs": [
-                "Tỉ lệ khung hành của ảnh tạo sinh. Điều chỉnh tỉ lệ se cập nhật Chiều Rộng và Chiều Dài tương ứng.",
-                "\"Tối ưu hoá\" sẽ đặt Chiều Rộng và Chiều Dài vào kích thước tối ưu cho model được chọn."
+                "Tỉ lệ khung hành của ảnh tạo sinh. Điều chỉnh tỉ lệ sẽ cập nhật chiều rộng và chiều dài tương ứng.",
+                "\"Tối ưu hoá\" sẽ đặt chiều rộng và chiều dài vào kích thước tối ưu cho model được chọn."
             ],
-            "heading": "Khung Hình"
+            "heading": "Tỉ Lệ"
         },
         "paramNegativeConditioning": {
             "heading": "Lệnh Tiêu Cực",
@@ -1289,7 +1289,7 @@
                 "Nơi trong quá trình xử lý tạo sinh mà refiner bắt đầu được dùng.",
                 "0 nghĩa là bộ refiner sẽ được dùng trong toàn bộ quá trình tạo sinh , 0.8 nghĩa là refiner sẽ được dùng trong 20% cuối cùng quá trình tạo sinh."
             ],
-            "heading": "Nơi Bắt Đầu Refiner"
+            "heading": "Bắt Đầu Refiner"
         },
         "paramUpscaleMethod": {
             "paragraphs": [
@@ -1310,9 +1310,9 @@
             "heading": "Độ Cấu Trúc"
         },
         "infillMethod": {
-            "heading": "Cách Thức Infill",
+            "heading": "Cách Infill",
             "paragraphs": [
-                "Cách thức làm infill trong quá trình inpaint/outpaint."
+                "Cách thức infill trong quá trình inpaint/outpaint."
             ]
         },
         "paramDenoisingStrength": {
@@ -1341,7 +1341,7 @@
                 "Điều khiển độ nhiễu ban đầu được dùng để tạo sinh.",
                 "Tắt lựa chọn \"Ngẫu Nhiên\" để tạo ra kết quá y hệt nhau với cùng một thiết lập tạo sinh."
             ],
-            "heading": "Tham Số Hạt Giống"
+            "heading": "Hạt Giống"
         },
         "clipSkip": {
             "heading": "CLIP Skip",
@@ -1366,7 +1366,7 @@
         "compositingCoherenceMode": {
             "heading": "Chế Độ",
             "paragraphs": [
-                "Cách thức được dùng để kết hợp ảnh với vùng bao phủ vừa được tạo sinh."
+                "Cách thức được dùng để liên kết ảnh với vùng bao phủ vừa được tạo sinh."
             ]
         },
         "paramModel": {
@@ -1391,7 +1391,7 @@
     },
     "models": {
         "addLora": "Thêm LoRA",
-        "concepts": "Khái Niệm",
+        "concepts": "LoRA",
         "loading": "đang tải",
         "lora": "LoRA",
         "noMatchingLoRAs": "Không có LoRA phù hợp",
@@ -1406,7 +1406,7 @@
         "postProcessing": "Xử Lý Hậu Kỳ (Shift + U)",
         "symmetry": "Tính Đối Xứng",
         "type": "Loại",
-        "seed": "Tham Số Hạt Giống",
+        "seed": "Hạt Giống",
         "processImage": "Xử Lý Hình Ảnh",
         "useSize": "Dùng Kích Thước",
         "invoke": {
@@ -1435,19 +1435,19 @@
             "collectionTooManyItems": "{{nodeLabel}} -> {{fieldLabel}}: quá nhiều mục, tối đa {{maxItems}}",
             "canvasIsSelectingObject": "Canvas đang bận (đang chọn đồ vật)"
         },
-        "cfgScale": "Thước Đo CFG",
-        "useSeed": "Dùng Tham Số Hạt Giống",
+        "cfgScale": "Thang CFG",
+        "useSeed": "Dùng Hạt Giống",
         "imageActions": "Hành Động Với Hình Ảnh",
-        "steps": "Tham Số Bước",
-        "aspect": "Khung Hình",
+        "steps": "Số Bước",
+        "aspect": "Tỉ Lệ",
         "coherenceMode": "Chế Độ",
-        "coherenceEdgeSize": "Kích Thước Cạnh",
-        "coherenceMinDenoise": "Tham Số Khử Nhiễu Nhỏ Nhất",
+        "coherenceEdgeSize": "Kích Cỡ Cạnh",
+        "coherenceMinDenoise": "Min Khử Nhiễu",
         "denoisingStrength": "Sức Mạnh Khử Nhiễu",
-        "infillMethod": "Cách Thức Infill",
+        "infillMethod": "Cách Infill",
         "setToOptimalSize": "Tối ưu hoá kích cỡ cho model",
-        "maskBlur": "Mask Blur",
-        "width": "Chiều Rộng",
+        "maskBlur": "Độ Mờ Vùng",
+        "width": "Rộng",
         "scale": "Tỉ Lệ",
         "recallMetadata": "Gợi Lại Metadata",
         "clipSkip": "CLIP Skip",
@@ -1455,7 +1455,7 @@
         "boxBlur": "Box Blur",
         "gaussianBlur": "Gaussian Blur",
         "staged": "Staged (Tăng khử nhiễu có hệ thống)",
-        "scaledHeight": "Tỉ Lệ Chiều Dài",
+        "scaledHeight": "Tỉ Lệ Dài",
         "cancel": {
             "cancel": "Huỷ"
         },
@@ -1463,12 +1463,12 @@
         "optimizedImageToImage": "Tối Ưu Hoá Hình Ảnh Sang Hình Ảnh",
         "sendToCanvas": "Gửi Vào Canvas",
         "sendToUpscale": "Gửi Vào Upscale",
-        "scaledWidth": "Tỉ Lệ Chiều Rộng",
+        "scaledWidth": "Tỉ Lệ Rộng",
         "scheduler": "Scheduler",
         "seamlessXAxis": "Trục X Liền Mạch",
         "seamlessYAxis": "Trục Y Liền Mạch",
         "guidance": "Hướng Dẫn",
-        "height": "Chiều Cao",
+        "height": "Dài",
         "noiseThreshold": "Ngưỡng Nhiễu",
         "negativePromptPlaceholder": "Lệnh Tiêu Cực",
         "iterations": "Lặp Lại",
@@ -1481,13 +1481,13 @@
         "useCpuNoise": "Dùng Độ Nhiễu CPU",
         "remixImage": "Phối Lại Hình Ảnh",
         "showOptionsPanel": "Hiển Thị Bảng Bên Cạnh (O hoặc T)",
-        "shuffle": "Xáo Trộn Tham Số Hạt Giống",
+        "shuffle": "Xáo Trộn",
         "setToOptimalSizeTooLarge": "$t(parameters.setToOptimalSize) (lớn quá)",
-        "cfgRescaleMultiplier": "CFG Rescale Multiplier",
+        "cfgRescaleMultiplier": "Hệ Số Nhân Thang CFG",
         "setToOptimalSizeTooSmall": "$t(parameters.setToOptimalSize) (nhỏ quá)",
         "images": "Ảnh Ban Đầu",
         "controlNetControlMode": "Chế Độ Điều Khiển",
-        "lockAspectRatio": "Khoá Tỉ Lệ Khung Hình",
+        "lockAspectRatio": "Khoá Tỉ Lệ",
         "swapDimensions": "Hoán Đổi Kích Thước",
         "copyImage": "Sao Chép Hình Ảnh",
         "downloadImage": "Tải Xuống Hình Ảnh",
@@ -1500,11 +1500,11 @@
     },
     "dynamicPrompts": {
         "seedBehaviour": {
-            "perIterationDesc": "Sử dụng tham số hạt giống khác nhau cho mỗi lần lặp lại",
-            "perPromptDesc": "Sử dụng tham số hạt giống khác nhau cho mỗi hình ảnh",
-            "label": "Hành Động Cho Tham Số Hạt Giống",
-            "perPromptLabel": "Tham Số Hạt Giống Mỗi Hình Ảnh",
-            "perIterationLabel": "Tham Số Hạt Giống Mỗi Lần Lặp Lại"
+            "perIterationDesc": "Sử dụng hạt giống khác nhau cho mỗi lần lặp lại",
+            "perPromptDesc": "Sử dụng hạt giống khác nhau cho mỗi hình ảnh",
+            "label": "Hành Động Cho Hạt Giống",
+            "perPromptLabel": "Một Hạt Giống Mỗi Ảnh",
+            "perIterationLabel": "Hạt Giống Mỗi Lần Lặp Lại"
         },
         "loading": "Tạo Sinh Dùng Dynamic Prompt...",
         "showDynamicPrompts": "HIện Dynamic Prompt",
@@ -1515,9 +1515,9 @@
     "settings": {
         "beta": "Beta",
         "general": "Cài Đặt Chung",
-        "confirmOnDelete": "Chắp Nhận Xoá",
+        "confirmOnDelete": "Xác Nhận Khi Xoá",
         "developer": "Nhà Phát Triển",
-        "confirmOnNewSession": "Chắp Nhận Mở Phiên Mới",
+        "confirmOnNewSession": "Xác Nhận Khi Mở Phiên Mới",
         "antialiasProgressImages": "Xử Lý Khử Răng Cưa Hình Ảnh",
         "models": "Models",
         "informationalPopoversDisabledDesc": "Hộp thoại hỗ trợ thông tin đã tắt. Bật lại trong Cài đặt.",
@@ -1549,20 +1549,20 @@
     },
     "sdxl": {
         "loading": "Đang Tải...",
-        "posAestheticScore": "Điểm Tích Cực Cho Tiêu Chuẩn",
-        "steps": "Tham Số Bước",
-        "refinerSteps": "Tham Số Bước Refiner",
+        "posAestheticScore": "Điểm Giống Tiêu Chuẩn",
+        "steps": "Số Bước",
+        "refinerSteps": "Số Bước Refiner",
         "refinermodel": "Model Refiner",
-        "refinerStart": "Nơi Bắt Đầu Refiner",
+        "refinerStart": "Bắt Đầu Refiner",
         "denoisingStrength": "Sức Mạnh Khử Nhiễu",
         "posStylePrompt": "Điểm Tích Cực Cho Lệnh Phong Cách",
         "scheduler": "Scheduler",
         "refiner": "Refiner",
-        "cfgScale": "Thước Đo CFG",
+        "cfgScale": "Thang CFG",
         "concatPromptStyle": "Liên Kết Lệnh & Phong Cách",
         "freePromptStyle": "Viết Lệnh Thủ Công Cho Phong Cách",
         "negStylePrompt": "Điểm Tiêu Cực Cho Lệnh Phong Cách",
-        "negAestheticScore": "Điểm Tiêu Cực Cho Tiêu Chuẩn",
+        "negAestheticScore": "Điểm Khác Tiêu Chuẩn",
         "noModelsAvailable": "Không có sẵn model"
     },
     "controlLayers": {
@@ -1992,7 +1992,8 @@
             "generation": "Generation",
             "system": "Hệ Thống",
             "canvas": "Canvas",
-            "logNamespaces": "Nơi Được Log"
+            "logNamespaces": "Vùng Ghi Log",
+            "dnd": "Kéo Thả"
         },
         "logLevel": {
             "logLevel": "Cấp Độ Log",
@@ -2156,8 +2157,7 @@
         "watchRecentReleaseVideos": "Xem Video Phát Hành Mới Nhất",
         "watchUiUpdatesOverview": "Xem Tổng Quan Về Những Cập Nhật Cho Giao Diện Người Dùng",
         "items": [
-            "<StrongComponent>Hướng Dẫn Khu Vực FLUX (beta)</StrongComponent>: Bản beta của Hướng Dẫn Khu Vực FLUX của chúng ta đã có mắt tại bảng điều khiển lệnh khu vực.",
-            "<StrongComponent>Nhiều Cải Tiến Ở UX</StrongComponent>: Một số nâng cấp nhỏ ở trải nghiệm và chất lượng người dùng trên toàn bộ ứng dụng."
+            "<StrongComponent>Hướng Dẫn Khu Vực FLUX (beta)</StrongComponent>: Bản beta của Hướng Dẫn Khu Vực FLUX của chúng ta đã có mắt tại bảng điều khiển lệnh khu vực."
         ]
     },
     "upsell": {

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/enqueueRequestedLinear.ts

@@ -2,7 +2,6 @@ import { logger } from 'app/logging/logger';
 import { enqueueRequested } from 'app/store/actions';
 import type { AppStartListening } from 'app/store/middleware/listenerMiddleware';
 import { extractMessageFromAssertionError } from 'common/util/extractMessageFromAssertionError';
-import type { Result } from 'common/util/result';
 import { withResult, withResultAsync } from 'common/util/result';
 import { $canvasManager } from 'features/controlLayers/store/ephemeral';
 import { prepareLinearUIBatch } from 'features/nodes/util/graph/buildLinearBatchConfig';
@@ -10,11 +9,9 @@ import { buildFLUXGraph } from 'features/nodes/util/graph/generation/buildFLUXGr
 import { buildSD1Graph } from 'features/nodes/util/graph/generation/buildSD1Graph';
 import { buildSD3Graph } from 'features/nodes/util/graph/generation/buildSD3Graph';
 import { buildSDXLGraph } from 'features/nodes/util/graph/generation/buildSDXLGraph';
-import type { Graph } from 'features/nodes/util/graph/generation/Graph';
 import { toast } from 'features/toast/toast';
 import { serializeError } from 'serialize-error';
 import { enqueueMutationFixedCacheKeyOptions, queueApi } from 'services/api/endpoints/queue';
-import type { Invocation } from 'services/api/types';
 import { assert, AssertionError } from 'tsafe';
 import type { JsonObject } from 'type-fest';
 
@@ -25,42 +22,32 @@ export const addEnqueueRequestedLinear = (startAppListening: AppStartListening)
     predicate: (action): action is ReturnType<typeof enqueueRequested> =>
       enqueueRequested.match(action) && action.payload.tabName === 'canvas',
     effect: async (action, { getState, dispatch }) => {
+      log.debug('Enqueue requested');
       const state = getState();
-      const model = state.params.model;
       const { prepend } = action.payload;
 
       const manager = $canvasManager.get();
-      assert(manager, 'No model found in state');
-
-      let buildGraphResult: Result<
-        {
-          g: Graph;
-          noise: Invocation<'noise' | 'flux_denoise' | 'sd3_denoise'>;
-          posCond: Invocation<'compel' | 'sdxl_compel_prompt' | 'flux_text_encoder' | 'sd3_text_encoder'>;
-        },
-        Error
-      >;
+      assert(manager, 'No canvas manager');
 
+      const model = state.params.model;
       assert(model, 'No model found in state');
       const base = model.base;
 
-      switch (base) {
-        case 'sdxl':
-          buildGraphResult = await withResultAsync(() => buildSDXLGraph(state, manager));
-          break;
-        case 'sd-1':
-        case `sd-2`:
-          buildGraphResult = await withResultAsync(() => buildSD1Graph(state, manager));
-          break;
-        case `sd-3`:
-          buildGraphResult = await withResultAsync(() => buildSD3Graph(state, manager));
-          break;
-        case `flux`:
-          buildGraphResult = await withResultAsync(() => buildFLUXGraph(state, manager));
-          break;
-        default:
-          assert(false, `No graph builders for base ${base}`);
-      }
+      const buildGraphResult = await withResultAsync(async () => {
+        switch (base) {
+          case 'sdxl':
+            return await buildSDXLGraph(state, manager);
+          case 'sd-1':
+          case `sd-2`:
+            return await buildSD1Graph(state, manager);
+          case `sd-3`:
+            return await buildSD3Graph(state, manager);
+          case `flux`:
+            return await buildFLUXGraph(state, manager);
+          default:
+            assert(false, `No graph builders for base ${base}`);
+        }
+      });
 
       if (buildGraphResult.isErr()) {
         let description: string | null = null;

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelsLoaded.ts

@@ -30,7 +30,7 @@ import { modelConfigsAdapterSelectors, modelsApi } from 'services/api/endpoints/
 import type { AnyModelConfig } from 'services/api/types';
 import {
   isCLIPEmbedModelConfig,
-  isControlNetOrT2IAdapterModelConfig,
+  isControlLayerModelConfig,
   isFluxVAEModelConfig,
   isIPAdapterModelConfig,
   isLoRAModelConfig,
@@ -190,7 +190,7 @@ const handleLoRAModels: ModelHandler = (models, state, dispatch, log) => {
 };
 
 const handleControlAdapterModels: ModelHandler = (models, state, dispatch, log) => {
-  const caModels = models.filter(isControlNetOrT2IAdapterModelConfig);
+  const caModels = models.filter(isControlLayerModelConfig);
   selectCanvasSlice(state).controlLayers.entities.forEach((entity) => {
     const selectedControlAdapterModel = entity.controlAdapter.model;
     // `null` is a valid control adapter model - no need to do anything.

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/socketConnected.ts

@@ -17,8 +17,6 @@ export const addSocketConnectedEventListener = (startAppListening: AppStartListe
   startAppListening({
     actionCreator: socketConnected,
     effect: async (action, { dispatch, getState, cancelActiveListeners, delay }) => {
-      log.debug('Connected');
-
       /**
        * The rest of this listener has recovery logic for when the socket disconnects and reconnects.
        *

invokeai/frontend/web/src/common/util/result.ts

@@ -57,7 +57,7 @@ export class Err<E> {
  * @template T The type of the value in the `Ok` case.
  * @template E The type of the error in the `Err` case.
  */
-export type Result<T, E = Error> = Ok<T> | Err<E>;
+type Result<T, E = Error> = Ok<T> | Err<E>;
 
 /**
  * Creates a successful result.

invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayerControlAdapter.tsx

@@ -26,7 +26,12 @@ import { replaceCanvasEntityObjectsWithImage } from 'features/imageActions/actio
 import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { PiBoundingBoxBold, PiShootingStarFill, PiUploadBold } from 'react-icons/pi';
-import type { ControlNetModelConfig, ImageDTO, T2IAdapterModelConfig } from 'services/api/types';
+import type {
+  ControlLoRAModelConfig,
+  ControlNetModelConfig,
+  ImageDTO,
+  T2IAdapterModelConfig,
+} from 'services/api/types';
 
 const buildSelectControlAdapter = (entityIdentifier: CanvasEntityIdentifier<'control_layer'>) =>
   createMemoizedAppSelector(selectCanvasSlice, (canvas) => {
@@ -66,7 +71,7 @@ export const ControlLayerControlAdapter = memo(() => {
   );
 
   const onChangeModel = useCallback(
-    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig) => {
+    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig) => {
       dispatch(controlLayerModelChanged({ entityIdentifier, modelConfig }));
       // When we change the model, we need may need to start filtering w/ the simplified filter mode, and/or change the
       // filter config.
@@ -97,12 +102,12 @@ export const ControlLayerControlAdapter = memo(() => {
       const filterConfig = defaultFilterForNewModel.buildDefaults();
       if (isFiltering) {
         adapter.filterer.$filterConfig.set(filterConfig);
+        // The user may have disabled auto-processing, so we should process the filter manually. This is essentially a
+        // no-op if auto-processing is already enabled, because the process method is debounced.
+        adapter.filterer.process();
       } else {
         adapter.filterer.start(filterConfig);
       }
-      // The user may have disabled auto-processing, so we should process the filter manually. This is essentially a
-      // no-op if auto-processing is already enabled, because the process method is debounced.
-      adapter.filterer.process();
     },
     [adapter.filterer, dispatch, entityIdentifier]
   );
@@ -158,7 +163,9 @@ export const ControlLayerControlAdapter = memo(() => {
         <input {...uploadApi.getUploadInputProps()} />
       </Flex>
       <Weight weight={controlAdapter.weight} onChange={onChangeWeight} />
-      <BeginEndStepPct beginEndStepPct={controlAdapter.beginEndStepPct} onChange={onChangeBeginEndStepPct} />
+      {controlAdapter.type !== 'control_lora' && (
+        <BeginEndStepPct beginEndStepPct={controlAdapter.beginEndStepPct} onChange={onChangeBeginEndStepPct} />
+      )}
       {controlAdapter.type === 'controlnet' && !isFLUX && (
         <ControlLayerControlAdapterControlMode
           controlMode={controlAdapter.controlMode}

invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayerControlAdapterModel.tsx

@@ -4,22 +4,27 @@ import { useGroupedModelCombobox } from 'common/hooks/useGroupedModelCombobox';
 import { selectBase } from 'features/controlLayers/store/paramsSlice';
 import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { useControlNetAndT2IAdapterModels } from 'services/api/hooks/modelsByType';
-import type { AnyModelConfig, ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
+import { useControlLayerModels } from 'services/api/hooks/modelsByType';
+import type {
+  AnyModelConfig,
+  ControlLoRAModelConfig,
+  ControlNetModelConfig,
+  T2IAdapterModelConfig,
+} from 'services/api/types';
 
 type Props = {
   modelKey: string | null;
-  onChange: (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig) => void;
+  onChange: (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig) => void;
 };
 
 export const ControlLayerControlAdapterModel = memo(({ modelKey, onChange: onChangeModel }: Props) => {
   const { t } = useTranslation();
   const currentBaseModel = useAppSelector(selectBase);
-  const [modelConfigs, { isLoading }] = useControlNetAndT2IAdapterModels();
+  const [modelConfigs, { isLoading }] = useControlLayerModels();
   const selectedModel = useMemo(() => modelConfigs.find((m) => m.key === modelKey), [modelConfigs, modelKey]);
 
   const _onChange = useCallback(
-    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | null) => {
+    (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig | null) => {
       if (!modelConfig) {
         return;
       }

invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts

@@ -18,6 +18,7 @@ import { selectCanvasSlice, selectEntity } from 'features/controlLayers/store/se
 import type {
   CanvasEntityIdentifier,
   CanvasRegionalGuidanceState,
+  ControlLoRAConfig,
   ControlNetConfig,
   IPAdapterConfig,
   T2IAdapterConfig,
@@ -26,8 +27,13 @@ import { initialControlNet, initialIPAdapter, initialT2IAdapter } from 'features
 import { zModelIdentifierField } from 'features/nodes/types/common';
 import { useCallback } from 'react';
 import { modelConfigsAdapterSelectors, selectModelConfigsQuery } from 'services/api/endpoints/models';
-import type { ControlNetModelConfig, IPAdapterModelConfig, T2IAdapterModelConfig } from 'services/api/types';
-import { isControlNetOrT2IAdapterModelConfig, isIPAdapterModelConfig } from 'services/api/types';
+import type {
+  ControlLoRAModelConfig,
+  ControlNetModelConfig,
+  IPAdapterModelConfig,
+  T2IAdapterModelConfig,
+} from 'services/api/types';
+import { isControlLayerModelConfig, isIPAdapterModelConfig } from 'services/api/types';
 
 /**
  * Selects the default control adapter configuration based on the model configurations and the base.
@@ -39,13 +45,13 @@ import { isControlNetOrT2IAdapterModelConfig, isIPAdapterModelConfig } from 'ser
 export const selectDefaultControlAdapter = createSelector(
   selectModelConfigsQuery,
   selectBase,
-  (query, base): ControlNetConfig | T2IAdapterConfig => {
+  (query, base): ControlNetConfig | T2IAdapterConfig | ControlLoRAConfig => {
     const { data } = query;
-    let model: ControlNetModelConfig | T2IAdapterModelConfig | null = null;
+    let model: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig | null = null;
     if (data) {
       const modelConfigs = modelConfigsAdapterSelectors
         .selectAll(data)
-        .filter(isControlNetOrT2IAdapterModelConfig)
+        .filter(isControlLayerModelConfig)
         .sort((a) => (a.type === 'controlnet' ? -1 : 1)); // Prefer ControlNet models
       const compatibleModels = modelConfigs.filter((m) => (base ? m.base === base : true));
       model = compatibleModels[0] ?? modelConfigs[0] ?? null;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts

@@ -1,4 +1,4 @@
-import { withResultAsync } from 'common/util/result';
+import { withResult, withResultAsync } from 'common/util/result';
 import { CanvasCacheModule } from 'features/controlLayers/konva/CanvasCacheModule';
 import type { CanvasEntityAdapter, CanvasEntityAdapterFromType } from 'features/controlLayers/konva/CanvasEntity/types';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
@@ -277,13 +277,19 @@ export class CanvasCompositorModule extends CanvasModuleBase {
       this.log.warn({ rect, imageName: cachedImageName }, 'Cached image name not found, recompositing');
     }
 
-    const canvas = this.getCompositeCanvas(adapters, rect, compositingOptions);
+    const getCompositeCanvasResult = withResult(() => this.getCompositeCanvas(adapters, rect, compositingOptions));
+
+    if (getCompositeCanvasResult.isErr()) {
+      this.log.error({ error: serializeError(getCompositeCanvasResult.error) }, 'Failed to get composite canvas');
+      throw getCompositeCanvasResult.error;
+    }
 
     this.$isProcessing.set(true);
-    const blobResult = await withResultAsync(() => canvasToBlob(canvas));
+    const blobResult = await withResultAsync(() => canvasToBlob(getCompositeCanvasResult.value));
     this.$isProcessing.set(false);
 
     if (blobResult.isErr()) {
+      this.log.error({ error: serializeError(blobResult.error) }, 'Failed to convert composite canvas to blob');
       throw blobResult.error;
     }
     const blob = blobResult.value;
@@ -489,37 +495,45 @@ export class CanvasCompositorModule extends CanvasModuleBase {
     this.log.debug({ rect }, 'Calculating generation mode');
 
     this.$isProcessing.set(true);
-    const compositeRasterLayerTransparency = await this.getTransparency(
-      rasterLayerAdapters,
-      rect,
-      compositeRasterLayerHash
-    );
+    const generationModeResult = await withResultAsync(async () => {
+      const compositeRasterLayerTransparency = await this.getTransparency(
+        rasterLayerAdapters,
+        rect,
+        compositeRasterLayerHash
+      );
+
+      const compositeInpaintMaskTransparency = await this.getTransparency(
+        inpaintMaskAdapters,
+        rect,
+        compositeInpaintMaskHash
+      );
+
+      let generationMode: GenerationMode;
+      if (compositeRasterLayerTransparency === 'FULLY_TRANSPARENT') {
+        // When the initial image is fully transparent, we are always doing txt2img
+        generationMode = 'txt2img';
+      } else if (compositeRasterLayerTransparency === 'PARTIALLY_TRANSPARENT') {
+        // When the initial image is partially transparent, we are always outpainting
+        generationMode = 'outpaint';
+      } else if (compositeInpaintMaskTransparency === 'FULLY_TRANSPARENT') {
+        // compositeLayerTransparency === 'OPAQUE'
+        // When the inpaint mask is fully transparent, we are doing img2img
+        generationMode = 'img2img';
+      } else {
+        // Else at least some of the inpaint mask is opaque, so we are inpainting
+        generationMode = 'inpaint';
+      }
+
+      this.manager.cache.generationModeCache.set(hash, generationMode);
+      return generationMode;
+    });
 
-    const compositeInpaintMaskTransparency = await this.getTransparency(
-      inpaintMaskAdapters,
-      rect,
-      compositeInpaintMaskHash
-    );
     this.$isProcessing.set(false);
-
-    let generationMode: GenerationMode;
-    if (compositeRasterLayerTransparency === 'FULLY_TRANSPARENT') {
-      // When the initial image is fully transparent, we are always doing txt2img
-      generationMode = 'txt2img';
-    } else if (compositeRasterLayerTransparency === 'PARTIALLY_TRANSPARENT') {
-      // When the initial image is partially transparent, we are always outpainting
-      generationMode = 'outpaint';
-    } else if (compositeInpaintMaskTransparency === 'FULLY_TRANSPARENT') {
-      // compositeLayerTransparency === 'OPAQUE'
-      // When the inpaint mask is fully transparent, we are doing img2img
-      generationMode = 'img2img';
-    } else {
-      // Else at least some of the inpaint mask is opaque, so we are inpainting
-      generationMode = 'inpaint';
+    if (generationModeResult.isErr()) {
+      this.log.error({ error: serializeError(generationModeResult.error) }, 'Failed to calculate generation mode');
+      throw generationModeResult.error;
     }
-
-    this.manager.cache.generationModeCache.set(hash, generationMode);
-    return generationMode;
+    return generationModeResult.value;
   };
 
   repr = () => {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterBase.ts

@@ -2,6 +2,7 @@ import type { Selector } from '@reduxjs/toolkit';
 import { createSelector } from '@reduxjs/toolkit';
 import type { RootState } from 'app/store/store';
 import { deepClone } from 'common/util/deepClone';
+import { withResultAsync } from 'common/util/result';
 import type { CanvasEntityBufferObjectRenderer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityBufferObjectRenderer';
 import type { CanvasEntityFilterer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityFilterer';
 import type { CanvasEntityObjectRenderer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer';
@@ -29,6 +30,7 @@ import {
   isRasterLayerEntityIdentifier,
   type Rect,
 } from 'features/controlLayers/store/types';
+import { toast } from 'features/toast/toast';
 import Konva from 'konva';
 import { atom } from 'nanostores';
 import rafThrottle from 'raf-throttle';
@@ -574,9 +576,16 @@ export abstract class CanvasEntityAdapterBase<
     return stableHash(arg);
   };
 
-  cropToBbox = (): Promise<ImageDTO> => {
+  cropToBbox = async (): Promise<ImageDTO> => {
     const { rect } = this.manager.stateApi.getBbox();
-    return this.renderer.rasterize({ rect, replaceObjects: true, attrs: { opacity: 1, filters: [] } });
+    const rasterizeResult = await withResultAsync(() =>
+      this.renderer.rasterize({ rect, replaceObjects: true, attrs: { opacity: 1, filters: [] } })
+    );
+    if (rasterizeResult.isErr()) {
+      toast({ status: 'error', title: 'Failed to crop to bbox' });
+      throw rasterizeResult.error;
+    }
+    return rasterizeResult.value;
   };
 
   destroy = (): void => {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityFilterer.ts

@@ -11,13 +11,14 @@ import type { FilterConfig } from 'features/controlLayers/store/filters';
 import { getFilterForModel, IMAGE_FILTERS } from 'features/controlLayers/store/filters';
 import type { CanvasImageState, CanvasRenderableEntityType } from 'features/controlLayers/store/types';
 import { imageDTOToImageObject } from 'features/controlLayers/store/util';
+import { toast } from 'features/toast/toast';
 import Konva from 'konva';
 import { debounce } from 'lodash-es';
 import { atom, computed } from 'nanostores';
 import type { Logger } from 'roarr';
 import { serializeError } from 'serialize-error';
 import { buildSelectModelConfig } from 'services/api/hooks/modelsByType';
-import { isControlNetOrT2IAdapterModelConfig } from 'services/api/types';
+import { isControlLayerModelConfig } from 'services/api/types';
 import stableHash from 'stable-hash';
 import type { Equals } from 'tsafe';
 import { assert } from 'tsafe';
@@ -203,7 +204,7 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
       // If the parent is a control layer adapter, we should check if the model has a default filter and set it if so
       const selectModelConfig = buildSelectModelConfig(
         this.parent.state.controlAdapter.model.key,
-        isControlNetOrT2IAdapterModelConfig
+        isControlLayerModelConfig
       );
       const modelConfig = this.manager.stateApi.runSelector(selectModelConfig);
       // This always returns a filter
@@ -246,6 +247,7 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
       this.parent.renderer.rasterize({ rect, attrs: { filters: [], opacity: 1 } })
     );
     if (rasterizeResult.isErr()) {
+      toast({ status: 'error', title: 'Failed to process filter' });
       this.log.error({ error: serializeError(rasterizeResult.error) }, 'Error rasterizing entity');
       this.$isProcessing.set(false);
       return;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts

@@ -27,6 +27,7 @@ import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
 import { throttle } from 'lodash-es';
 import type { Logger } from 'roarr';
+import { serializeError } from 'serialize-error';
 import { getImageDTOSafe, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
 import { assert } from 'tsafe';
@@ -485,30 +486,36 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase {
     this.log.trace({ rasterizeArgs }, 'Rasterizing entity');
     this.manager.stateApi.$rasterizingAdapter.set(this.parent);
 
-    const blob = await this.getBlob(rasterizeArgs);
-    if (this.manager._isDebugging) {
-      previewBlob(blob, 'Rasterized entity');
+    try {
+      const blob = await this.getBlob(rasterizeArgs);
+      if (this.manager._isDebugging) {
+        previewBlob(blob, 'Rasterized entity');
+      }
+      imageDTO = await uploadImage({
+        file: new File([blob], `${this.id}_rasterized.png`, { type: 'image/png' }),
+        image_category: 'other',
+        is_intermediate: true,
+        silent: true,
+      });
+      const imageObject = imageDTOToImageObject(imageDTO);
+      if (replaceObjects) {
+        await this.parent.bufferRenderer.setBuffer(imageObject);
+        this.parent.bufferRenderer.commitBuffer({ pushToState: false });
+      }
+      this.manager.stateApi.rasterizeEntity({
+        entityIdentifier: this.parent.entityIdentifier,
+        imageObject,
+        position: { x: Math.round(rect.x), y: Math.round(rect.y) },
+        replaceObjects,
+      });
+      this.manager.cache.imageNameCache.set(hash, imageDTO.image_name);
+      return imageDTO;
+    } catch (error) {
+      this.log.error({ rasterizeArgs, error: serializeError(error) }, 'Failed to rasterize entity');
+      throw error;
+    } finally {
+      this.manager.stateApi.$rasterizingAdapter.set(null);
     }
-    imageDTO = await uploadImage({
-      file: new File([blob], `${this.id}_rasterized.png`, { type: 'image/png' }),
-      image_category: 'other',
-      is_intermediate: true,
-      silent: true,
-    });
-    const imageObject = imageDTOToImageObject(imageDTO);
-    if (replaceObjects) {
-      await this.parent.bufferRenderer.setBuffer(imageObject);
-      this.parent.bufferRenderer.commitBuffer({ pushToState: false });
-    }
-    this.manager.stateApi.rasterizeEntity({
-      entityIdentifier: this.parent.entityIdentifier,
-      imageObject,
-      position: { x: Math.round(rect.x), y: Math.round(rect.y) },
-      replaceObjects,
-    });
-    this.manager.cache.imageNameCache.set(hash, imageDTO.image_name);
-    this.manager.stateApi.$rasterizingAdapter.set(null);
-    return imageDTO;
   };
 
   cloneObjectGroup = (arg: { attrs?: GroupConfig } = {}): Konva.Group => {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts

@@ -13,6 +13,7 @@ import {
 } from 'features/controlLayers/konva/util';
 import { selectSelectedEntityIdentifier } from 'features/controlLayers/store/selectors';
 import type { Coordinate, Rect, RectWithRotation } from 'features/controlLayers/store/types';
+import { toast } from 'features/toast/toast';
 import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
 import { clamp, debounce, get } from 'lodash-es';
@@ -779,6 +780,7 @@ export class CanvasEntityTransformer extends CanvasModuleBase {
       })
     );
     if (rasterizeResult.isErr()) {
+      toast({ status: 'error', title: 'Failed to apply transform' });
       this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity');
     }
     this.requestRectCalculation();

invokeai/frontend/web/src/features/controlLayers/konva/CanvasSegmentAnythingModule.ts

@@ -26,6 +26,7 @@ import type {
 import { SAM_POINT_LABEL_NUMBER_TO_STRING } from 'features/controlLayers/store/types';
 import { imageDTOToImageObject } from 'features/controlLayers/store/util';
 import { Graph } from 'features/nodes/util/graph/generation/Graph';
+import { toast } from 'features/toast/toast';
 import Konva from 'konva';
 import type { KonvaEventObject } from 'konva/lib/Node';
 import { debounce } from 'lodash-es';
@@ -571,6 +572,7 @@ export class CanvasSegmentAnythingModule extends CanvasModuleBase {
     );
 
     if (rasterizeResult.isErr()) {
+      toast({ status: 'error', title: 'Failed to select object' });
       this.log.error({ error: serializeError(rasterizeResult.error) }, 'Error rasterizing entity');
       this.$isProcessing.set(false);
       return;

invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts

@@ -18,6 +18,7 @@ import type {
   CanvasEntityType,
   CanvasInpaintMaskState,
   CanvasMetadata,
+  ControlLoRAConfig,
   EntityMovedByPayload,
   FillStyle,
   RegionalGuidanceReferenceImageState,
@@ -34,7 +35,13 @@ import { getGridSize, getIsSizeOptimal, getOptimalDimension } from 'features/par
 import type { IRect } from 'konva/lib/types';
 import { merge } from 'lodash-es';
 import type { UndoableOptions } from 'redux-undo';
-import type { ControlNetModelConfig, ImageDTO, IPAdapterModelConfig, T2IAdapterModelConfig } from 'services/api/types';
+import type {
+  ControlLoRAModelConfig,
+  ControlNetModelConfig,
+  ImageDTO,
+  IPAdapterModelConfig,
+  T2IAdapterModelConfig,
+} from 'services/api/types';
 import { assert } from 'tsafe';
 
 import type {
@@ -67,7 +74,10 @@ import {
   getReferenceImageState,
   getRegionalGuidanceState,
   imageDTOToImageWithDims,
+  initialControlLoRA,
+  initialControlNet,
   initialIPAdapter,
+  initialT2IAdapter,
 } from './util';
 
 const getInitialState = (): CanvasState => {
@@ -436,7 +446,7 @@ export const canvasSlice = createSlice({
       action: PayloadAction<
         EntityIdentifierPayload<
           {
-            modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | null;
+            modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig | null;
           },
           'control_layer'
         >
@@ -453,20 +463,69 @@ export const canvasSlice = createSlice({
       }
       layer.controlAdapter.model = zModelIdentifierField.parse(modelConfig);
 
-      // We may need to convert the CA to match the model
-      if (layer.controlAdapter.type === 't2i_adapter' && layer.controlAdapter.model.type === 'controlnet') {
-        // Converting from T2I Adapter to ControlNet - add `controlMode`
-        const controlNetConfig: ControlNetConfig = {
-          ...layer.controlAdapter,
-          type: 'controlnet',
-          controlMode: 'balanced',
-        };
-        layer.controlAdapter = controlNetConfig;
-      } else if (layer.controlAdapter.type === 'controlnet' && layer.controlAdapter.model.type === 't2i_adapter') {
-        // Converting from ControlNet to T2I Adapter - remove `controlMode`
-        const { controlMode: _, ...rest } = layer.controlAdapter;
-        const t2iAdapterConfig: T2IAdapterConfig = { ...rest, type: 't2i_adapter' };
-        layer.controlAdapter = t2iAdapterConfig;
+      // When converting between control layer types, we may need to add or remove properties. For example, ControlNet
+      // has a control mode, while T2I Adapter does not - otherwise they are the same.
+
+      switch (layer.controlAdapter.model.type) {
+        // Converting to T2I adapter from...
+        case 't2i_adapter': {
+          if (layer.controlAdapter.type === 'controlnet') {
+            // T2I Adapters have all the ControlNet properties, minus control mode - strip it
+            const { controlMode: _, ...rest } = layer.controlAdapter;
+            const t2iAdapterConfig: T2IAdapterConfig = { ...initialT2IAdapter, ...rest, type: 't2i_adapter' };
+            layer.controlAdapter = t2iAdapterConfig;
+          } else if (layer.controlAdapter.type === 'control_lora') {
+            // Control LoRAs have only model and weight
+            const t2iAdapterConfig: T2IAdapterConfig = {
+              ...initialT2IAdapter,
+              ...layer.controlAdapter,
+              type: 't2i_adapter',
+            };
+            layer.controlAdapter = t2iAdapterConfig;
+          }
+          break;
+        }
+
+        // Converting to ControlNet from...
+        case 'controlnet': {
+          if (layer.controlAdapter.type === 't2i_adapter') {
+            // ControlNets have all the T2I Adapter properties, plus control mode
+            const controlNetConfig: ControlNetConfig = {
+              ...initialControlNet,
+              ...layer.controlAdapter,
+              type: 'controlnet',
+            };
+            layer.controlAdapter = controlNetConfig;
+          } else if (layer.controlAdapter.type === 'control_lora') {
+            // ControlNets have all the Control LoRA properties, plus control mode and begin/end step pct
+            const controlNetConfig: ControlNetConfig = {
+              ...initialControlNet,
+              ...layer.controlAdapter,
+              type: 'controlnet',
+            };
+            layer.controlAdapter = controlNetConfig;
+          }
+          break;
+        }
+
+        // Converting to ControlLoRA from...
+        case 'control_lora': {
+          if (layer.controlAdapter.type === 'controlnet') {
+            // We only need the model and weight for Control LoRA
+            const { model, weight } = layer.controlAdapter;
+            const controlNetConfig: ControlLoRAConfig = { ...initialControlLoRA, model, weight };
+            layer.controlAdapter = controlNetConfig;
+          } else if (layer.controlAdapter.type === 't2i_adapter') {
+            // We only need the model and weight for Control LoRA
+            const { model, weight } = layer.controlAdapter;
+            const t2iAdapterConfig: ControlLoRAConfig = { ...initialControlLoRA, model, weight };
+            layer.controlAdapter = t2iAdapterConfig;
+          }
+          break;
+        }
+
+        default:
+          break;
       }
     },
     controlLayerControlModeChanged: (
@@ -497,7 +556,7 @@ export const canvasSlice = createSlice({
     ) => {
       const { entityIdentifier, beginEndStepPct } = action.payload;
       const layer = selectEntity(state, entityIdentifier);
-      if (!layer || !layer.controlAdapter) {
+      if (!layer || !layer.controlAdapter || layer.controlAdapter.type === 'control_lora') {
         return;
       }
       layer.controlAdapter.beginEndStepPct = beginEndStepPct;

invokeai/frontend/web/src/features/controlLayers/store/filters.ts

@@ -2,7 +2,7 @@ import { getPrefixedId } from 'features/controlLayers/konva/util';
 import type { ImageWithDims } from 'features/controlLayers/store/types';
 import { zModelIdentifierField } from 'features/nodes/types/common';
 import { Graph } from 'features/nodes/util/graph/generation/Graph';
-import type { ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
+import type { ControlLoRAModelConfig, ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
 import { assert } from 'tsafe';
 import { z } from 'zod';
 
@@ -454,7 +454,9 @@ const PROCESSOR_TO_FILTER_MAP: Record<string, FilterType> = {
  * Gets the default filter for a control model. If the model has a default, it will be used, otherwise the default
  * filter for the model type will be used.
  */
-export const getFilterForModel = (modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | null) => {
+export const getFilterForModel = (
+  modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig | null
+) => {
   if (!modelConfig) {
     // No model
     return null;

invokeai/frontend/web/src/features/controlLayers/store/paramsSlice.ts

@@ -11,6 +11,7 @@ import type {
   ParameterCLIPEmbedModel,
   ParameterCLIPGEmbedModel,
   ParameterCLIPLEmbedModel,
+  ParameterControlLoRAModel,
   ParameterGuidance,
   ParameterMaskBlurMethod,
   ParameterModel,
@@ -75,6 +76,7 @@ export type ParamsState = {
   clipEmbedModel: ParameterCLIPEmbedModel | null;
   clipLEmbedModel: ParameterCLIPLEmbedModel | null;
   clipGEmbedModel: ParameterCLIPGEmbedModel | null;
+  controlLora: ParameterControlLoRAModel | null;
 };
 
 const initialState: ParamsState = {
@@ -121,6 +123,7 @@ const initialState: ParamsState = {
   clipEmbedModel: null,
   clipLEmbedModel: null,
   clipGEmbedModel: null,
+  controlLora: null,
 };
 
 export const paramsSlice = createSlice({
@@ -195,6 +198,9 @@ export const paramsSlice = createSlice({
     t5EncoderModelSelected: (state, action: PayloadAction<ParameterT5EncoderModel | null>) => {
       state.t5EncoderModel = action.payload;
     },
+    controlLoRAModelSelected: (state, action: PayloadAction<ParameterControlLoRAModel | null>) => {
+      state.controlLora = action.payload;
+    },
     clipEmbedModelSelected: (state, action: PayloadAction<ParameterCLIPEmbedModel | null>) => {
       state.clipEmbedModel = action.payload;
     },

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -296,6 +296,13 @@ const zT2IAdapterConfig = z.object({
 });
 export type T2IAdapterConfig = z.infer<typeof zT2IAdapterConfig>;
 
+const zControlLoRAConfig = z.object({
+  type: z.literal('control_lora'),
+  weight: z.number().gte(-1).lte(2),
+  model: zServerValidatedModelIdentifierField.nullable(),
+});
+export type ControlLoRAConfig = z.infer<typeof zControlLoRAConfig>;
+
 export const zCanvasRasterLayerState = zCanvasEntityBase.extend({
   type: z.literal('raster_layer'),
   position: zCoordinate,
@@ -307,7 +314,7 @@ export type CanvasRasterLayerState = z.infer<typeof zCanvasRasterLayerState>;
 const zCanvasControlLayerState = zCanvasRasterLayerState.extend({
   type: z.literal('control_layer'),
   withTransparencyEffect: z.boolean(),
-  controlAdapter: z.discriminatedUnion('type', [zControlNetConfig, zT2IAdapterConfig]),
+  controlAdapter: z.discriminatedUnion('type', [zControlNetConfig, zT2IAdapterConfig, zControlLoRAConfig]),
 });
 export type CanvasControlLayerState = z.infer<typeof zCanvasControlLayerState>;
 

invokeai/frontend/web/src/features/controlLayers/store/util.ts

@@ -7,6 +7,7 @@ import type {
   CanvasRasterLayerState,
   CanvasReferenceImageState,
   CanvasRegionalGuidanceState,
+  ControlLoRAConfig,
   ControlNetConfig,
   ImageWithDims,
   IPAdapterConfig,
@@ -82,6 +83,11 @@ export const initialControlNet: ControlNetConfig = {
   beginEndStepPct: [0, 0.75],
   controlMode: 'balanced',
 };
+export const initialControlLoRA: ControlLoRAConfig = {
+  type: 'control_lora',
+  model: null,
+  weight: 0.75,
+};
 
 export const getReferenceImageState = (
   id: string,

invokeai/frontend/web/src/features/gallery/components/ImageContextMenu/ImageMenuItemNewLayerFromImageSubMenu.tsx

@@ -77,7 +77,7 @@ export const ImageMenuItemNewLayerFromImageSubMenu = memo(() => {
     });
   }, [imageDTO, imageViewer, store, t]);
 
-  const onClickNewRegionalReferenceImageFromImage = useCallback(() => {
+  const onClickNewGlobalReferenceImageFromImage = useCallback(() => {
     const { dispatch, getState } = store;
     createNewCanvasEntityFromImage({ imageDTO, type: 'reference_image', dispatch, getState });
     dispatch(sentImageToCanvas());
@@ -90,7 +90,7 @@ export const ImageMenuItemNewLayerFromImageSubMenu = memo(() => {
     });
   }, [imageDTO, imageViewer, store, t]);
 
-  const onClickNewGlobalReferenceImageFromImage = useCallback(() => {
+  const onClickNewRegionalReferenceImageFromImage = useCallback(() => {
     const { dispatch, getState } = store;
     createNewCanvasEntityFromImage({ imageDTO, type: 'regional_guidance_with_reference_image', dispatch, getState });
     dispatch(sentImageToCanvas());

invokeai/frontend/web/src/features/metadata/util/parsers.ts

@@ -27,6 +27,7 @@ import { zControlField, zIPAdapterField, zModelIdentifierField, zT2IAdapterField
 import type {
   ParameterCFGRescaleMultiplier,
   ParameterCFGScale,
+  ParameterControlLoRAModel,
   ParameterGuidance,
   ParameterHeight,
   ParameterHRFEnabled,
@@ -75,6 +76,7 @@ import {
 import { get, isArray, isString } from 'lodash-es';
 import { getImageDTOSafe } from 'services/api/endpoints/images';
 import {
+  isControlLoRAModelConfig,
   isControlNetModelConfig,
   isIPAdapterModelConfig,
   isLoRAModelConfig,
@@ -226,6 +228,14 @@ const parseVAEModel: MetadataParseFunc<ParameterVAEModel> = async (metadata) =>
   return modelIdentifier;
 };
 
+const parseControlLoRAModel: MetadataParseFunc<ParameterControlLoRAModel> = async (metadata) => {
+  const slora = await getProperty(metadata, 'control_lora', undefined);
+  const key = await getModelKey(slora, 'control_lora');
+  const sloraModelConfig = await fetchModelConfigWithTypeGuard(key, isControlLoRAModelConfig);
+  const modelIdentifier = zModelIdentifierField.parse(sloraModelConfig);
+  return modelIdentifier;
+};
+
 const parseLoRA: MetadataParseFunc<LoRA> = async (metadataItem) => {
   // Previously, the LoRA model identifier parts were stored in the LoRA metadata: `{key: ..., weight: 0.75}`
   const modelV1 = await getProperty(metadataItem, 'lora', undefined);
@@ -671,6 +681,7 @@ export const parsers = {
   mainModel: parseMainModel,
   refinerModel: parseRefinerModel,
   vaeModel: parseVAEModel,
+  controlLora: parseControlLoRAModel,
   lora: parseLoRA,
   loras: parseAllLoRAs,
   controlNet: parseControlNet,

invokeai/frontend/web/src/features/modelManagerV2/hooks/useControlAdapterModelDefaultSettings.ts

@@ -1,9 +1,9 @@
 import { isNil } from 'lodash-es';
 import { useMemo } from 'react';
-import type { ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
+import type { ControlLoRAModelConfig, ControlNetModelConfig, T2IAdapterModelConfig } from 'services/api/types';
 
-export const useControlNetOrT2IAdapterDefaultSettings = (
-  modelConfig: ControlNetModelConfig | T2IAdapterModelConfig
+export const useControlAdapterModelDefaultSettings = (
+  modelConfig: ControlNetModelConfig | T2IAdapterModelConfig | ControlLoRAModelConfig
 ) => {
   const defaultSettingsDefaults = useMemo(() => {
     return {

invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelList.tsx

@@ -11,6 +11,7 @@ import { useTranslation } from 'react-i18next';
 import {
   useCLIPEmbedModels,
   useCLIPVisionModels,
+  useControlLoRAModel,
   useControlNetModels,
   useEmbeddingModels,
   useIPAdapterModels,
@@ -92,6 +93,12 @@ const ModelList = () => {
     [t5EncoderModels, searchTerm, filteredModelType]
   );
 
+  const [controlLoRAModels, { isLoading: isLoadingControlLoRAModels }] = useControlLoRAModel();
+  const filteredControlLoRAModels = useMemo(
+    () => modelsFilter(controlLoRAModels, searchTerm, filteredModelType),
+    [controlLoRAModels, searchTerm, filteredModelType]
+  );
+
   const [clipEmbedModels, { isLoading: isLoadingClipEmbedModels }] = useCLIPEmbedModels({ excludeSubmodels: true });
   const filteredClipEmbedModels = useMemo(
     () => modelsFilter(clipEmbedModels, searchTerm, filteredModelType),
@@ -118,7 +125,8 @@ const ModelList = () => {
       filteredVAEModels.length +
       filteredSpandrelImageToImageModels.length +
       t5EncoderModels.length +
-      clipEmbedModels.length
+      clipEmbedModels.length +
+      controlLoRAModels.length
     );
   }, [
     filteredControlNetModels.length,
@@ -133,6 +141,7 @@ const ModelList = () => {
     filteredSpandrelImageToImageModels.length,
     t5EncoderModels.length,
     clipEmbedModels.length,
+    controlLoRAModels.length,
   ]);
 
   return (
@@ -195,6 +204,15 @@ const ModelList = () => {
         {!isLoadingT5EncoderModels && filteredT5EncoderModels.length > 0 && (
           <ModelListWrapper title={t('modelManager.t5Encoder')} modelList={filteredT5EncoderModels} key="t5-encoder" />
         )}
+        {/* Control Lora List */}
+        {isLoadingControlLoRAModels && <FetchingModelsLoader loadingMessage="Loading Control Loras..." />}
+        {!isLoadingControlLoRAModels && filteredControlLoRAModels.length > 0 && (
+          <ModelListWrapper
+            title={t('modelManager.controlLora')}
+            modelList={filteredControlLoRAModels}
+            key="control-lora"
+          />
+        )}
         {/* Clip Embed List */}
         {isLoadingClipEmbedModels && <FetchingModelsLoader loadingMessage="Loading Clip Embed Models..." />}
         {!isLoadingClipEmbedModels && filteredClipEmbedModels.length > 0 && (

invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelManagerPanel/ModelTypeFilter.tsx

@@ -24,6 +24,7 @@ export const ModelTypeFilter = memo(() => {
       ip_adapter: t('common.ipAdapter'),
       clip_vision: 'CLIP Vision',
       spandrel_image_to_image: t('modelManager.spandrelImageToImage'),
+      control_lora: t('modelManager.controlLora'),
     }),
     [t]
   );