Make quantized loading fast for both T5XXL and FLUX transformer.

Updated by "Cleanup translation files" hook in Weblate.

translationBot(ui): update translation files

Updated by "Cleanup translation files" hook in Weblate.

Co-authored-by: Hosted Weblate <hosted@weblate.org>
Translate-URL: https://hosted.weblate.org/projects/invokeai/web-ui/
Translation: InvokeAI/Web UI

invokeai/app/invocations/compel.py

@@ -80,12 +80,12 @@ class CompelInvocation(BaseInvocation):
 
         with (
             # apply all patches while the model is on the target device
-            text_encoder_info.model_on_device() as (model_state_dict, text_encoder),
+            text_encoder_info.model_on_device() as (cached_weights, text_encoder),
             tokenizer_info as tokenizer,
             ModelPatcher.apply_lora_text_encoder(
                 text_encoder,
                 loras=_lora_loader(),
-                model_state_dict=model_state_dict,
+                cached_weights=cached_weights,
             ),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.
             ModelPatcher.apply_clip_skip(text_encoder, self.clip.skipped_layers),
@@ -175,13 +175,13 @@ class SDXLPromptInvocationBase:
 
         with (
             # apply all patches while the model is on the target device
-            text_encoder_info.model_on_device() as (state_dict, text_encoder),
+            text_encoder_info.model_on_device() as (cached_weights, text_encoder),
             tokenizer_info as tokenizer,
             ModelPatcher.apply_lora(
                 text_encoder,
                 loras=_lora_loader(),
                 prefix=lora_prefix,
-                model_state_dict=state_dict,
+                cached_weights=cached_weights,
             ),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.
             ModelPatcher.apply_clip_skip(text_encoder, clip_field.skipped_layers),

invokeai/app/invocations/create_gradient_mask.py

@@ -39,7 +39,7 @@ class GradientMaskOutput(BaseInvocationOutput):
     title="Create Gradient Mask",
     tags=["mask", "denoise"],
     category="latents",
-    version="1.1.0",
+    version="1.2.0",
 )
 class CreateGradientMaskInvocation(BaseInvocation):
     """Creates mask for denoising model run."""
@@ -93,6 +93,7 @@ class CreateGradientMaskInvocation(BaseInvocation):
 
             # redistribute blur so that the original edges are 0 and blur outwards to 1
             blur_tensor = (blur_tensor - 0.5) * 2
+            blur_tensor[blur_tensor < 0] = 0.0
 
             threshold = 1 - self.minimum_denoise
 

invokeai/app/invocations/denoise_latents.py

@@ -62,6 +62,7 @@ from invokeai.backend.stable_diffusion.extensions.controlnet import ControlNetEx
 from invokeai.backend.stable_diffusion.extensions.freeu import FreeUExt
 from invokeai.backend.stable_diffusion.extensions.inpaint import InpaintExt
 from invokeai.backend.stable_diffusion.extensions.inpaint_model import InpaintModelExt
+from invokeai.backend.stable_diffusion.extensions.lora import LoRAExt
 from invokeai.backend.stable_diffusion.extensions.preview import PreviewExt
 from invokeai.backend.stable_diffusion.extensions.rescale_cfg import RescaleCFGExt
 from invokeai.backend.stable_diffusion.extensions.seamless import SeamlessExt
@@ -845,6 +846,16 @@ class DenoiseLatentsInvocation(BaseInvocation):
         if self.unet.freeu_config:
             ext_manager.add_extension(FreeUExt(self.unet.freeu_config))
 
+        ### lora
+        if self.unet.loras:
+            for lora_field in self.unet.loras:
+                ext_manager.add_extension(
+                    LoRAExt(
+                        node_context=context,
+                        model_id=lora_field.lora,
+                        weight=lora_field.weight,
+                    )
+                )
         ### seamless
         if self.unet.seamless_axes:
             ext_manager.add_extension(SeamlessExt(self.unet.seamless_axes))
@@ -964,14 +975,14 @@ class DenoiseLatentsInvocation(BaseInvocation):
         assert isinstance(unet_info.model, UNet2DConditionModel)
         with (
             ExitStack() as exit_stack,
-            unet_info.model_on_device() as (model_state_dict, unet),
+            unet_info.model_on_device() as (cached_weights, unet),
             ModelPatcher.apply_freeu(unet, self.unet.freeu_config),
             SeamlessExt.static_patch_model(unet, self.unet.seamless_axes),  # FIXME
             # Apply the LoRA after unet has been moved to its target device for faster patching.
             ModelPatcher.apply_lora_unet(
                 unet,
                 loras=_lora_loader(),
-                model_state_dict=model_state_dict,
+                cached_weights=cached_weights,
             ),
         ):
             assert isinstance(unet, UNet2DConditionModel)

invokeai/app/invocations/fields.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Any, Callable, Optional, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field, RootModel, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, RootModel, TypeAdapter, model_validator
 from pydantic.fields import _Unset
 from pydantic_core import PydanticUndefined
 
@@ -258,6 +258,14 @@ class BoundingBoxField(BaseModel):
         "when the bounding box was produced by a detector and has an associated confidence score.",
     )
 
+    @model_validator(mode="after")
+    def check_coords(self):
+        if self.x_min > self.x_max:
+            raise ValueError(f"x_min ({self.x_min}) is greater than x_max ({self.x_max}).")
+        if self.y_min > self.y_max:
+            raise ValueError(f"y_min ({self.y_min}) is greater than y_max ({self.y_max}).")
+        return self
+
 
 class MetadataField(RootModel[dict[str, Any]]):
     """

invokeai/app/invocations/flux_text_to_image.py

@@ -0,0 +1,278 @@
+from pathlib import Path
+from typing import Literal
+
+import torch
+from diffusers import AutoencoderKL, FlowMatchEulerDiscreteScheduler
+from diffusers.models.transformers.transformer_flux import FluxTransformer2DModel
+from diffusers.pipelines.flux.pipeline_flux import FluxPipeline
+from optimum.quanto import qfloat8
+from PIL import Image
+from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5TokenizerFast
+from transformers.models.auto import AutoModelForTextEncoding
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.quantization.fast_quantized_diffusion_model import FastQuantizedDiffusersModel
+from invokeai.backend.quantization.fast_quantized_transformers_model import FastQuantizedTransformersModel
+from invokeai.backend.util.devices import TorchDevice
+
+TFluxModelKeys = Literal["flux-schnell"]
+FLUX_MODELS: dict[TFluxModelKeys, str] = {"flux-schnell": "black-forest-labs/FLUX.1-schnell"}
+
+
+class QuantizedFluxTransformer2DModel(FastQuantizedDiffusersModel):
+    base_class = FluxTransformer2DModel
+
+
+class QuantizedModelForTextEncoding(FastQuantizedTransformersModel):
+    auto_class = AutoModelForTextEncoding
+
+
+@invocation(
+    "flux_text_to_image",
+    title="FLUX Text to Image",
+    tags=["image"],
+    category="image",
+    version="1.0.0",
+)
+class FluxTextToImageInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Text-to-image generation using a FLUX model."""
+
+    model: TFluxModelKeys = InputField(description="The FLUX model to use for text-to-image generation.")
+    use_8bit: bool = InputField(
+        default=False, description="Whether to quantize the transformer model to 8-bit precision."
+    )
+    positive_prompt: str = InputField(description="Positive prompt for text-to-image generation.")
+    width: int = InputField(default=1024, multiple_of=16, description="Width of the generated image.")
+    height: int = InputField(default=1024, multiple_of=16, description="Height of the generated image.")
+    num_steps: int = InputField(default=4, description="Number of diffusion steps.")
+    guidance: float = InputField(
+        default=4.0,
+        description="The guidance strength. Higher values adhere more strictly to the prompt, and will produce less diverse images.",
+    )
+    seed: int = InputField(default=0, description="Randomness seed for reproducibility.")
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        model_path = context.models.download_and_cache_model(FLUX_MODELS[self.model])
+
+        t5_embeddings, clip_embeddings = self._encode_prompt(context, model_path)
+        latents = self._run_diffusion(context, model_path, clip_embeddings, t5_embeddings)
+        image = self._run_vae_decoding(context, model_path, latents)
+        image_dto = context.images.save(image=image)
+        return ImageOutput.build(image_dto)
+
+    def _encode_prompt(self, context: InvocationContext, flux_model_dir: Path) -> tuple[torch.Tensor, torch.Tensor]:
+        # Determine the T5 max sequence length based on the model.
+        if self.model == "flux-schnell":
+            max_seq_len = 256
+        # elif self.model == "flux-dev":
+        #     max_seq_len = 512
+        else:
+            raise ValueError(f"Unknown model: {self.model}")
+
+        # Load the CLIP tokenizer.
+        clip_tokenizer_path = flux_model_dir / "tokenizer"
+        clip_tokenizer = CLIPTokenizer.from_pretrained(clip_tokenizer_path, local_files_only=True)
+        assert isinstance(clip_tokenizer, CLIPTokenizer)
+
+        # Load the T5 tokenizer.
+        t5_tokenizer_path = flux_model_dir / "tokenizer_2"
+        t5_tokenizer = T5TokenizerFast.from_pretrained(t5_tokenizer_path, local_files_only=True)
+        assert isinstance(t5_tokenizer, T5TokenizerFast)
+
+        clip_text_encoder_path = flux_model_dir / "text_encoder"
+        t5_text_encoder_path = flux_model_dir / "text_encoder_2"
+        with (
+            context.models.load_local_model(
+                model_path=clip_text_encoder_path, loader=self._load_flux_text_encoder
+            ) as clip_text_encoder,
+            context.models.load_local_model(
+                model_path=t5_text_encoder_path, loader=self._load_flux_text_encoder_2
+            ) as t5_text_encoder,
+        ):
+            assert isinstance(clip_text_encoder, CLIPTextModel)
+            assert isinstance(t5_text_encoder, T5EncoderModel)
+            pipeline = FluxPipeline(
+                scheduler=None,
+                vae=None,
+                text_encoder=clip_text_encoder,
+                tokenizer=clip_tokenizer,
+                text_encoder_2=t5_text_encoder,
+                tokenizer_2=t5_tokenizer,
+                transformer=None,
+            )
+
+            # prompt_embeds: T5 embeddings
+            # pooled_prompt_embeds: CLIP embeddings
+            prompt_embeds, pooled_prompt_embeds, text_ids = pipeline.encode_prompt(
+                prompt=self.positive_prompt,
+                prompt_2=self.positive_prompt,
+                device=TorchDevice.choose_torch_device(),
+                max_sequence_length=max_seq_len,
+            )
+
+        assert isinstance(prompt_embeds, torch.Tensor)
+        assert isinstance(pooled_prompt_embeds, torch.Tensor)
+        return prompt_embeds, pooled_prompt_embeds
+
+    def _run_diffusion(
+        self,
+        context: InvocationContext,
+        flux_model_dir: Path,
+        clip_embeddings: torch.Tensor,
+        t5_embeddings: torch.Tensor,
+    ):
+        scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(flux_model_dir / "scheduler", local_files_only=True)
+
+        # HACK(ryand): Manually empty the cache. Currently we don't check the size of the model before loading it from
+        # disk. Since the transformer model is large (24GB), there's a good chance that it will OOM on 32GB RAM systems
+        # if the cache is not empty.
+        context.models._services.model_manager.load.ram_cache.make_room(24 * 2**30)
+
+        transformer_path = flux_model_dir / "transformer"
+        with context.models.load_local_model(
+            model_path=transformer_path, loader=self._load_flux_transformer
+        ) as transformer:
+            assert isinstance(transformer, FluxTransformer2DModel)
+
+            flux_pipeline_with_transformer = FluxPipeline(
+                scheduler=scheduler,
+                vae=None,
+                text_encoder=None,
+                tokenizer=None,
+                text_encoder_2=None,
+                tokenizer_2=None,
+                transformer=transformer,
+            )
+
+            t5_embeddings = t5_embeddings.to(dtype=transformer.dtype)
+            clip_embeddings = clip_embeddings.to(dtype=transformer.dtype)
+
+            latents = flux_pipeline_with_transformer(
+                height=self.height,
+                width=self.width,
+                num_inference_steps=self.num_steps,
+                guidance_scale=self.guidance,
+                generator=torch.Generator().manual_seed(self.seed),
+                prompt_embeds=t5_embeddings,
+                pooled_prompt_embeds=clip_embeddings,
+                output_type="latent",
+                return_dict=False,
+            )[0]
+
+        assert isinstance(latents, torch.Tensor)
+        return latents
+
+    def _run_vae_decoding(
+        self,
+        context: InvocationContext,
+        flux_model_dir: Path,
+        latents: torch.Tensor,
+    ) -> Image.Image:
+        vae_path = flux_model_dir / "vae"
+        with context.models.load_local_model(model_path=vae_path, loader=self._load_flux_vae) as vae:
+            assert isinstance(vae, AutoencoderKL)
+
+            flux_pipeline_with_vae = FluxPipeline(
+                scheduler=None,
+                vae=vae,
+                text_encoder=None,
+                tokenizer=None,
+                text_encoder_2=None,
+                tokenizer_2=None,
+                transformer=None,
+            )
+
+            latents = flux_pipeline_with_vae._unpack_latents(
+                latents, self.height, self.width, flux_pipeline_with_vae.vae_scale_factor
+            )
+            latents = (
+                latents / flux_pipeline_with_vae.vae.config.scaling_factor
+            ) + flux_pipeline_with_vae.vae.config.shift_factor
+            latents = latents.to(dtype=vae.dtype)
+            image = flux_pipeline_with_vae.vae.decode(latents, return_dict=False)[0]
+            image = flux_pipeline_with_vae.image_processor.postprocess(image, output_type="pil")[0]
+
+        assert isinstance(image, Image.Image)
+        return image
+
+    @staticmethod
+    def _load_flux_text_encoder(path: Path) -> CLIPTextModel:
+        model = CLIPTextModel.from_pretrained(path, local_files_only=True)
+        assert isinstance(model, CLIPTextModel)
+        return model
+
+    def _load_flux_text_encoder_2(self, path: Path) -> T5EncoderModel:
+        if self.use_8bit:
+            model_8bit_path = path / "quantized"
+            if model_8bit_path.exists():
+                # The quantized model exists, load it.
+                # TODO(ryand): The requantize(...) operation in from_pretrained(...) is very slow. This seems like
+                # something that we should be able to make much faster.
+                q_model = QuantizedModelForTextEncoding.from_pretrained(model_8bit_path)
+
+                # Access the underlying wrapped model.
+                # We access the wrapped model, even though it is private, because it simplifies the type checking by
+                # always returning a T5EncoderModel from this function.
+                model = q_model._wrapped
+            else:
+                # The quantized model does not exist yet, quantize and save it.
+                # TODO(ryand): dtype?
+                model = T5EncoderModel.from_pretrained(path, local_files_only=True)
+                assert isinstance(model, T5EncoderModel)
+
+                q_model = QuantizedModelForTextEncoding.quantize(model, weights=qfloat8)
+
+                model_8bit_path.mkdir(parents=True, exist_ok=True)
+                q_model.save_pretrained(model_8bit_path)
+
+                # (See earlier comment about accessing the wrapped model.)
+                model = q_model._wrapped
+        else:
+            model = T5EncoderModel.from_pretrained(path, local_files_only=True)
+
+        assert isinstance(model, T5EncoderModel)
+        return model
+
+    def _load_flux_transformer(self, path: Path) -> FluxTransformer2DModel:
+        if self.use_8bit:
+            model_8bit_path = path / "quantized"
+            if model_8bit_path.exists():
+                # The quantized model exists, load it.
+                # TODO(ryand): The requantize(...) operation in from_pretrained(...) is very slow. This seems like
+                # something that we should be able to make much faster.
+                q_model = QuantizedFluxTransformer2DModel.from_pretrained(model_8bit_path)
+
+                # Access the underlying wrapped model.
+                # We access the wrapped model, even though it is private, because it simplifies the type checking by
+                # always returning a FluxTransformer2DModel from this function.
+                model = q_model._wrapped
+            else:
+                # The quantized model does not exist yet, quantize and save it.
+                # TODO(ryand): Loading in float16 and then quantizing seems to result in NaNs. In order to run this on
+                # GPUs that don't support bfloat16, we would need to host the quantized model instead of generating it
+                # here.
+                model = FluxTransformer2DModel.from_pretrained(path, local_files_only=True, torch_dtype=torch.bfloat16)
+                assert isinstance(model, FluxTransformer2DModel)
+
+                q_model = QuantizedFluxTransformer2DModel.quantize(model, weights=qfloat8)
+
+                model_8bit_path.mkdir(parents=True, exist_ok=True)
+                q_model.save_pretrained(model_8bit_path)
+
+                # (See earlier comment about accessing the wrapped model.)
+                model = q_model._wrapped
+        else:
+            model = FluxTransformer2DModel.from_pretrained(path, local_files_only=True, torch_dtype=torch.bfloat16)
+
+        assert isinstance(model, FluxTransformer2DModel)
+        return model
+
+    @staticmethod
+    def _load_flux_vae(path: Path) -> AutoencoderKL:
+        model = AutoencoderKL.from_pretrained(path, local_files_only=True)
+        assert isinstance(model, AutoencoderKL)
+        return model

invokeai/app/invocations/grounding_dino.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Literal
 
 import torch
 from PIL import Image
@@ -12,7 +13,11 @@ from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.image_util.grounding_dino.detection_result import DetectionResult
 from invokeai.backend.image_util.grounding_dino.grounding_dino_pipeline import GroundingDinoPipeline
 
-GROUNDING_DINO_MODEL_ID = "IDEA-Research/grounding-dino-tiny"
+GroundingDinoModelKey = Literal["grounding-dino-tiny", "grounding-dino-base"]
+GROUNDING_DINO_MODEL_IDS: dict[GroundingDinoModelKey, str] = {
+    "grounding-dino-tiny": "IDEA-Research/grounding-dino-tiny",
+    "grounding-dino-base": "IDEA-Research/grounding-dino-base",
+}
 
 
 @invocation(
@@ -23,14 +28,14 @@ GROUNDING_DINO_MODEL_ID = "IDEA-Research/grounding-dino-tiny"
     version="1.0.0",
 )
 class GroundingDinoInvocation(BaseInvocation):
-    """Runs a Grounding DINO model (https://arxiv.org/pdf/2303.05499). Performs zero-shot bounding-box object detection
-    from a text prompt.
+    """Runs a Grounding DINO model. Performs zero-shot bounding-box object detection from a text prompt."""
 
-    Reference:
-    - https://huggingface.co/docs/transformers/v4.43.3/en/model_doc/grounding-dino#grounded-sam
-    - https://github.com/NielsRogge/Transformers-Tutorials/blob/a39f33ac1557b02ebfb191ea7753e332b5ca933f/Grounding%20DINO/GroundingDINO_with_Segment_Anything.ipynb
-    """
+    # Reference:
+    # - https://arxiv.org/pdf/2303.05499
+    # - https://huggingface.co/docs/transformers/v4.43.3/en/model_doc/grounding-dino#grounded-sam
+    # - https://github.com/NielsRogge/Transformers-Tutorials/blob/a39f33ac1557b02ebfb191ea7753e332b5ca933f/Grounding%20DINO/GroundingDINO_with_Segment_Anything.ipynb
 
+    model: GroundingDinoModelKey = InputField(description="The Grounding DINO model to use.")
     prompt: str = InputField(description="The prompt describing the object to segment.")
     image: ImageField = InputField(description="The image to segment.")
     detection_threshold: float = InputField(
@@ -89,7 +94,7 @@ class GroundingDinoInvocation(BaseInvocation):
         labels = [label if label.endswith(".") else label + "." for label in labels]
 
         with context.models.load_remote_model(
-            source=GROUNDING_DINO_MODEL_ID, loader=GroundingDinoInvocation._load_grounding_dino
+            source=GROUNDING_DINO_MODEL_IDS[self.model], loader=GroundingDinoInvocation._load_grounding_dino
         ) as detector:
             assert isinstance(detector, GroundingDinoPipeline)
             return detector.detect(image=image, candidate_labels=labels, threshold=threshold)

invokeai/app/invocations/mask.py

@@ -138,8 +138,7 @@ class MaskTensorToImageInvocation(BaseInvocation, WithMetadata, WithBoard):
         # Ensure that the mask is binary.
         if mask.dtype != torch.bool:
             mask = mask > 0.5
-        mask_np = mask.float().cpu().detach().numpy() * 255
-        mask_np = mask_np.astype(np.uint8)
+        mask_np = (mask.float() * 255).byte().cpu().numpy()
 
         mask_pil = Image.fromarray(mask_np, mode="L")
         image_dto = context.images.save(image=mask_pil)

invokeai/app/invocations/primitives.py

@@ -485,9 +485,27 @@ class BoundingBoxOutput(BaseInvocationOutput):
 class BoundingBoxCollectionOutput(BaseInvocationOutput):
     """Base class for nodes that output a collection of bounding boxes"""
 
-    collection: list[BoundingBoxField] = OutputField(
-        description="The output bounding boxes.",
-    )
+    collection: list[BoundingBoxField] = OutputField(description="The output bounding boxes.", title="Bounding Boxes")
+
+
+@invocation(
+    "bounding_box",
+    title="Bounding Box",
+    tags=["primitives", "segmentation", "collection", "bounding box"],
+    category="primitives",
+    version="1.0.0",
+)
+class BoundingBoxInvocation(BaseInvocation):
+    """Create a bounding box manually by supplying box coordinates"""
+
+    x_min: int = InputField(default=0, description="x-coordinate of the bounding box's top left vertex")
+    y_min: int = InputField(default=0, description="y-coordinate of the bounding box's top left vertex")
+    x_max: int = InputField(default=0, description="x-coordinate of the bounding box's bottom right vertex")
+    y_max: int = InputField(default=0, description="y-coordinate of the bounding box's bottom right vertex")
+
+    def invoke(self, context: InvocationContext) -> BoundingBoxOutput:
+        bounding_box = BoundingBoxField(x_min=self.x_min, y_min=self.y_min, x_max=self.x_max, y_max=self.y_max)
+        return BoundingBoxOutput(bounding_box=bounding_box)
 
 
 # endregion

invokeai/app/invocations/segment_anything.py

@@ -13,26 +13,32 @@ from invokeai.app.invocations.fields import BoundingBoxField, ImageField, InputF
 from invokeai.app.invocations.primitives import MaskOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.image_util.segment_anything.mask_refinement import mask_to_polygon, polygon_to_mask
-from invokeai.backend.image_util.segment_anything.segment_anything_model import SegmentAnythingModel
+from invokeai.backend.image_util.segment_anything.segment_anything_pipeline import SegmentAnythingPipeline
 
-SEGMENT_ANYTHING_MODEL_ID = "facebook/sam-vit-base"
+SegmentAnythingModelKey = Literal["segment-anything-base", "segment-anything-large", "segment-anything-huge"]
+SEGMENT_ANYTHING_MODEL_IDS: dict[SegmentAnythingModelKey, str] = {
+    "segment-anything-base": "facebook/sam-vit-base",
+    "segment-anything-large": "facebook/sam-vit-large",
+    "segment-anything-huge": "facebook/sam-vit-huge",
+}
 
 
 @invocation(
-    "segment_anything_model",
-    title="Segment Anything Model",
+    "segment_anything",
+    title="Segment Anything",
     tags=["prompt", "segmentation"],
     category="segmentation",
     version="1.0.0",
 )
-class SegmentAnythingModelInvocation(BaseInvocation):
-    """Runs a Segment Anything Model (https://arxiv.org/pdf/2304.02643).
+class SegmentAnythingInvocation(BaseInvocation):
+    """Runs a Segment Anything Model."""
 
-    Reference:
-    - https://huggingface.co/docs/transformers/v4.43.3/en/model_doc/grounding-dino#grounded-sam
-    - https://github.com/NielsRogge/Transformers-Tutorials/blob/a39f33ac1557b02ebfb191ea7753e332b5ca933f/Grounding%20DINO/GroundingDINO_with_Segment_Anything.ipynb
-    """
+    # Reference:
+    # - https://arxiv.org/pdf/2304.02643
+    # - https://huggingface.co/docs/transformers/v4.43.3/en/model_doc/grounding-dino#grounded-sam
+    # - https://github.com/NielsRogge/Transformers-Tutorials/blob/a39f33ac1557b02ebfb191ea7753e332b5ca933f/Grounding%20DINO/GroundingDINO_with_Segment_Anything.ipynb
 
+    model: SegmentAnythingModelKey = InputField(description="The Segment Anything model to use.")
     image: ImageField = InputField(description="The image to segment.")
     bounding_boxes: list[BoundingBoxField] = InputField(description="The bounding boxes to prompt the SAM model with.")
     apply_polygon_refinement: bool = InputField(
@@ -75,7 +81,7 @@ class SegmentAnythingModelInvocation(BaseInvocation):
 
         sam_processor = AutoProcessor.from_pretrained(model_path, local_files_only=True)
         assert isinstance(sam_processor, SamProcessor)
-        return SegmentAnythingModel(sam_model=sam_model, sam_processor=sam_processor)
+        return SegmentAnythingPipeline(sam_model=sam_model, sam_processor=sam_processor)
 
     def _segment(
         self,
@@ -88,10 +94,10 @@ class SegmentAnythingModelInvocation(BaseInvocation):
 
         with (
             context.models.load_remote_model(
-                source=SEGMENT_ANYTHING_MODEL_ID, loader=SegmentAnythingModelInvocation._load_sam_model
+                source=SEGMENT_ANYTHING_MODEL_IDS[self.model], loader=SegmentAnythingInvocation._load_sam_model
             ) as sam_pipeline,
         ):
-            assert isinstance(sam_pipeline, SegmentAnythingModel)
+            assert isinstance(sam_pipeline, SegmentAnythingPipeline)
             masks = sam_pipeline.segment(image=image, bounding_boxes=sam_bounding_boxes)
 
         masks = self._process_masks(masks)

invokeai/app/services/image_files/image_files_disk.py

@@ -1,11 +1,10 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654) and the InvokeAI Team
 from pathlib import Path
 from queue import Queue
-from typing import Dict, Optional, Union
+from typing import Optional, Union
 
 from PIL import Image, PngImagePlugin
 from PIL.Image import Image as PILImageType
-from send2trash import send2trash
 
 from invokeai.app.services.image_files.image_files_base import ImageFileStorageBase
 from invokeai.app.services.image_files.image_files_common import (
@@ -20,18 +19,12 @@ from invokeai.app.util.thumbnails import get_thumbnail_name, make_thumbnail
 class DiskImageFileStorage(ImageFileStorageBase):
     """Stores images on disk"""
 
-    __output_folder: Path
-    __cache_ids: Queue  # TODO: this is an incredibly naive cache
-    __cache: Dict[Path, PILImageType]
-    __max_cache_size: int
-    __invoker: Invoker
-
     def __init__(self, output_folder: Union[str, Path]):
-        self.__cache = {}
-        self.__cache_ids = Queue()
+        self.__cache: dict[Path, PILImageType] = {}
+        self.__cache_ids = Queue[Path]()
         self.__max_cache_size = 10  # TODO: get this from config
 
-        self.__output_folder: Path = output_folder if isinstance(output_folder, Path) else Path(output_folder)
+        self.__output_folder = output_folder if isinstance(output_folder, Path) else Path(output_folder)
         self.__thumbnails_folder = self.__output_folder / "thumbnails"
         # Validate required output folders at launch
         self.__validate_storage_folders()
@@ -103,7 +96,7 @@ class DiskImageFileStorage(ImageFileStorageBase):
             image_path = self.get_path(image_name)
 
             if image_path.exists():
-                send2trash(image_path)
+                image_path.unlink()
             if image_path in self.__cache:
                 del self.__cache[image_path]
 
@@ -111,7 +104,7 @@ class DiskImageFileStorage(ImageFileStorageBase):
             thumbnail_path = self.get_path(thumbnail_name, True)
 
             if thumbnail_path.exists():
-                send2trash(thumbnail_path)
+                thumbnail_path.unlink()
             if thumbnail_path in self.__cache:
                 del self.__cache[thumbnail_path]
         except Exception as e:

invokeai/app/services/model_images/model_images_default.py

@@ -2,7 +2,6 @@ from pathlib import Path
 
 from PIL import Image
 from PIL.Image import Image as PILImageType
-from send2trash import send2trash
 
 from invokeai.app.services.invoker import Invoker
 from invokeai.app.services.model_images.model_images_base import ModelImageFileStorageBase
@@ -70,7 +69,7 @@ class ModelImageFileStorageDisk(ModelImageFileStorageBase):
             if not self._validate_path(path):
                 raise ModelImageFileNotFoundException
 
-            send2trash(path)
+            path.unlink()
 
         except Exception as e:
             raise ModelImageFileDeleteException from e

invokeai/backend/image_util/grounding_dino/grounding_dino_pipeline.py

@@ -18,6 +18,7 @@ class GroundingDinoPipeline(RawModel):
 
     def detect(self, image: Image.Image, candidate_labels: list[str], threshold: float = 0.1) -> list[DetectionResult]:
         results = self._pipeline(image=image, candidate_labels=candidate_labels, threshold=threshold)
+        assert results is not None
         results = [DetectionResult.model_validate(result) for result in results]
         return results
 

invokeai/backend/image_util/segment_anything/segment_anything_pipeline.py

@@ -8,7 +8,7 @@ from transformers.models.sam.processing_sam import SamProcessor
 from invokeai.backend.raw_model import RawModel
 
 
-class SegmentAnythingModel(RawModel):
+class SegmentAnythingPipeline(RawModel):
     """A wrapper class for the transformers SAM model and processor that makes it compatible with the model manager."""
 
     def __init__(self, sam_model: SamModel, sam_processor: SamProcessor):

invokeai/backend/load_flux_model.py

@@ -0,0 +1,129 @@
+import json
+import os
+import time
+from pathlib import Path
+from typing import Union
+
+import torch
+from diffusers.models.model_loading_utils import load_state_dict
+from diffusers.models.transformers.transformer_flux import FluxTransformer2DModel
+from diffusers.utils import (
+    CONFIG_NAME,
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFETENSORS_WEIGHTS_NAME,
+    _get_checkpoint_shard_files,
+    is_accelerate_available,
+)
+from optimum.quanto import qfloat8
+from optimum.quanto.models import QuantizedDiffusersModel
+from optimum.quanto.models.shared_dict import ShardedStateDict
+
+from invokeai.backend.requantize import requantize
+
+
+class QuantizedFluxTransformer2DModel(QuantizedDiffusersModel):
+    base_class = FluxTransformer2DModel
+
+    @classmethod
+    def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike]):
+        if cls.base_class is None:
+            raise ValueError("The `base_class` attribute needs to be configured.")
+
+        if not is_accelerate_available():
+            raise ValueError("Reloading a quantized diffusers model requires the accelerate library.")
+        from accelerate import init_empty_weights
+
+        if os.path.isdir(model_name_or_path):
+            # Look for a quantization map
+            qmap_path = os.path.join(model_name_or_path, cls._qmap_name())
+            if not os.path.exists(qmap_path):
+                raise ValueError(f"No quantization map found in {model_name_or_path}: is this a quantized model ?")
+
+            # Look for original model config file.
+            model_config_path = os.path.join(model_name_or_path, CONFIG_NAME)
+            if not os.path.exists(model_config_path):
+                raise ValueError(f"{CONFIG_NAME} not found in {model_name_or_path}.")
+
+            with open(qmap_path, "r", encoding="utf-8") as f:
+                qmap = json.load(f)
+
+            with open(model_config_path, "r", encoding="utf-8") as f:
+                original_model_cls_name = json.load(f)["_class_name"]
+            configured_cls_name = cls.base_class.__name__
+            if configured_cls_name != original_model_cls_name:
+                raise ValueError(
+                    f"Configured base class ({configured_cls_name}) differs from what was derived from the provided configuration ({original_model_cls_name})."
+                )
+
+            # Create an empty model
+            config = cls.base_class.load_config(model_name_or_path)
+            with init_empty_weights():
+                model = cls.base_class.from_config(config)
+
+            # Look for the index of a sharded checkpoint
+            checkpoint_file = os.path.join(model_name_or_path, SAFE_WEIGHTS_INDEX_NAME)
+            if os.path.exists(checkpoint_file):
+                # Convert the checkpoint path to a list of shards
+                _, sharded_metadata = _get_checkpoint_shard_files(model_name_or_path, checkpoint_file)
+                # Create a mapping for the sharded safetensor files
+                state_dict = ShardedStateDict(model_name_or_path, sharded_metadata["weight_map"])
+            else:
+                # Look for a single checkpoint file
+                checkpoint_file = os.path.join(model_name_or_path, SAFETENSORS_WEIGHTS_NAME)
+                if not os.path.exists(checkpoint_file):
+                    raise ValueError(f"No safetensor weights found in {model_name_or_path}.")
+                # Get state_dict from model checkpoint
+                state_dict = load_state_dict(checkpoint_file)
+
+            # Requantize and load quantized weights from state_dict
+            requantize(model, state_dict=state_dict, quantization_map=qmap)
+            model.eval()
+            return cls(model)
+        else:
+            raise NotImplementedError("Reloading quantized models directly from the hub is not supported yet.")
+
+
+def load_flux_transformer(path: Path) -> FluxTransformer2DModel:
+    # model = FluxTransformer2DModel.from_pretrained(path, local_files_only=True, torch_dtype=torch.bfloat16)
+    model_8bit_path = path / "quantized"
+    if model_8bit_path.exists():
+        # The quantized model exists, load it.
+        # TODO(ryand): The requantize(...) operation in from_pretrained(...) is very slow. This seems like
+        # something that we should be able to make much faster.
+        q_model = QuantizedFluxTransformer2DModel.from_pretrained(model_8bit_path)
+
+        # Access the underlying wrapped model.
+        # We access the wrapped model, even though it is private, because it simplifies the type checking by
+        # always returning a FluxTransformer2DModel from this function.
+        model = q_model._wrapped
+    else:
+        # The quantized model does not exist yet, quantize and save it.
+        # TODO(ryand): Loading in float16 and then quantizing seems to result in NaNs. In order to run this on
+        # GPUs that don't support bfloat16, we would need to host the quantized model instead of generating it
+        # here.
+        model = FluxTransformer2DModel.from_pretrained(path, local_files_only=True, torch_dtype=torch.bfloat16)
+        assert isinstance(model, FluxTransformer2DModel)
+
+        q_model = QuantizedFluxTransformer2DModel.quantize(model, weights=qfloat8)
+
+        model_8bit_path.mkdir(parents=True, exist_ok=True)
+        q_model.save_pretrained(model_8bit_path)
+
+        # (See earlier comment about accessing the wrapped model.)
+        model = q_model._wrapped
+
+    assert isinstance(model, FluxTransformer2DModel)
+    return model
+
+
+def main():
+    start = time.time()
+    model = load_flux_transformer(
+        Path("/data/invokeai/models/.download_cache/black-forest-labs_flux.1-schnell/FLUX.1-schnell/transformer/")
+    )
+    print(f"Time to load: {time.time() - start}s")
+    print("hi")
+
+
+if __name__ == "__main__":
+    main()

invokeai/backend/lora.py

@@ -3,12 +3,13 @@
 
 import bisect
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Set, Tuple, Union
 
 import torch
 from safetensors.torch import load_file
 from typing_extensions import Self
 
+import invokeai.backend.util.logging as logger
 from invokeai.backend.model_manager import BaseModelType
 from invokeai.backend.raw_model import RawModel
 
@@ -46,9 +47,19 @@ class LoRALayerBase:
         self.rank = None  # set in layer implementation
         self.layer_key = layer_key
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         raise NotImplementedError()
 
+    def get_bias(self, orig_bias: torch.Tensor) -> Optional[torch.Tensor]:
+        return self.bias
+
+    def get_parameters(self, orig_module: torch.nn.Module) -> Dict[str, torch.Tensor]:
+        params = {"weight": self.get_weight(orig_module.weight)}
+        bias = self.get_bias(orig_module.bias)
+        if bias is not None:
+            params["bias"] = bias
+        return params
+
     def calc_size(self) -> int:
         model_size = 0
         for val in [self.bias]:
@@ -60,6 +71,17 @@ class LoRALayerBase:
         if self.bias is not None:
             self.bias = self.bias.to(device=device, dtype=dtype)
 
+    def check_keys(self, values: Dict[str, torch.Tensor], known_keys: Set[str]):
+        """Log a warning if values contains unhandled keys."""
+        # {"alpha", "bias_indices", "bias_values", "bias_size"} are hard-coded, because they are handled by
+        # `LoRALayerBase`. Sub-classes should provide the known_keys that they handled.
+        all_known_keys = known_keys | {"alpha", "bias_indices", "bias_values", "bias_size"}
+        unknown_keys = set(values.keys()) - all_known_keys
+        if unknown_keys:
+            logger.warning(
+                f"Unexpected keys found in LoRA/LyCORIS layer, model might work incorrectly! Keys: {unknown_keys}"
+            )
+
 
 # TODO: find and debug lora/locon with bias
 class LoRALayer(LoRALayerBase):
@@ -76,14 +98,19 @@ class LoRALayer(LoRALayerBase):
 
         self.up = values["lora_up.weight"]
         self.down = values["lora_down.weight"]
-        if "lora_mid.weight" in values:
-            self.mid: Optional[torch.Tensor] = values["lora_mid.weight"]
-        else:
-            self.mid = None
+        self.mid = values.get("lora_mid.weight", None)
 
         self.rank = self.down.shape[0]
+        self.check_keys(
+            values,
+            {
+                "lora_up.weight",
+                "lora_down.weight",
+                "lora_mid.weight",
+            },
+        )
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         if self.mid is not None:
             up = self.up.reshape(self.up.shape[0], self.up.shape[1])
             down = self.down.reshape(self.down.shape[0], self.down.shape[1])
@@ -125,20 +152,23 @@ class LoHALayer(LoRALayerBase):
         self.w1_b = values["hada_w1_b"]
         self.w2_a = values["hada_w2_a"]
         self.w2_b = values["hada_w2_b"]
-
-        if "hada_t1" in values:
-            self.t1: Optional[torch.Tensor] = values["hada_t1"]
-        else:
-            self.t1 = None
-
-        if "hada_t2" in values:
-            self.t2: Optional[torch.Tensor] = values["hada_t2"]
-        else:
-            self.t2 = None
+        self.t1 = values.get("hada_t1", None)
+        self.t2 = values.get("hada_t2", None)
 
         self.rank = self.w1_b.shape[0]
+        self.check_keys(
+            values,
+            {
+                "hada_w1_a",
+                "hada_w1_b",
+                "hada_w2_a",
+                "hada_w2_b",
+                "hada_t1",
+                "hada_t2",
+            },
+        )
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         if self.t1 is None:
             weight: torch.Tensor = (self.w1_a @ self.w1_b) * (self.w2_a @ self.w2_b)
 
@@ -186,37 +216,39 @@ class LoKRLayer(LoRALayerBase):
     ):
         super().__init__(layer_key, values)
 
-        if "lokr_w1" in values:
-            self.w1: Optional[torch.Tensor] = values["lokr_w1"]
-            self.w1_a = None
-            self.w1_b = None
-        else:
-            self.w1 = None
+        self.w1 = values.get("lokr_w1", None)
+        if self.w1 is None:
             self.w1_a = values["lokr_w1_a"]
             self.w1_b = values["lokr_w1_b"]
 
-        if "lokr_w2" in values:
-            self.w2: Optional[torch.Tensor] = values["lokr_w2"]
-            self.w2_a = None
-            self.w2_b = None
-        else:
-            self.w2 = None
+        self.w2 = values.get("lokr_w2", None)
+        if self.w2 is None:
             self.w2_a = values["lokr_w2_a"]
             self.w2_b = values["lokr_w2_b"]
 
-        if "lokr_t2" in values:
-            self.t2: Optional[torch.Tensor] = values["lokr_t2"]
-        else:
-            self.t2 = None
+        self.t2 = values.get("lokr_t2", None)
 
-        if "lokr_w1_b" in values:
-            self.rank = values["lokr_w1_b"].shape[0]
-        elif "lokr_w2_b" in values:
-            self.rank = values["lokr_w2_b"].shape[0]
+        if self.w1_b is not None:
+            self.rank = self.w1_b.shape[0]
+        elif self.w2_b is not None:
+            self.rank = self.w2_b.shape[0]
         else:
             self.rank = None  # unscaled
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+        self.check_keys(
+            values,
+            {
+                "lokr_w1",
+                "lokr_w1_a",
+                "lokr_w1_b",
+                "lokr_w2",
+                "lokr_w2_a",
+                "lokr_w2_b",
+                "lokr_t2",
+            },
+        )
+
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         w1: Optional[torch.Tensor] = self.w1
         if w1 is None:
             assert self.w1_a is not None
@@ -272,7 +304,9 @@ class LoKRLayer(LoRALayerBase):
 
 
 class FullLayer(LoRALayerBase):
+    # bias handled in LoRALayerBase(calc_size, to)
     # weight: torch.Tensor
+    # bias: Optional[torch.Tensor]
 
     def __init__(
         self,
@@ -282,15 +316,12 @@ class FullLayer(LoRALayerBase):
         super().__init__(layer_key, values)
 
         self.weight = values["diff"]
-
-        if len(values.keys()) > 1:
-            _keys = list(values.keys())
-            _keys.remove("diff")
-            raise NotImplementedError(f"Unexpected keys in lora diff layer: {_keys}")
+        self.bias = values.get("diff_b", None)
 
         self.rank = None  # unscaled
+        self.check_keys(values, {"diff", "diff_b"})
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         return self.weight
 
     def calc_size(self) -> int:
@@ -319,8 +350,9 @@ class IA3Layer(LoRALayerBase):
         self.on_input = values["on_input"]
 
         self.rank = None  # unscaled
+        self.check_keys(values, {"weight", "on_input"})
 
-    def get_weight(self, orig_weight: Optional[torch.Tensor]) -> torch.Tensor:
+    def get_weight(self, orig_weight: torch.Tensor) -> torch.Tensor:
         weight = self.weight
         if not self.on_input:
             weight = weight.reshape(-1, 1)
@@ -458,16 +490,19 @@ class LoRAModelRaw(RawModel):  # (torch.nn.Module):
             state_dict = cls._convert_sdxl_keys_to_diffusers_format(state_dict)
 
         for layer_key, values in state_dict.items():
+            # Detect layers according to LyCORIS detection logic(`weight_list_det`)
+            # https://github.com/KohakuBlueleaf/LyCORIS/tree/8ad8000efb79e2b879054da8c9356e6143591bad/lycoris/modules
+
             # lora and locon
-            if "lora_down.weight" in values:
+            if "lora_up.weight" in values:
                 layer: AnyLoRALayer = LoRALayer(layer_key, values)
 
             # loha
-            elif "hada_w1_b" in values:
+            elif "hada_w1_a" in values:
                 layer = LoHALayer(layer_key, values)
 
             # lokr
-            elif "lokr_w1_b" in values or "lokr_w1" in values:
+            elif "lokr_w1" in values or "lokr_w1_a" in values:
                 layer = LoKRLayer(layer_key, values)
 
             # diff
@@ -475,7 +510,7 @@ class LoRAModelRaw(RawModel):  # (torch.nn.Module):
                 layer = FullLayer(layer_key, values)
 
             # ia3
-            elif "weight" in values and "on_input" in values:
+            elif "on_input" in values:
                 layer = IA3Layer(layer_key, values)
 
             else:

invokeai/backend/model_manager/load/model_util.py

@@ -12,7 +12,7 @@ from diffusers.schedulers.scheduling_utils import SchedulerMixin
 from transformers import CLIPTokenizer
 
 from invokeai.backend.image_util.grounding_dino.grounding_dino_pipeline import GroundingDinoPipeline
-from invokeai.backend.image_util.segment_anything.segment_anything_model import SegmentAnythingModel
+from invokeai.backend.image_util.segment_anything.segment_anything_pipeline import SegmentAnythingPipeline
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
 from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager.config import AnyModel
@@ -44,7 +44,7 @@ def calc_model_size_by_data(logger: logging.Logger, model: AnyModel) -> int:
             LoRAModelRaw,
             SpandrelImageToImageModel,
             GroundingDinoPipeline,
-            SegmentAnythingModel,
+            SegmentAnythingPipeline,
         ),
     ):
         return model.calc_size()

invokeai/backend/model_manager/util/select_hf_files.py

@@ -54,6 +54,7 @@ def filter_files(
                 "lora_weights.safetensors",
                 "weights.pb",
                 "onnx_data",
+                "spiece.model", # Added for `black-forest-labs/FLUX.1-schnell`.
             )
         ):
             paths.append(file)
@@ -62,7 +63,7 @@ def filter_files(
         # downloading random checkpoints that might also be in the repo. However there is no guarantee
         # that a checkpoint doesn't contain "model" in its name, and no guarantee that future diffusers models
         # will adhere to this naming convention, so this is an area to be careful of.
-        elif re.search(r"model(\.[^.]+)?\.(safetensors|bin|onnx|xml|pth|pt|ckpt|msgpack)$", file.name):
+        elif re.search(r"model.*\.(safetensors|bin|onnx|xml|pth|pt|ckpt|msgpack)$", file.name):
             paths.append(file)
 
     # limit search to subfolder if requested
@@ -97,7 +98,9 @@ def _filter_by_variant(files: List[Path], variant: ModelRepoVariant) -> Set[Path
             if variant == ModelRepoVariant.Flax:
                 result.add(path)
 
-        elif path.suffix in [".json", ".txt"]:
+        # Note: '.model' was added to support:
+        # https://huggingface.co/black-forest-labs/FLUX.1-schnell/blob/768d12a373ed5cc9ef9a9dea7504dc09fcc14842/tokenizer_2/spiece.model
+        elif path.suffix in [".json", ".txt", ".model"]:
             result.add(path)
 
         elif variant in [
@@ -140,6 +143,23 @@ def _filter_by_variant(files: List[Path], variant: ModelRepoVariant) -> Set[Path
             continue
 
     for candidate_list in subfolder_weights.values():
+        # Check if at least one of the files has the explicit fp16 variant.
+        at_least_one_fp16 = False
+        for candidate in candidate_list:
+            if len(candidate.path.suffixes) == 2 and candidate.path.suffixes[0] == ".fp16":
+                at_least_one_fp16 = True
+                break
+
+        if not at_least_one_fp16:
+            # If none of the candidates in this candidate_list have the explicit fp16 variant label, then this
+            # candidate_list probably doesn't adhere to the variant naming convention that we expected. In this case,
+            # we'll simply keep all the candidates. An example of a model that hits this case is
+            # `black-forest-labs/FLUX.1-schnell` (as of commit 012d2fd).
+            for candidate in candidate_list:
+                result.add(candidate.path)
+
+        # The candidate_list seems to have the expected variant naming convention. We'll select the highest scoring
+        # candidate.
         highest_score_candidate = max(candidate_list, key=lambda candidate: candidate.score)
         if highest_score_candidate:
             result.add(highest_score_candidate.path)

invokeai/backend/model_patcher.py

@@ -17,8 +17,9 @@ from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager import AnyModel
 from invokeai.backend.model_manager.load.optimizations import skip_torch_weight_init
 from invokeai.backend.onnx.onnx_runtime import IAIOnnxRuntimeModel
+from invokeai.backend.stable_diffusion.extensions.lora import LoRAExt
 from invokeai.backend.textual_inversion import TextualInversionManager, TextualInversionModelRaw
-from invokeai.backend.util.devices import TorchDevice
+from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 """
 loras = [
@@ -85,13 +86,13 @@ class ModelPatcher:
         cls,
         unet: UNet2DConditionModel,
         loras: Iterator[Tuple[LoRAModelRaw, float]],
-        model_state_dict: Optional[Dict[str, torch.Tensor]] = None,
+        cached_weights: Optional[Dict[str, torch.Tensor]] = None,
     ) -> Generator[None, None, None]:
         with cls.apply_lora(
             unet,
             loras=loras,
             prefix="lora_unet_",
-            model_state_dict=model_state_dict,
+            cached_weights=cached_weights,
         ):
             yield
 
@@ -101,9 +102,9 @@ class ModelPatcher:
         cls,
         text_encoder: CLIPTextModel,
         loras: Iterator[Tuple[LoRAModelRaw, float]],
-        model_state_dict: Optional[Dict[str, torch.Tensor]] = None,
+        cached_weights: Optional[Dict[str, torch.Tensor]] = None,
     ) -> Generator[None, None, None]:
-        with cls.apply_lora(text_encoder, loras=loras, prefix="lora_te_", model_state_dict=model_state_dict):
+        with cls.apply_lora(text_encoder, loras=loras, prefix="lora_te_", cached_weights=cached_weights):
             yield
 
     @classmethod
@@ -113,7 +114,7 @@ class ModelPatcher:
         model: AnyModel,
         loras: Iterator[Tuple[LoRAModelRaw, float]],
         prefix: str,
-        model_state_dict: Optional[Dict[str, torch.Tensor]] = None,
+        cached_weights: Optional[Dict[str, torch.Tensor]] = None,
     ) -> Generator[None, None, None]:
         """
         Apply one or more LoRAs to a model.
@@ -121,66 +122,26 @@ class ModelPatcher:
         :param model: The model to patch.
         :param loras: An iterator that returns the LoRA to patch in and its patch weight.
         :param prefix: A string prefix that precedes keys used in the LoRAs weight layers.
-        :model_state_dict: Read-only copy of the model's state dict in CPU, for unpatching purposes.
+        :cached_weights: Read-only copy of the model's state dict in CPU, for unpatching purposes.
         """
-        original_weights = {}
+        original_weights = OriginalWeightsStorage(cached_weights)
         try:
-            with torch.no_grad():
-                for lora, lora_weight in loras:
-                    # assert lora.device.type == "cpu"
-                    for layer_key, layer in lora.layers.items():
-                        if not layer_key.startswith(prefix):
-                            continue
+            for lora_model, lora_weight in loras:
+                LoRAExt.patch_model(
+                    model=model,
+                    prefix=prefix,
+                    lora=lora_model,
+                    lora_weight=lora_weight,
+                    original_weights=original_weights,
+                )
+                del lora_model
 
-                        # TODO(ryand): A non-negligible amount of time is currently spent resolving LoRA keys. This
-                        # should be improved in the following ways:
-                        # 1. The key mapping could be more-efficiently pre-computed. This would save time every time a
-                        #    LoRA model is applied.
-                        # 2. From an API perspective, there's no reason that the `ModelPatcher` should be aware of the
-                        #    intricacies of Stable Diffusion key resolution. It should just expect the input LoRA
-                        #    weights to have valid keys.
-                        assert isinstance(model, torch.nn.Module)
-                        module_key, module = cls._resolve_lora_key(model, layer_key, prefix)
-
-                        # All of the LoRA weight calculations will be done on the same device as the module weight.
-                        # (Performance will be best if this is a CUDA device.)
-                        device = module.weight.device
-                        dtype = module.weight.dtype
-
-                        if module_key not in original_weights:
-                            if model_state_dict is not None:  # we were provided with the CPU copy of the state dict
-                                original_weights[module_key] = model_state_dict[module_key + ".weight"]
-                            else:
-                                original_weights[module_key] = module.weight.detach().to(device="cpu", copy=True)
-
-                        layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
-
-                        # We intentionally move to the target device first, then cast. Experimentally, this was found to
-                        # be significantly faster for 16-bit CPU tensors being moved to a CUDA device than doing the
-                        # same thing in a single call to '.to(...)'.
-                        layer.to(device=device)
-                        layer.to(dtype=torch.float32)
-                        # TODO(ryand): Using torch.autocast(...) over explicit casting may offer a speed benefit on CUDA
-                        # devices here. Experimentally, it was found to be very slow on CPU. More investigation needed.
-                        layer_weight = layer.get_weight(module.weight) * (lora_weight * layer_scale)
-                        layer.to(device=TorchDevice.CPU_DEVICE)
-
-                        assert isinstance(layer_weight, torch.Tensor)  # mypy thinks layer_weight is a float|Any ??!
-                        if module.weight.shape != layer_weight.shape:
-                            # TODO: debug on lycoris
-                            assert hasattr(layer_weight, "reshape")
-                            layer_weight = layer_weight.reshape(module.weight.shape)
-
-                        assert isinstance(layer_weight, torch.Tensor)  # mypy thinks layer_weight is a float|Any ??!
-                        module.weight += layer_weight.to(dtype=dtype)
-
-            yield  # wait for context manager exit
+            yield
 
         finally:
-            assert hasattr(model, "get_submodule")  # mypy not picking up fact that torch.nn.Module has get_submodule()
             with torch.no_grad():
-                for module_key, weight in original_weights.items():
-                    model.get_submodule(module_key).weight.copy_(weight)
+                for param_key, weight in original_weights.get_changed_weights():
+                    model.get_parameter(param_key).copy_(weight)
 
     @classmethod
     @contextmanager

invokeai/backend/quantization/fast_quantized_diffusion_model.py

@@ -0,0 +1,77 @@
+import json
+import os
+from typing import Union
+
+from diffusers.models.model_loading_utils import load_state_dict
+from diffusers.utils import (
+    CONFIG_NAME,
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFETENSORS_WEIGHTS_NAME,
+    _get_checkpoint_shard_files,
+    is_accelerate_available,
+)
+from optimum.quanto.models import QuantizedDiffusersModel
+from optimum.quanto.models.shared_dict import ShardedStateDict
+
+from invokeai.backend.requantize import requantize
+
+
+class FastQuantizedDiffusersModel(QuantizedDiffusersModel):
+    @classmethod
+    def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike]):
+        """We override the `from_pretrained()` method in order to use our custom `requantize()` implementation."""
+        if cls.base_class is None:
+            raise ValueError("The `base_class` attribute needs to be configured.")
+
+        if not is_accelerate_available():
+            raise ValueError("Reloading a quantized diffusers model requires the accelerate library.")
+        from accelerate import init_empty_weights
+
+        if os.path.isdir(model_name_or_path):
+            # Look for a quantization map
+            qmap_path = os.path.join(model_name_or_path, cls._qmap_name())
+            if not os.path.exists(qmap_path):
+                raise ValueError(f"No quantization map found in {model_name_or_path}: is this a quantized model ?")
+
+            # Look for original model config file.
+            model_config_path = os.path.join(model_name_or_path, CONFIG_NAME)
+            if not os.path.exists(model_config_path):
+                raise ValueError(f"{CONFIG_NAME} not found in {model_name_or_path}.")
+
+            with open(qmap_path, "r", encoding="utf-8") as f:
+                qmap = json.load(f)
+
+            with open(model_config_path, "r", encoding="utf-8") as f:
+                original_model_cls_name = json.load(f)["_class_name"]
+            configured_cls_name = cls.base_class.__name__
+            if configured_cls_name != original_model_cls_name:
+                raise ValueError(
+                    f"Configured base class ({configured_cls_name}) differs from what was derived from the provided configuration ({original_model_cls_name})."
+                )
+
+            # Create an empty model
+            config = cls.base_class.load_config(model_name_or_path)
+            with init_empty_weights():
+                model = cls.base_class.from_config(config)
+
+            # Look for the index of a sharded checkpoint
+            checkpoint_file = os.path.join(model_name_or_path, SAFE_WEIGHTS_INDEX_NAME)
+            if os.path.exists(checkpoint_file):
+                # Convert the checkpoint path to a list of shards
+                _, sharded_metadata = _get_checkpoint_shard_files(model_name_or_path, checkpoint_file)
+                # Create a mapping for the sharded safetensor files
+                state_dict = ShardedStateDict(model_name_or_path, sharded_metadata["weight_map"])
+            else:
+                # Look for a single checkpoint file
+                checkpoint_file = os.path.join(model_name_or_path, SAFETENSORS_WEIGHTS_NAME)
+                if not os.path.exists(checkpoint_file):
+                    raise ValueError(f"No safetensor weights found in {model_name_or_path}.")
+                # Get state_dict from model checkpoint
+                state_dict = load_state_dict(checkpoint_file)
+
+            # Requantize and load quantized weights from state_dict
+            requantize(model, state_dict=state_dict, quantization_map=qmap)
+            model.eval()
+            return cls(model)
+        else:
+            raise NotImplementedError("Reloading quantized models directly from the hub is not supported yet.")

invokeai/backend/quantization/fast_quantized_transformers_model.py

@@ -0,0 +1,61 @@
+import json
+import os
+from typing import Union
+
+from optimum.quanto.models import QuantizedTransformersModel
+from optimum.quanto.models.shared_dict import ShardedStateDict
+from transformers import AutoConfig
+from transformers.modeling_utils import get_checkpoint_shard_files, load_state_dict
+from transformers.utils import SAFE_WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_NAME, is_accelerate_available
+
+from invokeai.backend.requantize import requantize
+
+
+class FastQuantizedTransformersModel(QuantizedTransformersModel):
+    @classmethod
+    def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike]):
+        """We override the `from_pretrained()` method in order to use our custom `requantize()` implementation."""
+        if cls.auto_class is None:
+            raise ValueError(
+                "Quantized models cannot be reloaded using {cls}: use a specialized quantized class such as QuantizedModelForCausalLM instead."
+            )
+        if not is_accelerate_available():
+            raise ValueError("Reloading a quantized transformers model requires the accelerate library.")
+        from accelerate import init_empty_weights
+
+        if os.path.isdir(model_name_or_path):
+            # Look for a quantization map
+            qmap_path = os.path.join(model_name_or_path, cls._qmap_name())
+            if not os.path.exists(qmap_path):
+                raise ValueError(f"No quantization map found in {model_name_or_path}: is this a quantized model ?")
+            with open(qmap_path, "r", encoding="utf-8") as f:
+                qmap = json.load(f)
+            # Create an empty model
+            config = AutoConfig.from_pretrained(model_name_or_path)
+            with init_empty_weights():
+                model = cls.auto_class.from_config(config)
+            # Look for the index of a sharded checkpoint
+            checkpoint_file = os.path.join(model_name_or_path, SAFE_WEIGHTS_INDEX_NAME)
+            if os.path.exists(checkpoint_file):
+                # Convert the checkpoint path to a list of shards
+                checkpoint_file, sharded_metadata = get_checkpoint_shard_files(model_name_or_path, checkpoint_file)
+                # Create a mapping for the sharded safetensor files
+                state_dict = ShardedStateDict(model_name_or_path, sharded_metadata["weight_map"])
+            else:
+                # Look for a single checkpoint file
+                checkpoint_file = os.path.join(model_name_or_path, SAFE_WEIGHTS_NAME)
+                if not os.path.exists(checkpoint_file):
+                    raise ValueError(f"No safetensor weights found in {model_name_or_path}.")
+                # Get state_dict from model checkpoint
+                state_dict = load_state_dict(checkpoint_file)
+            # Requantize and load quantized weights from state_dict
+            requantize(model, state_dict=state_dict, quantization_map=qmap)
+            if getattr(model.config, "tie_word_embeddings", True):
+                # Tie output weight embeddings to input weight embeddings
+                # Note that if they were quantized they would NOT be tied
+                model.tie_weights()
+            # Set model in evaluation mode as it is done in transformers
+            model.eval()
+            return cls(model)
+        else:
+            raise NotImplementedError("Reloading quantized models directly from the hub is not supported yet.")

invokeai/backend/requantize.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict
+
+import torch
+from optimum.quanto.quantize import _quantize_submodule
+
+# def custom_freeze(model: torch.nn.Module):
+#     for name, m in model.named_modules():
+#         if isinstance(m, QModuleMixin):
+#             m.weight =
+#             m.freeze()
+
+
+def requantize(
+    model: torch.nn.Module,
+    state_dict: Dict[str, Any],
+    quantization_map: Dict[str, Dict[str, str]],
+    device: torch.device = None,
+):
+    if device is None:
+        device = next(model.parameters()).device
+        if device.type == "meta":
+            device = torch.device("cpu")
+
+    # Quantize the model with parameters from the quantization map
+    for name, m in model.named_modules():
+        qconfig = quantization_map.get(name, None)
+        if qconfig is not None:
+            weights = qconfig["weights"]
+            if weights == "none":
+                weights = None
+            activations = qconfig["activations"]
+            if activations == "none":
+                activations = None
+            _quantize_submodule(model, name, m, weights=weights, activations=activations)
+
+    # Move model parameters and buffers to CPU before materializing quantized weights
+    for name, m in model.named_modules():
+
+        def move_tensor(t, device):
+            if t.device.type == "meta":
+                return torch.empty_like(t, device=device)
+            return t.to(device)
+
+        for name, param in m.named_parameters(recurse=False):
+            setattr(m, name, torch.nn.Parameter(move_tensor(param, "cpu")))
+        for name, param in m.named_buffers(recurse=False):
+            setattr(m, name, move_tensor(param, "cpu"))
+    # Freeze model and move to target device
+    # freeze(model)
+    # model.to(device)
+
+    # Load the quantized model weights
+    model.load_state_dict(state_dict, strict=False)

invokeai/backend/stable_diffusion/extensions/base.py

@@ -2,14 +2,14 @@ from __future__ import annotations
 
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Callable, Dict, List, Optional
+from typing import TYPE_CHECKING, Callable, Dict, List
 
-import torch
 from diffusers import UNet2DConditionModel
 
 if TYPE_CHECKING:
     from invokeai.backend.stable_diffusion.denoise_context import DenoiseContext
     from invokeai.backend.stable_diffusion.extension_callback_type import ExtensionCallbackType
+    from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 
 @dataclass
@@ -56,5 +56,17 @@ class ExtensionBase:
         yield None
 
     @contextmanager
-    def patch_unet(self, unet: UNet2DConditionModel, cached_weights: Optional[Dict[str, torch.Tensor]] = None):
-        yield None
+    def patch_unet(self, unet: UNet2DConditionModel, original_weights: OriginalWeightsStorage):
+        """A context manager for applying patches to the UNet model. The context manager's lifetime spans the entire
+        diffusion process. Weight unpatching is handled upstream, and is achieved by saving unchanged weights by
+        `original_weights.save` function. Note that this enables some performance optimization by avoiding redundant
+        operations. All other patches (e.g. changes to tensor shapes, function monkey-patches, etc.) should be unpatched
+        by this context manager.
+
+        Args:
+            unet (UNet2DConditionModel): The UNet model on execution device to patch.
+            original_weights (OriginalWeightsStorage): A storage with copy of the model's original weights in CPU, for
+                unpatching purposes. Extension should save tensor which being modified in this storage, also extensions
+                can access original weights values.
+        """
+        yield

invokeai/backend/stable_diffusion/extensions/freeu.py

@@ -1,15 +1,15 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Dict, Optional
+from typing import TYPE_CHECKING
 
-import torch
 from diffusers import UNet2DConditionModel
 
 from invokeai.backend.stable_diffusion.extensions.base import ExtensionBase
 
 if TYPE_CHECKING:
     from invokeai.app.shared.models import FreeUConfig
+    from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 
 class FreeUExt(ExtensionBase):
@@ -21,7 +21,7 @@ class FreeUExt(ExtensionBase):
         self._freeu_config = freeu_config
 
     @contextmanager
-    def patch_unet(self, unet: UNet2DConditionModel, cached_weights: Optional[Dict[str, torch.Tensor]] = None):
+    def patch_unet(self, unet: UNet2DConditionModel, original_weights: OriginalWeightsStorage):
         unet.enable_freeu(
             b1=self._freeu_config.b1,
             b2=self._freeu_config.b2,

invokeai/backend/stable_diffusion/extensions/lora.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+from diffusers import UNet2DConditionModel
+
+from invokeai.backend.stable_diffusion.extensions.base import ExtensionBase
+from invokeai.backend.util.devices import TorchDevice
+
+if TYPE_CHECKING:
+    from invokeai.app.invocations.model import ModelIdentifierField
+    from invokeai.app.services.shared.invocation_context import InvocationContext
+    from invokeai.backend.lora import LoRAModelRaw
+    from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
+
+
+class LoRAExt(ExtensionBase):
+    def __init__(
+        self,
+        node_context: InvocationContext,
+        model_id: ModelIdentifierField,
+        weight: float,
+    ):
+        super().__init__()
+        self._node_context = node_context
+        self._model_id = model_id
+        self._weight = weight
+
+    @contextmanager
+    def patch_unet(self, unet: UNet2DConditionModel, original_weights: OriginalWeightsStorage):
+        lora_model = self._node_context.models.load(self._model_id).model
+        self.patch_model(
+            model=unet,
+            prefix="lora_unet_",
+            lora=lora_model,
+            lora_weight=self._weight,
+            original_weights=original_weights,
+        )
+        del lora_model
+
+        yield
+
+    @classmethod
+    @torch.no_grad()
+    def patch_model(
+        cls,
+        model: torch.nn.Module,
+        prefix: str,
+        lora: LoRAModelRaw,
+        lora_weight: float,
+        original_weights: OriginalWeightsStorage,
+    ):
+        """
+        Apply one or more LoRAs to a model.
+        :param model: The model to patch.
+        :param lora: LoRA model to patch in.
+        :param lora_weight: LoRA patch weight.
+        :param prefix: A string prefix that precedes keys used in the LoRAs weight layers.
+        :param original_weights: Storage with original weights, filled by weights which lora patches, used for unpatching.
+        """
+
+        if lora_weight == 0:
+            return
+
+        # assert lora.device.type == "cpu"
+        for layer_key, layer in lora.layers.items():
+            if not layer_key.startswith(prefix):
+                continue
+
+            # TODO(ryand): A non-negligible amount of time is currently spent resolving LoRA keys. This
+            # should be improved in the following ways:
+            # 1. The key mapping could be more-efficiently pre-computed. This would save time every time a
+            #    LoRA model is applied.
+            # 2. From an API perspective, there's no reason that the `ModelPatcher` should be aware of the
+            #    intricacies of Stable Diffusion key resolution. It should just expect the input LoRA
+            #    weights to have valid keys.
+            assert isinstance(model, torch.nn.Module)
+            module_key, module = cls._resolve_lora_key(model, layer_key, prefix)
+
+            # All of the LoRA weight calculations will be done on the same device as the module weight.
+            # (Performance will be best if this is a CUDA device.)
+            device = module.weight.device
+            dtype = module.weight.dtype
+
+            layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
+
+            # We intentionally move to the target device first, then cast. Experimentally, this was found to
+            # be significantly faster for 16-bit CPU tensors being moved to a CUDA device than doing the
+            # same thing in a single call to '.to(...)'.
+            layer.to(device=device)
+            layer.to(dtype=torch.float32)
+
+            # TODO(ryand): Using torch.autocast(...) over explicit casting may offer a speed benefit on CUDA
+            # devices here. Experimentally, it was found to be very slow on CPU. More investigation needed.
+            for param_name, lora_param_weight in layer.get_parameters(module).items():
+                param_key = module_key + "." + param_name
+                module_param = module.get_parameter(param_name)
+
+                # save original weight
+                original_weights.save(param_key, module_param)
+
+                if module_param.shape != lora_param_weight.shape:
+                    # TODO: debug on lycoris
+                    lora_param_weight = lora_param_weight.reshape(module_param.shape)
+
+                lora_param_weight *= lora_weight * layer_scale
+                module_param += lora_param_weight.to(dtype=dtype)
+
+            layer.to(device=TorchDevice.CPU_DEVICE)
+
+    @staticmethod
+    def _resolve_lora_key(model: torch.nn.Module, lora_key: str, prefix: str) -> Tuple[str, torch.nn.Module]:
+        assert "." not in lora_key
+
+        if not lora_key.startswith(prefix):
+            raise Exception(f"lora_key with invalid prefix: {lora_key}, {prefix}")
+
+        module = model
+        module_key = ""
+        key_parts = lora_key[len(prefix) :].split("_")
+
+        submodule_name = key_parts.pop(0)
+
+        while len(key_parts) > 0:
+            try:
+                module = module.get_submodule(submodule_name)
+                module_key += "." + submodule_name
+                submodule_name = key_parts.pop(0)
+            except Exception:
+                submodule_name += "_" + key_parts.pop(0)
+
+        module = module.get_submodule(submodule_name)
+        module_key = (module_key + "." + submodule_name).lstrip(".")
+
+        return (module_key, module)

invokeai/backend/stable_diffusion/extensions_manager.py

@@ -7,6 +7,7 @@ import torch
 from diffusers import UNet2DConditionModel
 
 from invokeai.app.services.session_processor.session_processor_common import CanceledException
+from invokeai.backend.util.original_weights_storage import OriginalWeightsStorage
 
 if TYPE_CHECKING:
     from invokeai.backend.stable_diffusion.denoise_context import DenoiseContext
@@ -67,9 +68,15 @@ class ExtensionsManager:
         if self._is_canceled and self._is_canceled():
             raise CanceledException
 
-        # TODO: create weight patch logic in PR with extension which uses it
-        with ExitStack() as exit_stack:
-            for ext in self._extensions:
-                exit_stack.enter_context(ext.patch_unet(unet, cached_weights))
+        original_weights = OriginalWeightsStorage(cached_weights)
+        try:
+            with ExitStack() as exit_stack:
+                for ext in self._extensions:
+                    exit_stack.enter_context(ext.patch_unet(unet, original_weights))
 
-            yield None
+                yield None
+
+        finally:
+            with torch.no_grad():
+                for param_key, weight in original_weights.get_changed_weights():
+                    unet.get_parameter(param_key).copy_(weight)

invokeai/backend/stable_diffusion/schedulers/schedulers.py

@@ -20,10 +20,14 @@ from diffusers import (
 )
 from diffusers.schedulers.scheduling_utils import SchedulerMixin
 
+# TODO: add dpmpp_3s/dpmpp_3s_k when fix released
+# https://github.com/huggingface/diffusers/issues/9007
+
 SCHEDULER_NAME_VALUES = Literal[
     "ddim",
     "ddpm",
     "deis",
+    "deis_k",
     "lms",
     "lms_k",
     "pndm",
@@ -33,16 +37,21 @@ SCHEDULER_NAME_VALUES = Literal[
     "euler_k",
     "euler_a",
     "kdpm_2",
+    "kdpm_2_k",
     "kdpm_2_a",
+    "kdpm_2_a_k",
     "dpmpp_2s",
     "dpmpp_2s_k",
     "dpmpp_2m",
     "dpmpp_2m_k",
     "dpmpp_2m_sde",
     "dpmpp_2m_sde_k",
+    "dpmpp_3m",
+    "dpmpp_3m_k",
     "dpmpp_sde",
     "dpmpp_sde_k",
     "unipc",
+    "unipc_k",
     "lcm",
     "tcd",
 ]
@@ -50,7 +59,8 @@ SCHEDULER_NAME_VALUES = Literal[
 SCHEDULER_MAP: dict[SCHEDULER_NAME_VALUES, tuple[Type[SchedulerMixin], dict[str, Any]]] = {
     "ddim": (DDIMScheduler, {}),
     "ddpm": (DDPMScheduler, {}),
-    "deis": (DEISMultistepScheduler, {}),
+    "deis": (DEISMultistepScheduler, {"use_karras_sigmas": False}),
+    "deis_k": (DEISMultistepScheduler, {"use_karras_sigmas": True}),
     "lms": (LMSDiscreteScheduler, {"use_karras_sigmas": False}),
     "lms_k": (LMSDiscreteScheduler, {"use_karras_sigmas": True}),
     "pndm": (PNDMScheduler, {}),
@@ -59,17 +69,28 @@ SCHEDULER_MAP: dict[SCHEDULER_NAME_VALUES, tuple[Type[SchedulerMixin], dict[str,
     "euler": (EulerDiscreteScheduler, {"use_karras_sigmas": False}),
     "euler_k": (EulerDiscreteScheduler, {"use_karras_sigmas": True}),
     "euler_a": (EulerAncestralDiscreteScheduler, {}),
-    "kdpm_2": (KDPM2DiscreteScheduler, {}),
-    "kdpm_2_a": (KDPM2AncestralDiscreteScheduler, {}),
-    "dpmpp_2s": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": False}),
-    "dpmpp_2s_k": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": True}),
-    "dpmpp_2m": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False}),
-    "dpmpp_2m_k": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True}),
-    "dpmpp_2m_sde": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False, "algorithm_type": "sde-dpmsolver++"}),
-    "dpmpp_2m_sde_k": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True, "algorithm_type": "sde-dpmsolver++"}),
+    "kdpm_2": (KDPM2DiscreteScheduler, {"use_karras_sigmas": False}),
+    "kdpm_2_k": (KDPM2DiscreteScheduler, {"use_karras_sigmas": True}),
+    "kdpm_2_a": (KDPM2AncestralDiscreteScheduler, {"use_karras_sigmas": False}),
+    "kdpm_2_a_k": (KDPM2AncestralDiscreteScheduler, {"use_karras_sigmas": True}),
+    "dpmpp_2s": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": False, "solver_order": 2}),
+    "dpmpp_2s_k": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": True, "solver_order": 2}),
+    "dpmpp_2m": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False, "solver_order": 2}),
+    "dpmpp_2m_k": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True, "solver_order": 2}),
+    "dpmpp_2m_sde": (
+        DPMSolverMultistepScheduler,
+        {"use_karras_sigmas": False, "solver_order": 2, "algorithm_type": "sde-dpmsolver++"},
+    ),
+    "dpmpp_2m_sde_k": (
+        DPMSolverMultistepScheduler,
+        {"use_karras_sigmas": True, "solver_order": 2, "algorithm_type": "sde-dpmsolver++"},
+    ),
+    "dpmpp_3m": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False, "solver_order": 3}),
+    "dpmpp_3m_k": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True, "solver_order": 3}),
     "dpmpp_sde": (DPMSolverSDEScheduler, {"use_karras_sigmas": False, "noise_sampler_seed": 0}),
     "dpmpp_sde_k": (DPMSolverSDEScheduler, {"use_karras_sigmas": True, "noise_sampler_seed": 0}),
-    "unipc": (UniPCMultistepScheduler, {"cpu_only": True}),
+    "unipc": (UniPCMultistepScheduler, {"use_karras_sigmas": False, "cpu_only": True}),
+    "unipc_k": (UniPCMultistepScheduler, {"use_karras_sigmas": True, "cpu_only": True}),
     "lcm": (LCMScheduler, {}),
     "tcd": (TCDScheduler, {}),
 }

invokeai/backend/util/hotfixes.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 import diffusers
 import torch
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.loaders import FromOriginalControlNetMixin
+from diffusers.loaders.single_file_model import FromOriginalModelMixin
 from diffusers.models.attention_processor import AttentionProcessor, AttnProcessor
 from diffusers.models.controlnet import ControlNetConditioningEmbedding, ControlNetOutput, zero_module
 from diffusers.models.embeddings import (
@@ -32,7 +32,7 @@ from invokeai.backend.util.logging import InvokeAILogger
 logger = InvokeAILogger.get_logger(__name__)
 
 
-class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalControlNetMixin):
+class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalModelMixin):
     """
     A ControlNet model.
 

invokeai/backend/util/original_weights_storage.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Dict, Iterator, Optional, Tuple
+
+import torch
+
+from invokeai.backend.util.devices import TorchDevice
+
+
+class OriginalWeightsStorage:
+    """A class for tracking the original weights of a model for patch/unpatch operations."""
+
+    def __init__(self, cached_weights: Optional[Dict[str, torch.Tensor]] = None):
+        # The original weights of the model.
+        self._weights: dict[str, torch.Tensor] = {}
+        # The keys of the weights that have been changed (via `save()`) during the lifetime of this instance.
+        self._changed_weights: set[str] = set()
+        if cached_weights:
+            self._weights.update(cached_weights)
+
+    def save(self, key: str, weight: torch.Tensor, copy: bool = True):
+        self._changed_weights.add(key)
+        if key in self._weights:
+            return
+
+        self._weights[key] = weight.detach().to(device=TorchDevice.CPU_DEVICE, copy=copy)
+
+    def get(self, key: str, copy: bool = False) -> Optional[torch.Tensor]:
+        weight = self._weights.get(key, None)
+        if weight is not None and copy:
+            weight = weight.clone()
+        return weight
+
+    def contains(self, key: str) -> bool:
+        return key in self._weights
+
+    def get_changed_weights(self) -> Iterator[Tuple[str, torch.Tensor]]:
+        for key in self._changed_weights:
+            yield key, self._weights[key]

invokeai/frontend/web/public/locales/de.json

@@ -91,7 +91,8 @@
         "viewingDesc": "Bilder in großer Galerie ansehen",
         "tab": "Tabulator",
         "enabled": "Aktiviert",
-        "disabled": "Ausgeschaltet"
+        "disabled": "Ausgeschaltet",
+        "dontShowMeThese": "Zeig mir diese nicht"
     },
     "gallery": {
         "galleryImageSize": "Bildgröße",
@@ -106,7 +107,6 @@
         "download": "Runterladen",
         "setCurrentImage": "Setze aktuelle Bild",
         "featuresWillReset": "Wenn Sie dieses Bild löschen, werden diese Funktionen sofort zurückgesetzt.",
-        "deleteImageBin": "Gelöschte Bilder werden an den Papierkorb Ihres Betriebssystems gesendet.",
         "unableToLoad": "Galerie kann nicht geladen werden",
         "downloadSelection": "Auswahl herunterladen",
         "currentlyInUse": "Dieses Bild wird derzeit in den folgenden Funktionen verwendet:",
@@ -628,7 +628,10 @@
         "private": "Private Ordner",
         "shared": "Geteilte Ordner",
         "archiveBoard": "Ordner archivieren",
-        "archived": "Archiviert"
+        "archived": "Archiviert",
+        "noBoards": "Kein {boardType}} Ordner",
+        "hideBoards": "Ordner verstecken",
+        "viewBoards": "Ordner ansehen"
     },
     "controlnet": {
         "showAdvanced": "Zeige Erweitert",
@@ -943,6 +946,21 @@
             "paragraphs": [
                 "Reduziert das Ausgangsbild auf die Breite und Höhe des Ausgangsbildes. Empfohlen zu aktivieren."
             ]
+        },
+        "structure": {
+            "paragraphs": [
+                "Die Struktur steuert, wie genau sich das Ausgabebild an das Layout des Originals hält. Eine niedrige Struktur erlaubt größere Änderungen, während eine hohe Struktur die ursprüngliche Komposition und das Layout strikter beibehält."
+            ]
+        },
+        "creativity": {
+            "paragraphs": [
+                "Die Kreativität bestimmt den Grad der Freiheit, die dem Modell beim Hinzufügen von Details gewährt wird. Eine niedrige Kreativität hält sich eng an das Originalbild, während eine hohe Kreativität mehr Veränderungen zulässt. Bei der Verwendung eines Prompts erhöht eine hohe Kreativität den Einfluss des Prompts."
+            ]
+        },
+        "scale": {
+            "paragraphs": [
+                "Die Skalierung steuert die Größe des Ausgabebildes und basiert auf einem Vielfachen der Auflösung des Originalbildes. So würde z. B. eine 2-fache Hochskalierung eines 1024x1024px Bildes eine 2048x2048px große Ausgabe erzeugen."
+            ]
         }
     },
     "invocationCache": {

invokeai/frontend/web/public/locales/en.json

@@ -31,7 +31,8 @@
         "deleteBoard": "Delete Board",
         "deleteBoardAndImages": "Delete Board and Images",
         "deleteBoardOnly": "Delete Board Only",
-        "deletedBoardsCannotbeRestored": "Deleted boards cannot be restored",
+        "deletedBoardsCannotbeRestored": "Deleted boards cannot be restored. Selecting 'Delete Board Only' will move images to an uncategorized state.",
+        "deletedPrivateBoardsCannotbeRestored": "Deleted boards cannot be restored. Selecting 'Delete Board Only' will move images to a private uncategorized state for the image's creator.",
         "hideBoards": "Hide Boards",
         "loading": "Loading...",
         "menuItemAutoAdd": "Auto-add to this Board",
@@ -372,7 +373,6 @@
         "dropToUpload": "$t(gallery.drop) to Upload",
         "deleteImage_one": "Delete Image",
         "deleteImage_other": "Delete {{count}} Images",
-        "deleteImageBin": "Deleted images will be sent to your operating system's Bin.",
         "deleteImagePermanent": "Deleted images cannot be restored.",
         "displayBoardSearch": "Display Board Search",
         "displaySearch": "Display Search",
@@ -1052,11 +1052,7 @@
         "remixImage": "Remix Image",
         "usePrompt": "Use Prompt",
         "useSeed": "Use Seed",
-        "width": "Width",
-        "isAllowedToUpscale": {
-            "useX2Model": "Image is too large to upscale with x4 model, use x2 model",
-            "tooLarge": "Image is too large to upscale, select smaller image"
-        }
+        "width": "Width"
     },
     "dynamicPrompts": {
         "showDynamicPrompts": "Show Dynamic Prompts",
@@ -1677,6 +1673,8 @@
     },
     "upscaling": {
         "creativity": "Creativity",
+        "exceedsMaxSize": "Upscale settings exceed max size limit",
+        "exceedsMaxSizeDetails": "Max upscale limit is {{maxUpscaleDimension}}x{{maxUpscaleDimension}} pixels. Please try a smaller image or decrease your scale selection.",
         "structure": "Structure",
         "upscaleModel": "Upscale Model",
         "postProcessingModel": "Post-Processing Model",

invokeai/frontend/web/public/locales/es.json

@@ -88,7 +88,6 @@
         "deleteImage_one": "Eliminar Imagen",
         "deleteImage_many": "",
         "deleteImage_other": "",
-        "deleteImageBin": "Las imágenes eliminadas se enviarán a la papelera de tu sistema operativo.",
         "deleteImagePermanent": "Las imágenes eliminadas no se pueden restaurar.",
         "assets": "Activos",
         "autoAssignBoardOnClick": "Asignación automática de tableros al hacer clic"

invokeai/frontend/web/public/locales/it.json

@@ -89,7 +89,8 @@
         "enabled": "Abilitato",
         "disabled": "Disabilitato",
         "comparingDesc": "Confronta due immagini",
-        "comparing": "Confronta"
+        "comparing": "Confronta",
+        "dontShowMeThese": "Non mostrarmi questi"
     },
     "gallery": {
         "galleryImageSize": "Dimensione dell'immagine",
@@ -101,7 +102,6 @@
         "deleteImage_many": "Elimina {{count}} immagini",
         "deleteImage_other": "Elimina {{count}} immagini",
         "deleteImagePermanent": "Le immagini eliminate non possono essere ripristinate.",
-        "deleteImageBin": "Le immagini eliminate verranno spostate nel cestino del tuo sistema operativo.",
         "assets": "Risorse",
         "autoAssignBoardOnClick": "Assegna automaticamente la bacheca al clic",
         "featuresWillReset": "Se elimini questa immagine, quelle funzionalità verranno immediatamente ripristinate.",
@@ -154,7 +154,9 @@
         "selectAllOnPage": "Seleziona tutto nella pagina",
         "selectAllOnBoard": "Seleziona tutto nella bacheca",
         "exitBoardSearch": "Esci da Ricerca bacheca",
-        "exitSearch": "Esci dalla ricerca"
+        "exitSearch": "Esci dalla ricerca",
+        "go": "Vai",
+        "jump": "Salta"
     },
     "hotkeys": {
         "keyboardShortcuts": "Tasti di scelta rapida",
@@ -571,10 +573,6 @@
         },
         "useCpuNoise": "Usa la CPU per generare rumore",
         "iterations": "Iterazioni",
-        "isAllowedToUpscale": {
-            "useX2Model": "L'immagine è troppo grande per l'ampliamento con il modello x4, utilizza il modello x2",
-            "tooLarge": "L'immagine è troppo grande per l'ampliamento, seleziona un'immagine più piccola"
-        },
         "imageActions": "Azioni Immagine",
         "cfgRescaleMultiplier": "Moltiplicatore riscala CFG",
         "useSize": "Usa Dimensioni",
@@ -630,7 +628,9 @@
         "enableNSFWChecker": "Abilita controllo NSFW",
         "enableInvisibleWatermark": "Abilita filigrana invisibile",
         "enableInformationalPopovers": "Abilita testo informativo a comparsa",
-        "reloadingIn": "Ricaricando in"
+        "reloadingIn": "Ricaricando in",
+        "informationalPopoversDisabled": "Testo informativo a comparsa disabilitato",
+        "informationalPopoversDisabledDesc": "I testi informativi a comparsa sono disabilitati. Attivali nelle impostazioni."
     },
     "toast": {
         "uploadFailed": "Caricamento fallito",
@@ -951,7 +951,7 @@
         "deleteBoardOnly": "solo la Bacheca",
         "deleteBoard": "Elimina Bacheca",
         "deleteBoardAndImages": "Bacheca e Immagini",
-        "deletedBoardsCannotbeRestored": "Le bacheche eliminate non possono essere ripristinate",
+        "deletedBoardsCannotbeRestored": "Le bacheche eliminate non possono essere ripristinate. Selezionando \"Elimina solo bacheca\" le immagini verranno spostate nella bacheca \"Non categorizzato\".",
         "movingImagesToBoard_one": "Spostare {{count}} immagine nella bacheca:",
         "movingImagesToBoard_many": "Spostare {{count}} immagini nella bacheca:",
         "movingImagesToBoard_other": "Spostare {{count}} immagini nella bacheca:",
@@ -972,7 +972,8 @@
         "addPrivateBoard": "Aggiungi una Bacheca Privata",
         "noBoards": "Nessuna bacheca {{boardType}}",
         "hideBoards": "Nascondi bacheche",
-        "viewBoards": "Visualizza bacheche"
+        "viewBoards": "Visualizza bacheche",
+        "deletedPrivateBoardsCannotbeRestored": "Le bacheche cancellate non possono essere ripristinate. Selezionando 'Cancella solo bacheca', le immagini verranno spostate nella bacheca \"Non categorizzato\" privata dell'autore dell'immagine."
     },
     "controlnet": {
         "contentShuffleDescription": "Rimescola il contenuto di un'immagine",
@@ -1516,6 +1517,30 @@
             "paragraphs": [
                 "Metodo con cui applicare l'adattatore IP corrente."
             ]
+        },
+        "scale": {
+            "heading": "Scala",
+            "paragraphs": [
+                "La scala controlla la dimensione dell'immagine di uscita e si basa su un multiplo della risoluzione dell'immagine di ingresso. Ad esempio, un ampliamento 2x su un'immagine 1024x1024 produrrebbe in uscita a 2048x2048."
+            ]
+        },
+        "upscaleModel": {
+            "paragraphs": [
+                "Il modello di ampliamento ridimensiona l'immagine alle dimensioni di uscita prima che vengano aggiunti i dettagli. È possibile utilizzare qualsiasi modello di ampliamento supportato, ma alcuni sono specializzati per diversi tipi di immagini, come foto o disegni al tratto."
+            ],
+            "heading": "Modello di ampliamento"
+        },
+        "creativity": {
+            "heading": "Creatività",
+            "paragraphs": [
+                "La creatività controlla quanta libertà è concessa al modello quando si aggiungono dettagli. Una creatività bassa rimane vicina all'immagine originale, mentre una creatività alta consente più cambiamenti. Quando si usa un prompt, una creatività alta aumenta l'influenza del prompt."
+            ]
+        },
+        "structure": {
+            "heading": "Struttura",
+            "paragraphs": [
+                "La struttura determina quanto l'immagine finale rispecchierà il layout dell'originale. Una struttura bassa permette cambiamenti significativi, mentre una struttura alta conserva la composizione e il layout originali."
+            ]
         }
     },
     "sdxl": {

invokeai/frontend/web/public/locales/ja.json

@@ -109,7 +109,6 @@
         "drop": "ドロップ",
         "dropOrUpload": "$t(gallery.drop) またはアップロード",
         "deleteImage_other": "画像を削除",
-        "deleteImageBin": "削除された画像はOSのゴミ箱に送られます。",
         "deleteImagePermanent": "削除された画像は復元できません。",
         "download": "ダウンロード",
         "unableToLoad": "ギャラリーをロードできません",

invokeai/frontend/web/public/locales/ko.json

@@ -70,7 +70,6 @@
         "gallerySettings": "갤러리 설정",
         "deleteSelection": "선택 항목 삭제",
         "featuresWillReset": "이 이미지를 삭제하면 해당 기능이 즉시 재설정됩니다.",
-        "deleteImageBin": "삭제된 이미지는 운영 체제의 Bin으로 전송됩니다.",
         "assets": "자산",
         "problemDeletingImagesDesc": "하나 이상의 이미지를 삭제할 수 없습니다",
         "noImagesInGallery": "보여줄 이미지가 없음",

invokeai/frontend/web/public/locales/nl.json

@@ -97,7 +97,6 @@
         "noImagesInGallery": "Geen afbeeldingen om te tonen",
         "deleteImage_one": "Verwijder afbeelding",
         "deleteImage_other": "",
-        "deleteImageBin": "Verwijderde afbeeldingen worden naar de prullenbak van je besturingssysteem gestuurd.",
         "deleteImagePermanent": "Verwijderde afbeeldingen kunnen niet worden hersteld.",
         "assets": "Eigen onderdelen",
         "autoAssignBoardOnClick": "Ken automatisch bord toe bij klikken",
@@ -467,10 +466,6 @@
             },
             "imageNotProcessedForControlAdapter": "De afbeelding van controle-adapter #{{number}} is niet verwerkt"
         },
-        "isAllowedToUpscale": {
-            "useX2Model": "Afbeelding is te groot om te vergroten met het x4-model. Gebruik hiervoor het x2-model",
-            "tooLarge": "Afbeelding is te groot om te vergoten. Kies een kleinere afbeelding"
-        },
         "patchmatchDownScaleSize": "Verklein",
         "useCpuNoise": "Gebruik CPU-ruis",
         "imageActions": "Afbeeldingshandeling",

invokeai/frontend/web/public/locales/ru.json

@@ -100,7 +100,6 @@
         "loadMore": "Показать больше",
         "noImagesInGallery": "Изображений нет",
         "deleteImagePermanent": "Удаленные изображения невозможно восстановить.",
-        "deleteImageBin": "Удаленные изображения будут отправлены в корзину вашей операционной системы.",
         "deleteImage_one": "Удалить изображение",
         "deleteImage_few": "Удалить {{count}} изображения",
         "deleteImage_many": "Удалить {{count}} изображений",
@@ -567,10 +566,6 @@
                 "ipAdapterNoImageSelected": "изображение IP-адаптера не выбрано"
             }
         },
-        "isAllowedToUpscale": {
-            "useX2Model": "Изображение слишком велико для увеличения с помощью модели x4. Используйте модель x2",
-            "tooLarge": "Изображение слишком велико для увеличения. Выберите изображение меньшего размера"
-        },
         "cfgRescaleMultiplier": "Множитель масштабирования CFG",
         "patchmatchDownScaleSize": "уменьшить",
         "useCpuNoise": "Использовать шум CPU",

invokeai/frontend/web/public/locales/tr.json

@@ -278,7 +278,6 @@
         "enable": "Aç"
     },
     "gallery": {
-        "deleteImageBin": "Silinen görseller işletim sisteminin çöp kutusuna gönderilir.",
         "deleteImagePermanent": "Silinen görseller geri getirilemez.",
         "assets": "Özkaynaklar",
         "autoAssignBoardOnClick": "Tıklanan Panoya Otomatik Atama",
@@ -622,10 +621,6 @@
         "controlNetControlMode": "Yönetim Kipi",
         "general": "Genel",
         "seamlessYAxis": "Dikişsiz Döşeme Y Ekseni",
-        "isAllowedToUpscale": {
-            "tooLarge": "Görsel, büyütme işlemi için çok büyük, daha küçük bir boyut seçin",
-            "useX2Model": "Görsel 4 kat büyütme işlemi için çok geniş, 2 kat büyütmeyi kullanın"
-        },
         "maskBlur": "Bulandırma",
         "images": "Görseller",
         "info": "Bilgi",

invokeai/frontend/web/public/locales/zh_CN.json

@@ -6,7 +6,7 @@
         "settingsLabel": "设置",
         "img2img": "图生图",
         "unifiedCanvas": "统一画布",
-        "nodes": "工作流编辑器",
+        "nodes": "工作流",
         "upload": "上传",
         "load": "加载",
         "statusDisconnected": "未连接",
@@ -86,7 +86,12 @@
         "editing": "编辑中",
         "green": "绿",
         "blue": "蓝",
-        "editingDesc": "在控制图层画布上编辑"
+        "editingDesc": "在控制图层画布上编辑",
+        "goTo": "前往",
+        "dontShowMeThese": "请勿显示这些内容",
+        "beta": "测试版",
+        "toResolve": "解决",
+        "tab": "标签页"
     },
     "gallery": {
         "galleryImageSize": "预览大小",
@@ -94,8 +99,7 @@
         "autoSwitchNewImages": "自动切换到新图像",
         "loadMore": "加载更多",
         "noImagesInGallery": "无图像可用于显示",
-        "deleteImage_other": "删除图片",
-        "deleteImageBin": "被删除的图片会发送到你操作系统的回收站。",
+        "deleteImage_other": "删除{{count}}张图片",
         "deleteImagePermanent": "删除的图片无法被恢复。",
         "assets": "素材",
         "autoAssignBoardOnClick": "点击后自动分配面板",
@@ -133,7 +137,24 @@
         "hover": "悬停",
         "selectAllOnPage": "选择本页全部",
         "swapImages": "交换图像",
-        "compareOptions": "比较选项"
+        "compareOptions": "比较选项",
+        "exitBoardSearch": "退出面板搜索",
+        "exitSearch": "退出搜索",
+        "oldestFirst": "最旧在前",
+        "sortDirection": "排序方向",
+        "showStarredImagesFirst": "优先显示收藏的图片",
+        "compareHelp3": "按 <Kbd>C</Kbd> 键对调正在比较的图片。",
+        "showArchivedBoards": "显示已归档的面板",
+        "newestFirst": "最新在前",
+        "compareHelp4": "按 <Kbd>Z</Kbd>或 <Kbd>Esc</Kbd> 键退出。",
+        "searchImages": "按元数据搜索",
+        "jump": "跳过",
+        "compareHelp2": "按 <Kbd>M</Kbd> 键切换不同的比较模式。",
+        "displayBoardSearch": "显示面板搜索",
+        "displaySearch": "显示搜索",
+        "stretchToFit": "拉伸以适应",
+        "exitCompare": "退出对比",
+        "compareHelp1": "在点击图库中的图片或使用箭头键切换比较图片时，请按住<Kbd>Alt</Kbd> 键。"
     },
     "hotkeys": {
         "keyboardShortcuts": "快捷键",
@@ -348,7 +369,19 @@
             "desc": "打开和关闭选项和图库面板",
             "title": "开关选项和图库"
         },
-        "clearSearch": "清除检索项"
+        "clearSearch": "清除检索项",
+        "toggleViewer": {
+            "desc": "在当前标签页的图片查看模式和编辑工作区之间切换.",
+            "title": "切换图片查看器"
+        },
+        "postProcess": {
+            "desc": "使用选定的后期处理模型对当前图像进行处理",
+            "title": "处理图像"
+        },
+        "remixImage": {
+            "title": "重新混合图像",
+            "desc": "使用当前图像的所有参数，但不包括随机种子"
+        }
     },
     "modelManager": {
         "modelManager": "模型管理器",
@@ -396,14 +429,71 @@
         "modelConversionFailed": "模型转换失败",
         "baseModel": "基底模型",
         "convertingModelBegin": "模型转换中. 请稍候.",
-        "predictionType": "预测类型（适用于 Stable Diffusion 2.x 模型和部分 Stable Diffusion 1.x 模型）",
+        "predictionType": "预测类型",
         "advanced": "高级",
         "modelType": "模型类别",
         "variant": "变体",
         "vae": "VAE",
         "alpha": "Alpha",
         "vaePrecision": "VAE 精度",
-        "noModelSelected": "无选中的模型"
+        "noModelSelected": "无选中的模型",
+        "modelImageUpdateFailed": "模型图像更新失败",
+        "scanFolder": "扫描文件夹",
+        "path": "路径",
+        "pathToConfig": "配置路径",
+        "cancel": "取消",
+        "hfTokenUnableToVerify": "无法验证HuggingFace token",
+        "install": "安装",
+        "simpleModelPlaceholder": "本地文件或diffusers文件夹的URL或路径",
+        "hfTokenInvalidErrorMessage": "无效或缺失的HuggingFace token.",
+        "noModelsInstalledDesc1": "安装模型时使用",
+        "inplaceInstallDesc": "安装模型时，不复制文件，直接从原位置加载。如果关闭此选项，模型文件将在安装过程中被复制到Invoke管理的模型文件夹中.",
+        "installAll": "安装全部",
+        "noModelsInstalled": "无已安装的模型",
+        "urlOrLocalPathHelper": "链接应该指向单个文件.本地路径可以指向单个文件,或者对于单个扩散模型(diffusers model),可以指向一个文件夹.",
+        "modelSettings": "模型设置",
+        "useDefaultSettings": "使用默认设置",
+        "scanPlaceholder": "本地文件夹路径",
+        "installRepo": "安装仓库",
+        "modelImageDeleted": "模型图像已删除",
+        "modelImageDeleteFailed": "模型图像删除失败",
+        "scanFolderHelper": "此文件夹将进行递归扫描以寻找模型.对于大型文件夹,这可能需要一些时间.",
+        "scanResults": "扫描结果",
+        "noMatchingModels": "无匹配的模型",
+        "pruneTooltip": "清理队列中已完成的导入任务",
+        "urlOrLocalPath": "链接或本地路径",
+        "localOnly": "仅本地",
+        "hfTokenHelperText": "需要HuggingFace token才能使用Checkpoint模型。点击此处创建或获取您的token.",
+        "huggingFaceHelper": "如果在此代码库中检测到多个模型，系统将提示您选择其中一个进行安装.",
+        "hfTokenUnableToVerifyErrorMessage": "无法验证HuggingFace token.可能是网络问题所致.请稍后再试.",
+        "hfTokenSaved": "HuggingFace token已保存",
+        "imageEncoderModelId": "图像编码器模型ID",
+        "modelImageUpdated": "模型图像已更新",
+        "modelName": "模型名称",
+        "prune": "清理",
+        "repoVariant": "代码库版本",
+        "defaultSettings": "默认设置",
+        "inplaceInstall": "就地安装",
+        "main": "主界面",
+        "starterModels": "初始模型",
+        "installQueue": "安装队列",
+        "hfTokenInvalidErrorMessage2": "更新于其中 ",
+        "hfTokenInvalid": "无效或缺失的HuggingFace token",
+        "mainModelTriggerPhrases": "主模型触发词",
+        "typePhraseHere": "在此输入触发词",
+        "triggerPhrases": "触发词",
+        "metadata": "元数据",
+        "deleteModelImage": "删除模型图片",
+        "edit": "编辑",
+        "source": "来源",
+        "uploadImage": "上传图像",
+        "addModels": "添加模型",
+        "textualInversions": "文本逆向生成",
+        "upcastAttention": "是否为高精度权重",
+        "defaultSettingsSaved": "默认设置已保存",
+        "huggingFacePlaceholder": "所有者或模型名称",
+        "huggingFaceRepoID": "HuggingFace仓库ID",
+        "loraTriggerPhrases": "LoRA 触发词"
     },
     "parameters": {
         "images": "图像",
@@ -446,7 +536,7 @@
         "scheduler": "调度器",
         "general": "通用",
         "controlNetControlMode": "控制模式",
-        "maskBlur": "模糊",
+        "maskBlur": "遮罩模糊",
         "invoke": {
             "noNodesInGraph": "节点图中无节点",
             "noModelSelected": "无已选中的模型",
@@ -460,7 +550,21 @@
             "noPrompts": "没有已生成的提示词",
             "noControlImageForControlAdapter": "有 #{{number}} 个 Control Adapter 缺失控制图像",
             "noModelForControlAdapter": "有 #{{number}} 个 Control Adapter 没有选择模型。",
-            "incompatibleBaseModelForControlAdapter": "有 #{{number}} 个 Control Adapter 模型与主模型不兼容。"
+            "incompatibleBaseModelForControlAdapter": "有 #{{number}} 个 Control Adapter 模型与主模型不兼容。",
+            "layer": {
+                "initialImageNoImageSelected": "未选择初始图像",
+                "controlAdapterImageNotProcessed": "Control Adapter图像尚未处理",
+                "ipAdapterNoModelSelected": "未选择IP adapter",
+                "controlAdapterNoModelSelected": "未选择Control Adapter模型",
+                "controlAdapterNoImageSelected": "未选择Control Adapter图像",
+                "rgNoPromptsOrIPAdapters": "无文本提示或IP Adapters",
+                "controlAdapterIncompatibleBaseModel": "Control Adapter的基础模型不兼容",
+                "ipAdapterIncompatibleBaseModel": "IP Adapter的基础模型不兼容",
+                "t2iAdapterIncompatibleDimensions": "T2I Adapter需要图像尺寸为{{multiple}}的倍数",
+                "ipAdapterNoImageSelected": "未选择IP Adapter图像",
+                "rgNoRegion": "未选择区域"
+            },
+            "imageNotProcessedForControlAdapter": "Control Adapter #{{number}} 的图像未处理"
         },
         "patchmatchDownScaleSize": "缩小",
         "clipSkip": "CLIP 跳过层",
@@ -468,10 +572,6 @@
         "coherenceMode": "模式",
         "imageActions": "图像操作",
         "iterations": "迭代数",
-        "isAllowedToUpscale": {
-            "useX2Model": "图像太大，无法使用 x4 模型，使用 x2 模型作为替代",
-            "tooLarge": "图像太大无法进行放大，请选择更小的图像"
-        },
         "cfgRescaleMultiplier": "CFG 重缩放倍数",
         "useSize": "使用尺寸",
         "setToOptimalSize": "优化模型大小",
@@ -479,7 +579,21 @@
         "lockAspectRatio": "锁定纵横比",
         "swapDimensions": "交换尺寸",
         "aspect": "纵横",
-        "setToOptimalSizeTooLarge": "$t(parameters.setToOptimalSize) （可能过大）"
+        "setToOptimalSizeTooLarge": "$t(parameters.setToOptimalSize) （可能过大）",
+        "globalNegativePromptPlaceholder": "全局反向提示词",
+        "remixImage": "重新混合图像",
+        "coherenceEdgeSize": "边缘尺寸",
+        "postProcessing": "后处理（Shift + U）",
+        "infillMosaicTileWidth": "瓦片宽度",
+        "sendToUpscale": "发送到放大",
+        "processImage": "处理图像",
+        "globalPositivePromptPlaceholder": "全局正向提示词",
+        "globalSettings": "全局设置",
+        "infillMosaicTileHeight": "瓦片高度",
+        "infillMosaicMinColor": "最小颜色",
+        "infillMosaicMaxColor": "最大颜色",
+        "infillColorValue": "填充颜色",
+        "coherenceMinDenoise": "最小去噪"
     },
     "settings": {
         "models": "模型",
@@ -509,7 +623,9 @@
         "enableNSFWChecker": "启用成人内容检测器",
         "enableInvisibleWatermark": "启用不可见水印",
         "enableInformationalPopovers": "启用信息弹窗",
-        "reloadingIn": "重新加载中"
+        "reloadingIn": "重新加载中",
+        "informationalPopoversDisabled": "信息提示框已禁用",
+        "informationalPopoversDisabledDesc": "信息提示框已被禁用.请在设置中重新启用."
     },
     "toast": {
         "uploadFailed": "上传失败",
@@ -518,16 +634,16 @@
         "canvasMerged": "画布已合并",
         "sentToImageToImage": "已发送到图生图",
         "sentToUnifiedCanvas": "已发送到统一画布",
-        "parametersNotSet": "参数未设定",
+        "parametersNotSet": "参数未恢复",
         "metadataLoadFailed": "加载元数据失败",
         "uploadFailedInvalidUploadDesc": "必须是单张的 PNG 或 JPEG 图片",
         "connected": "服务器连接",
-        "parameterSet": "参数已设定",
-        "parameterNotSet": "参数未设定",
+        "parameterSet": "参数已恢复",
+        "parameterNotSet": "参数未恢复",
         "serverError": "服务器错误",
         "canceled": "处理取消",
         "problemCopyingImage": "无法复制图像",
-        "modelAddedSimple": "已添加模型",
+        "modelAddedSimple": "模型已加入队列",
         "imageSavingFailed": "图像保存失败",
         "canvasSentControlnetAssets": "画布已发送到 ControlNet & 素材",
         "problemCopyingCanvasDesc": "无法导出基础层",
@@ -557,12 +673,28 @@
         "canvasSavedGallery": "画布已保存到图库",
         "imageUploadFailed": "图像上传失败",
         "problemImportingMask": "导入遮罩时出现问题",
-        "baseModelChangedCleared_other": "基础模型已更改, 已清除或禁用 {{count}} 个不兼容的子模型",
+        "baseModelChangedCleared_other": "已清除或禁用{{count}}个不兼容的子模型",
         "setAsCanvasInitialImage": "设为画布初始图像",
         "invalidUpload": "无效的上传",
         "problemDeletingWorkflow": "删除工作流时出现问题",
         "workflowDeleted": "已删除工作流",
-        "problemRetrievingWorkflow": "检索工作流时发生问题"
+        "problemRetrievingWorkflow": "检索工作流时发生问题",
+        "baseModelChanged": "基础模型已更改",
+        "problemDownloadingImage": "无法下载图像",
+        "outOfMemoryError": "内存不足错误",
+        "parameters": "参数",
+        "resetInitialImage": "重置初始图像",
+        "parameterNotSetDescWithMessage": "无法恢复 {{parameter}}: {{message}}",
+        "parameterSetDesc": "已恢复 {{parameter}}",
+        "parameterNotSetDesc": "无法恢复{{parameter}}",
+        "sessionRef": "会话: {{sessionId}}",
+        "somethingWentWrong": "出现错误",
+        "prunedQueue": "已清理队列",
+        "uploadInitialImage": "上传初始图像",
+        "outOfMemoryErrorDesc": "您当前的生成设置已超出系统处理能力.请调整设置后再次尝试.",
+        "parametersSet": "参数已恢复",
+        "errorCopied": "错误信息已复制",
+        "modelImportCanceled": "模型导入已取消"
     },
     "unifiedCanvas": {
         "layer": "图层",
@@ -616,7 +748,15 @@
         "antialiasing": "抗锯齿",
         "showResultsOn": "显示结果 (开)",
         "showResultsOff": "显示结果 (关)",
-        "saveMask": "保存 $t(unifiedCanvas.mask)"
+        "saveMask": "保存 $t(unifiedCanvas.mask)",
+        "coherenceModeBoxBlur": "盒子模糊",
+        "showBoundingBox": "显示边界框",
+        "coherenceModeGaussianBlur": "高斯模糊",
+        "coherenceModeStaged": "分阶段",
+        "hideBoundingBox": "隐藏边界框",
+        "initialFitImageSize": "在拖放时调整图像大小以适配",
+        "invertBrushSizeScrollDirection": "反转滚动操作以调整画笔大小",
+        "discardCurrent": "放弃当前设置"
     },
     "accessibility": {
         "invokeProgressBar": "Invoke 进度条",
@@ -746,11 +886,11 @@
         "unableToExtractSchemaNameFromRef": "无法从参考中提取架构名",
         "unknownOutput": "未知输出：{{name}}",
         "unknownErrorValidatingWorkflow": "验证工作流时出现未知错误",
-        "collectionFieldType": "{{name}} 合集",
+        "collectionFieldType": "{{name}}(合集)",
         "unknownNodeType": "未知节点类型",
         "targetNodeDoesNotExist": "无效的边缘：{{node}} 的目标/输入节点不存在",
         "unknownFieldType": "$t(nodes.unknownField) 类型：{{type}}",
-        "collectionOrScalarFieldType": "{{name}} 合集 | 标量",
+        "collectionOrScalarFieldType": "{{name}} (单一项目或项目集合)",
         "nodeVersion": "节点版本",
         "deletedInvalidEdge": "已删除无效的边缘 {{source}} -> {{target}}",
         "unknownInput": "未知输入：{{name}}",
@@ -759,7 +899,27 @@
         "newWorkflow": "新建工作流",
         "newWorkflowDesc": "是否创建一个新的工作流？",
         "newWorkflowDesc2": "当前工作流有未保存的更改。",
-        "unsupportedAnyOfLength": "联合（union）数据类型数目过多 ({{count}})"
+        "unsupportedAnyOfLength": "联合（union）数据类型数目过多 ({{count}})",
+        "resetToDefaultValue": "重置为默认值",
+        "clearWorkflowDesc2": "您当前的工作流有未保存的更改.",
+        "missingNode": "缺少调用节点",
+        "missingInvocationTemplate": "缺少调用模版",
+        "noFieldsViewMode": "此工作流程未选择任何要显示的字段.请查看完整工作流程以进行配置.",
+        "reorderLinearView": "调整线性视图顺序",
+        "viewMode": "在线性视图中使用",
+        "showEdgeLabelsHelp": "在边缘上显示标签，指示连接的节点",
+        "cannotMixAndMatchCollectionItemTypes": "集合项目类型不能混用",
+        "missingFieldTemplate": "缺少字段模板",
+        "editMode": "在工作流编辑器中编辑",
+        "showEdgeLabels": "显示边缘标签",
+        "clearWorkflowDesc": "是否清除当前工作流并创建新的？",
+        "graph": "图表",
+        "noGraph": "无图表",
+        "edit": "编辑",
+        "clearWorkflow": "清除工作流",
+        "imageAccessError": "无法找到图像 {{image_name}}，正在恢复默认设置",
+        "boardAccessError": "无法找到面板 {{board_id}}，正在恢复默认设置",
+        "modelAccessError": "无法找到模型 {{key}}，正在恢复默认设置"
     },
     "controlnet": {
         "resize": "直接缩放",
@@ -799,7 +959,7 @@
         "mediapipeFaceDescription": "使用 Mediapipe 检测面部",
         "depthZoeDescription": "使用 Zoe 生成深度图",
         "hedDescription": "整体嵌套边缘检测",
-        "setControlImageDimensions": "设定控制图像尺寸宽/高为",
+        "setControlImageDimensions": "复制尺寸到宽度/高度(为模型优化)",
         "amult": "角度倍率 (a_mult)",
         "bgth": "背景移除阈值 (bg_th)",
         "lineartAnimeDescription": "动漫风格线稿处理",
@@ -810,7 +970,7 @@
         "addControlNet": "添加 $t(common.controlNet)",
         "addIPAdapter": "添加 $t(common.ipAdapter)",
         "safe": "保守模式",
-        "scribble": "草绘 (scribble)",
+        "scribble": "草绘",
         "maxFaces": "最大面部数",
         "pidi": "PIDI",
         "normalBae": "Normal BAE",
@@ -925,7 +1085,8 @@
         "steps": "步数",
         "posStylePrompt": "正向样式提示词",
         "refiner": "Refiner",
-        "freePromptStyle": "手动输入样式提示词"
+        "freePromptStyle": "手动输入样式提示词",
+        "refinerSteps": "精炼步数"
     },
     "metadata": {
         "positivePrompt": "正向提示词",
@@ -952,7 +1113,12 @@
         "recallParameters": "召回参数",
         "noRecallParameters": "未找到要召回的参数",
         "vae": "VAE",
-        "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)"
+        "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)",
+        "allPrompts": "所有提示",
+        "parsingFailed": "解析失败",
+        "recallParameter": "调用{{label}}",
+        "imageDimensions": "图像尺寸",
+        "parameterSet": "已设置参数{{parameter}}"
     },
     "models": {
         "noMatchingModels": "无相匹配的模型",
@@ -965,7 +1131,8 @@
         "esrganModel": "ESRGAN 模型",
         "addLora": "添加 LoRA",
         "lora": "LoRA",
-        "defaultVAE": "默认 VAE"
+        "defaultVAE": "默认 VAE",
+        "concepts": "概念"
     },
     "boards": {
         "autoAddBoard": "自动添加面板",
@@ -987,8 +1154,23 @@
         "deleteBoardOnly": "仅删除面板",
         "deleteBoard": "删除面板",
         "deleteBoardAndImages": "删除面板和图像",
-        "deletedBoardsCannotbeRestored": "已删除的面板无法被恢复",
-        "movingImagesToBoard_other": "移动 {{count}} 张图像到面板："
+        "deletedBoardsCannotbeRestored": "删除的面板无法恢复。选择“仅删除面板”选项后，相关图片将会被移至未分类区域。",
+        "movingImagesToBoard_other": "移动 {{count}} 张图像到面板：",
+        "selectedForAutoAdd": "已选中自动添加",
+        "hideBoards": "隐藏面板",
+        "noBoards": "没有{{boardType}}类型的面板",
+        "unarchiveBoard": "恢复面板",
+        "viewBoards": "查看面板",
+        "addPrivateBoard": "创建私密面板",
+        "addSharedBoard": "创建共享面板",
+        "boards": "面板",
+        "imagesWithCount_other": "{{count}}张图片",
+        "deletedPrivateBoardsCannotbeRestored": "删除的面板无法恢复。选择“仅删除面板”后，相关图片将会被移至图片创建者的私密未分类区域。",
+        "private": "私密面板",
+        "shared": "共享面板",
+        "archiveBoard": "归档面板",
+        "archived": "已归档",
+        "assetsWithCount_other": "{{count}}项资源"
     },
     "dynamicPrompts": {
         "seedBehaviour": {
@@ -1030,32 +1212,33 @@
         "paramVAEPrecision": {
             "heading": "VAE 精度",
             "paragraphs": [
-                "VAE 编解码过程种使用的精度。FP16/半精度以微小的图像变化为代价提高效率。"
+                "在VAE编码和解码过程中使用的精度.",
+                "Fp16/半精度更高效，但可能会造成图像的一些微小差异."
             ]
         },
         "compositingCoherenceMode": {
             "heading": "模式",
             "paragraphs": [
-                "一致性层模式。"
+                "用于将新生成的遮罩区域与原图像融合的方法."
             ]
         },
         "controlNetResizeMode": {
             "heading": "缩放模式",
             "paragraphs": [
-                "ControlNet 输入图像适应输出图像大小的方法。"
+                "调整Control Adapter输入图像大小以适应输出图像尺寸的方法."
             ]
         },
         "clipSkip": {
             "paragraphs": [
-                "选择要跳过 CLIP 模型多少层。",
-                "部分模型跳过特定数值的层时效果会更好。"
+                "跳过CLIP模型的层数.",
+                "某些模型更适合结合CLIP Skip功能使用."
             ],
             "heading": "CLIP 跳过层"
         },
         "paramModel": {
             "heading": "模型",
             "paragraphs": [
-                "用于去噪过程的模型。"
+                "用于图像生成的模型.不同的模型经过训练,专门用于产生不同的美学效果和内容."
             ]
         },
         "paramIterations": {
@@ -1087,19 +1270,21 @@
         "paramScheduler": {
             "heading": "调度器",
             "paragraphs": [
-                "调度器 (采样器) 定义如何在图像迭代过程中添加噪声，或者定义如何根据一个模型的输出来更新采样。"
+                "生成过程中所使用的调度器.",
+                "每个调度器决定了在生成过程中如何逐步向图像添加噪声，或者如何根据模型的输出更新样本."
             ]
         },
         "controlNetWeight": {
             "heading": "权重",
             "paragraphs": [
-                "ControlNet 对生成图像的影响强度。"
+                "Control Adapter的权重.权重越高,对最终图像的影响越大."
             ]
         },
         "paramCFGScale": {
             "heading": "CFG 等级",
             "paragraphs": [
-                "控制提示词对生成过程的影响程度。"
+                "控制提示对生成过程的影响程度.",
+                "较高的CFG比例值可能会导致生成结果过度饱和和扭曲. "
             ]
         },
         "paramSteps": {
@@ -1117,28 +1302,29 @@
             ]
         },
         "lora": {
-            "heading": "LoRA 权重",
+            "heading": "LoRA",
             "paragraphs": [
-                "更高的 LoRA 权重会对最终图像产生更大的影响。"
+                "与基础模型结合使用的轻量级模型."
             ]
         },
         "infillMethod": {
             "heading": "填充方法",
             "paragraphs": [
-                "填充选定区域的方式。"
+                "在重绘过程中使用的填充方法."
             ]
         },
         "controlNetBeginEnd": {
             "heading": "开始 / 结束步数百分比",
             "paragraphs": [
-                "去噪过程中在哪部分步数应用 ControlNet。",
-                "在组合处理开始阶段应用 ControlNet，且在引导细节生成的结束阶段应用 ControlNet。"
+                "去噪过程中将应用Control Adapter 的部分.",
+                "通常，在去噪过程初期应用的Control Adapters用于指导整体构图，而在后期应用的Control Adapters则用于调整细节。"
             ]
         },
         "scaleBeforeProcessing": {
             "heading": "处理前缩放",
             "paragraphs": [
-                "生成图像前将所选区域缩放为最适合模型的大小。"
+                "\"自动\"选项会在图像生成之前将所选区域调整到最适合模型的大小.",
+                "\"手动\"选项允许您在图像生成之前自行选择所选区域的宽度和高度."
             ]
         },
         "paramDenoisingStrength": {
@@ -1152,13 +1338,13 @@
             "heading": "种子",
             "paragraphs": [
                 "控制用于生成的起始噪声。",
-                "禁用 “随机种子” 来以相同设置生成相同的结果。"
+                "禁用\"随机\"选项,以使用相同的生成设置产生一致的结果."
             ]
         },
         "controlNetControlMode": {
             "heading": "控制模式",
             "paragraphs": [
-                "给提示词或 ControlNet 增加更大的权重。"
+                "在提示词和ControlNet之间分配更多的权重."
             ]
         },
         "dynamicPrompts": {
@@ -1199,7 +1385,171 @@
         "paramCFGRescaleMultiplier": {
             "heading": "CFG 重缩放倍数",
             "paragraphs": [
-                "CFG 引导的重缩放倍率，用于通过 zero-terminal SNR (ztsnr) 训练的模型。推荐设为 0.7。"
+                "CFG指导的重缩放乘数，适用于使用零终端信噪比（ztsnr）训练的模型.",
+                "对于这些模型,建议的数值为0.7."
+            ]
+        },
+        "imageFit": {
+            "paragraphs": [
+                "将初始图像调整到与输出图像相同的宽度和高度.建议启用此功能."
+            ],
+            "heading": "将初始图像适配到输出大小"
+        },
+        "paramAspect": {
+            "paragraphs": [
+                "生成图像的宽高比.调整宽高比会相应地更新图像的宽度和高度.",
+                "选择\"优化\"将把图像的宽度和高度设置为所选模型的最优尺寸."
+            ],
+            "heading": "宽高比"
+        },
+        "refinerSteps": {
+            "paragraphs": [
+                "在图像生成过程中的细化阶段将执行的步骤数.",
+                "与生成步骤相似."
+            ],
+            "heading": "步数"
+        },
+        "compositingMaskBlur": {
+            "heading": "遮罩模糊",
+            "paragraphs": [
+                "遮罩的模糊范围."
+            ]
+        },
+        "compositingCoherenceMinDenoise": {
+            "paragraphs": [
+                "连贯模式下的最小去噪力度",
+                "在图像修复或重绘过程中，连贯区域的最小去噪力度"
+            ],
+            "heading": "最小去噪"
+        },
+        "loraWeight": {
+            "paragraphs": [
+                "LoRA的权重,权重越高对最终图像的影响越大."
+            ],
+            "heading": "权重"
+        },
+        "paramHrf": {
+            "heading": "启用高分辨率修复",
+            "paragraphs": [
+                "以高于模型最优分辨率的大分辨率生成高质量图像.这通常用于防止生成图像中出现重复内容."
+            ]
+        },
+        "compositingCoherenceEdgeSize": {
+            "paragraphs": [
+                "连贯处理的边缘尺寸."
+            ],
+            "heading": "边缘尺寸"
+        },
+        "paramWidth": {
+            "paragraphs": [
+                "生成图像的宽度.必须是8的倍数."
+            ],
+            "heading": "宽度"
+        },
+        "refinerScheduler": {
+            "paragraphs": [
+                "在图像生成过程中的细化阶段所使用的调度程序.",
+                "与生成调度程序相似."
+            ],
+            "heading": "调度器"
+        },
+        "seamlessTilingXAxis": {
+            "paragraphs": [
+                "沿水平轴将图像进行无缝平铺."
+            ],
+            "heading": "无缝平铺X轴"
+        },
+        "paramUpscaleMethod": {
+            "heading": "放大方法",
+            "paragraphs": [
+                "用于高分辨率修复的图像放大方法."
+            ]
+        },
+        "refinerModel": {
+            "paragraphs": [
+                "在图像生成过程中的细化阶段所使用的模型.",
+                "与生成模型相似."
+            ],
+            "heading": "精炼模型"
+        },
+        "paramHeight": {
+            "paragraphs": [
+                "生成图像的高度.必须是8的倍数."
+            ],
+            "heading": "高"
+        },
+        "patchmatchDownScaleSize": {
+            "heading": "缩小",
+            "paragraphs": [
+                "在填充之前图像缩小的程度.",
+                "较高的缩小比例会提升处理速度，但可能会降低图像质量."
+            ]
+        },
+        "seamlessTilingYAxis": {
+            "heading": "Y轴上的无缝平铺",
+            "paragraphs": [
+                "沿垂直轴将图像进行无缝平铺."
+            ]
+        },
+        "ipAdapterMethod": {
+            "paragraphs": [
+                "当前IP Adapter的应用方法."
+            ],
+            "heading": "方法"
+        },
+        "controlNetProcessor": {
+            "paragraphs": [
+                "处理输入图像以引导生成过程的方法.不同的处理器会在生成图像中产生不同的效果或风格."
+            ],
+            "heading": "处理器"
+        },
+        "refinerPositiveAestheticScore": {
+            "paragraphs": [
+                "根据训练数据，对生成结果进行加权，使其更接近于具有高美学评分的图像."
+            ],
+            "heading": "正面美学评分"
+        },
+        "refinerStart": {
+            "paragraphs": [
+                "在图像生成过程中精炼阶段开始被使用的时刻.",
+                "0表示精炼器将全程参与图像生成,0.8表示细化器仅在生成过程的最后20%阶段被使用."
+            ],
+            "heading": "精炼开始"
+        },
+        "refinerCfgScale": {
+            "paragraphs": [
+                "控制提示对生成过程的影响程度.",
+                "与生成CFG Scale相似."
+            ]
+        },
+        "structure": {
+            "heading": "结构",
+            "paragraphs": [
+                "结构决定了输出图像在多大程度上保持原始图像的布局.较低的结构设置允许进行较大的变化,而较高的结构设置则会严格保持原始图像的构图和布局."
+            ]
+        },
+        "creativity": {
+            "paragraphs": [
+                "创造力决定了模型在添加细节时的自由度.较低的创造力会使生成结果更接近原始图像，而较高的创造力则允许更多的变化.在使用提示时，较高的创造力会增加提示对生成结果的影响."
+            ],
+            "heading": "创造力"
+        },
+        "refinerNegativeAestheticScore": {
+            "paragraphs": [
+                "根据训练数据，对生成结果进行加权，使其更接近于具有低美学评分的图像."
+            ],
+            "heading": "负面美学评分"
+        },
+        "upscaleModel": {
+            "heading": "放大模型",
+            "paragraphs": [
+                "上采样模型在添加细节之前将图像放大到输出尺寸.虽然可以使用任何支持的上采样模型，但有些模型更适合处理特定类型的图像，例如照片或线条画."
+            ]
+        },
+        "scale": {
+            "heading": "缩放",
+            "paragraphs": [
+                "比例控制决定了输出图像的大小,它是基于输入图像分辨率的倍数来计算的.例如对一张1024x1024的图像进行2倍上采样，将会得到一张2048x2048的输出图像."
             ]
         }
     },
@@ -1259,7 +1609,16 @@
         "updated": "已更新",
         "userWorkflows": "我的工作流",
         "projectWorkflows": "项目工作流",
-        "opened": "已打开"
+        "opened": "已打开",
+        "noRecentWorkflows": "没有最近的工作流",
+        "workflowCleared": "工作流已清除",
+        "saveWorkflowToProject": "保存工作流到项目",
+        "noWorkflows": "无工作流",
+        "convertGraph": "转换图表",
+        "loadWorkflow": "$t(common.load) 工作流",
+        "noUserWorkflows": "没有用户工作流",
+        "loadFromGraph": "从图表加载工作流",
+        "autoLayout": "自动布局"
     },
     "app": {
         "storeNotInitialized": "商店尚未初始化"
@@ -1287,5 +1646,68 @@
     "prompt": {
         "addPromptTrigger": "添加提示词触发器",
         "noMatchingTriggers": "没有匹配的触发器"
+    },
+    "controlLayers": {
+        "autoNegative": "自动反向",
+        "opacityFilter": "透明度滤镜",
+        "deleteAll": "删除所有",
+        "moveForward": "向前移动",
+        "layers_other": "层",
+        "globalControlAdapterLayer": "全局 $t(controlnet.controlAdapter_one) $t(unifiedCanvas.layer)",
+        "moveBackward": "向后移动",
+        "regionalGuidance": "区域导向",
+        "controlLayers": "控制层",
+        "moveToBack": "移动到后面",
+        "brushSize": "笔刷尺寸",
+        "moveToFront": "移动到前面",
+        "addLayer": "添加层",
+        "deletePrompt": "删除提示词",
+        "resetRegion": "重置区域",
+        "debugLayers": "调试图层",
+        "maskPreviewColor": "遮罩预览颜色",
+        "addPositivePrompt": "添加 $t(common.positivePrompt)",
+        "addNegativePrompt": "添加 $t(common.negativePrompt)",
+        "addIPAdapter": "添加 $t(common.ipAdapter)",
+        "globalIPAdapterLayer": "全局 $t(common.ipAdapter) $t(unifiedCanvas.layer)",
+        "globalInitialImage": "全局初始图像",
+        "noLayersAdded": "没有层被添加",
+        "globalIPAdapter": "全局 $t(common.ipAdapter)",
+        "resetProcessor": "重置处理器至默认值",
+        "globalMaskOpacity": "全局遮罩透明度",
+        "rectangle": "矩形",
+        "opacity": "透明度",
+        "clearProcessor": "清除处理器",
+        "globalControlAdapter": "全局 $t(controlnet.controlAdapter_one)"
+    },
+    "ui": {
+        "tabs": {
+            "generation": "生成",
+            "queue": "队列",
+            "canvas": "画布",
+            "upscaling": "放大中",
+            "workflows": "工作流",
+            "models": "模型"
+        }
+    },
+    "upscaling": {
+        "structure": "结构",
+        "upscaleModel": "放大模型",
+        "missingUpscaleModel": "缺少放大模型",
+        "missingTileControlNetModel": "没有安装有效的tile ControlNet 模型",
+        "missingUpscaleInitialImage": "缺少用于放大的原始图像",
+        "creativity": "创造力",
+        "postProcessingModel": "后处理模型",
+        "scale": "缩放",
+        "tileControlNetModelDesc": "根据所选的主模型架构，选择相应的Tile ControlNet模型",
+        "upscaleModelDesc": "图像放大（图像到图像转换）模型",
+        "postProcessingMissingModelWarning": "请访问 <LinkComponent>模型管理器</LinkComponent>来安装一个后处理(图像到图像转换)模型.",
+        "missingModelsWarning": "请访问<LinkComponent>模型管理器</LinkComponent> 安装所需的模型：",
+        "mainModelDesc": "主模型（SD1.5或SDXL架构）"
+    },
+    "upsell": {
+        "inviteTeammates": "邀请团队成员",
+        "professional": "专业",
+        "professionalUpsell": "可在 Invoke 的专业版中使用.点击此处或访问 invoke.com/pricing 了解更多详情.",
+        "shareAccess": "共享访问权限"
     }
 }

invokeai/frontend/web/src/app/components/App.tsx

@@ -16,6 +16,8 @@ import { useStarterModelsToast } from 'features/modelManagerV2/hooks/useStarterM
 import { configChanged } from 'features/system/store/configSlice';
 import { languageSelector } from 'features/system/store/systemSelectors';
 import InvokeTabs from 'features/ui/components/InvokeTabs';
+import type { InvokeTabName } from 'features/ui/store/tabMap';
+import { setActiveTab } from 'features/ui/store/uiSlice';
 import { AnimatePresence } from 'framer-motion';
 import i18n from 'i18n';
 import { size } from 'lodash-es';
@@ -34,9 +36,10 @@ interface Props {
     imageName: string;
     action: 'sendToImg2Img' | 'sendToCanvas' | 'useAllParameters';
   };
+  destination?: InvokeTabName | undefined;
 }
 
-const App = ({ config = DEFAULT_CONFIG, selectedImage }: Props) => {
+const App = ({ config = DEFAULT_CONFIG, selectedImage, destination }: Props) => {
   const language = useAppSelector(languageSelector);
   const logger = useLogger('system');
   const dispatch = useAppDispatch();
@@ -67,6 +70,12 @@ const App = ({ config = DEFAULT_CONFIG, selectedImage }: Props) => {
     }
   }, [dispatch, config, logger]);
 
+  useEffect(() => {
+    if (destination) {
+      dispatch(setActiveTab(destination));
+    }
+  }, [dispatch, destination]);
+
   useEffect(() => {
     dispatch(appStarted());
   }, [dispatch]);

invokeai/frontend/web/src/app/components/InvokeAIUI.tsx

@@ -19,6 +19,7 @@ import type { PartialAppConfig } from 'app/types/invokeai';
 import Loading from 'common/components/Loading/Loading';
 import AppDndContext from 'features/dnd/components/AppDndContext';
 import type { WorkflowCategory } from 'features/nodes/types/workflow';
+import type { InvokeTabName } from 'features/ui/store/tabMap';
 import type { PropsWithChildren, ReactNode } from 'react';
 import React, { lazy, memo, useEffect, useMemo } from 'react';
 import { Provider } from 'react-redux';
@@ -43,6 +44,7 @@ interface Props extends PropsWithChildren {
     imageName: string;
     action: 'sendToImg2Img' | 'sendToCanvas' | 'useAllParameters';
   };
+  destination?: InvokeTabName;
   customStarUi?: CustomStarUi;
   socketOptions?: Partial<ManagerOptions & SocketOptions>;
   isDebugging?: boolean;
@@ -62,6 +64,7 @@ const InvokeAIUI = ({
   projectUrl,
   queueId,
   selectedImage,
+  destination,
   customStarUi,
   socketOptions,
   isDebugging = false,
@@ -218,7 +221,7 @@ const InvokeAIUI = ({
         <React.Suspense fallback={<Loading />}>
           <ThemeLocaleProvider>
             <AppDndContext>
-              <App config={config} selectedImage={selectedImage} />
+              <App config={config} selectedImage={selectedImage} destination={destination} />
             </AppDndContext>
           </ThemeLocaleProvider>
         </React.Suspense>

invokeai/frontend/web/src/app/types/invokeai.ts

@@ -65,11 +65,15 @@ export type AppConfig = {
    */
   shouldUpdateImagesOnConnect: boolean;
   shouldFetchMetadataFromApi: boolean;
+  /**
+   * Sets a size limit for outputs on the upscaling tab. This is a maximum dimension, so the actual max number of pixels
+   * will be the square of this value.
+   */
+  maxUpscaleDimension?: number;
   allowPrivateBoards: boolean;
   disabledTabs: InvokeTabName[];
   disabledFeatures: AppFeature[];
   disabledSDFeatures: SDFeature[];
-  canRestoreDeletedImagesFromBin: boolean;
   nodesAllowlist: string[] | undefined;
   nodesDenylist: string[] | undefined;
   metadataFetchDebounce?: number;

invokeai/frontend/web/src/common/hooks/useIsReadyToEnqueue.ts

@@ -16,6 +16,7 @@ import { selectWorkflowSettingsSlice } from 'features/nodes/store/workflowSettin
 import { isInvocationNode } from 'features/nodes/types/invocation';
 import { selectGenerationSlice } from 'features/parameters/store/generationSlice';
 import { selectUpscalelice } from 'features/parameters/store/upscaleSlice';
+import { selectConfigSlice } from 'features/system/store/configSlice';
 import { selectSystemSlice } from 'features/system/store/systemSlice';
 import { activeTabNameSelector } from 'features/ui/store/uiSelectors';
 import i18n from 'i18next';
@@ -42,6 +43,7 @@ const createSelector = (templates: Templates) =>
       selectControlLayersSlice,
       activeTabNameSelector,
       selectUpscalelice,
+      selectConfigSlice,
     ],
     (
       controlAdapters,
@@ -52,7 +54,8 @@ const createSelector = (templates: Templates) =>
       dynamicPrompts,
       controlLayers,
       activeTabName,
-      upscale
+      upscale,
+      config
     ) => {
       const { model } = generation;
       const { size } = controlLayers.present;
@@ -209,6 +212,16 @@ const createSelector = (templates: Templates) =>
         } else if (activeTabName === 'upscaling') {
           if (!upscale.upscaleInitialImage) {
             reasons.push({ content: i18n.t('upscaling.missingUpscaleInitialImage') });
+          } else if (config.maxUpscaleDimension) {
+            const { width, height } = upscale.upscaleInitialImage;
+            const { scale } = upscale;
+
+            const maxPixels = config.maxUpscaleDimension ** 2;
+            const upscaledPixels = width * scale * height * scale;
+
+            if (upscaledPixels > maxPixels) {
+              reasons.push({ content: i18n.t('upscaling.exceedsMaxSize') });
+            }
           }
           if (!upscale.upscaleModel) {
             reasons.push({ content: i18n.t('upscaling.missingUpscaleModel') });

invokeai/frontend/web/src/features/deleteImageModal/components/DeleteImageModal.tsx

@@ -56,7 +56,6 @@ const DeleteImageModal = () => {
   const dispatch = useAppDispatch();
   const { t } = useTranslation();
   const shouldConfirmOnDelete = useAppSelector((s) => s.system.shouldConfirmOnDelete);
-  const canRestoreDeletedImagesFromBin = useAppSelector((s) => s.config.canRestoreDeletedImagesFromBin);
   const isModalOpen = useAppSelector((s) => s.deleteImageModal.isModalOpen);
   const { imagesToDelete, imagesUsage, imageUsageSummary } = useAppSelector(selectImageUsages);
 
@@ -90,7 +89,7 @@ const DeleteImageModal = () => {
       <Flex direction="column" gap={3}>
         <ImageUsageMessage imageUsage={imageUsageSummary} />
         <Divider />
-        <Text>{canRestoreDeletedImagesFromBin ? t('gallery.deleteImageBin') : t('gallery.deleteImagePermanent')}</Text>
+        <Text>{t('gallery.deleteImagePermanent')}</Text>
         <Text>{t('common.areYouSure')}</Text>
         <FormControl>
           <FormLabel>{t('common.dontAskMeAgain')}</FormLabel>

invokeai/frontend/web/src/features/gallery/components/Boards/DeleteBoardModal.tsx

@@ -35,7 +35,6 @@ type Props = {
 const DeleteBoardModal = (props: Props) => {
   const { boardToDelete, setBoardToDelete } = props;
   const { t } = useTranslation();
-  const canRestoreDeletedImagesFromBin = useAppSelector((s) => s.config.canRestoreDeletedImagesFromBin);
   const { currentData: boardImageNames, isFetching: isFetchingBoardNames } = useListAllImageNamesForBoardQuery(
     boardToDelete?.board_id ?? skipToken
   );
@@ -120,10 +119,12 @@ const DeleteBoardModal = (props: Props) => {
                   bottomMessage={t('boards.bottomMessage')}
                 />
               )}
-              <Text>{t('boards.deletedBoardsCannotbeRestored')}</Text>
               <Text>
-                {canRestoreDeletedImagesFromBin ? t('gallery.deleteImageBin') : t('gallery.deleteImagePermanent')}
+                {boardToDelete.is_private
+                  ? t('boards.deletedPrivateBoardsCannotbeRestored')
+                  : t('boards.deletedBoardsCannotbeRestored')}
               </Text>
+              <Text>{t('gallery.deleteImagePermanent')}</Text>
             </Flex>
           </AlertDialogBody>
           <AlertDialogFooter>

invokeai/frontend/web/src/features/nodes/types/common.ts

@@ -32,6 +32,7 @@ export const zSchedulerField = z.enum([
   'ddpm',
   'dpmpp_2s',
   'dpmpp_2m',
+  'dpmpp_3m',
   'dpmpp_2m_sde',
   'dpmpp_sde',
   'heun',
@@ -40,12 +41,17 @@ export const zSchedulerField = z.enum([
   'pndm',
   'unipc',
   'euler_k',
+  'deis_k',
   'dpmpp_2s_k',
   'dpmpp_2m_k',
+  'dpmpp_3m_k',
   'dpmpp_2m_sde_k',
   'dpmpp_sde_k',
   'heun_k',
+  'kdpm_2_k',
+  'kdpm_2_a_k',
   'lms_k',
+  'unipc_k',
   'euler_a',
   'kdpm_2_a',
   'lcm',

invokeai/frontend/web/src/features/nodes/util/graph/buildMultidiffusionUpscaleGraph.ts

@@ -125,19 +125,11 @@ export const buildMultidiffusionUpscaleGraph = async (state: RootState): Promise
     g.addEdge(modelNode, 'unet', tiledMultidiffusionNode, 'unet');
     addSDXLLoRas(state, g, tiledMultidiffusionNode, modelNode, null, posCondNode, negCondNode);
 
-    const modelConfig = await fetchModelConfigWithTypeGuard(model.key, isNonRefinerMainModelConfig);
-
     g.upsertMetadata({
-      cfg_scale,
       positive_prompt: positivePrompt,
       negative_prompt: negativePrompt,
       positive_style_prompt: positiveStylePrompt,
       negative_style_prompt: negativeStylePrompt,
-      model: Graph.getModelMetadataField(modelConfig),
-      seed,
-      steps,
-      scheduler,
-      vae: vae ?? undefined,
     });
   } else {
     posCondNode = g.addNode({
@@ -166,24 +158,33 @@ export const buildMultidiffusionUpscaleGraph = async (state: RootState): Promise
     g.addEdge(modelNode, 'unet', tiledMultidiffusionNode, 'unet');
     addLoRAs(state, g, tiledMultidiffusionNode, modelNode, null, clipSkipNode, posCondNode, negCondNode);
 
-    const modelConfig = await fetchModelConfigWithTypeGuard(model.key, isNonRefinerMainModelConfig);
-    const upscaleModelConfig = await fetchModelConfigWithTypeGuard(upscaleModel.key, isSpandrelImageToImageModelConfig);
-
     g.upsertMetadata({
-      cfg_scale,
       positive_prompt: positivePrompt,
       negative_prompt: negativePrompt,
-      model: Graph.getModelMetadataField(modelConfig),
-      seed,
-      steps,
-      scheduler,
-      vae: vae ?? undefined,
-      upscale_model: Graph.getModelMetadataField(upscaleModelConfig),
-      creativity,
-      structure,
     });
   }
 
+  const modelConfig = await fetchModelConfigWithTypeGuard(model.key, isNonRefinerMainModelConfig);
+  const upscaleModelConfig = await fetchModelConfigWithTypeGuard(upscaleModel.key, isSpandrelImageToImageModelConfig);
+
+  g.upsertMetadata({
+    cfg_scale,
+    model: Graph.getModelMetadataField(modelConfig),
+    seed,
+    steps,
+    scheduler,
+    vae: vae ?? undefined,
+    upscale_model: Graph.getModelMetadataField(upscaleModelConfig),
+    creativity,
+    structure,
+    upscale_initial_image: {
+      image_name: upscaleInitialImage.image_name,
+      width: upscaleInitialImage.width,
+      height: upscaleInitialImage.height,
+    },
+    upscale_scale: scale,
+  });
+
   g.setMetadataReceivingNode(l2iNode);
   g.addEdgeToMetadata(upscaleNode, 'width', 'width');
   g.addEdgeToMetadata(upscaleNode, 'height', 'height');

invokeai/frontend/web/src/features/parameters/hooks/useIsTooLargeToUpscale.ts

@@ -0,0 +1,29 @@
+import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
+import { useAppSelector } from 'app/store/storeHooks';
+import { selectUpscalelice } from 'features/parameters/store/upscaleSlice';
+import { selectConfigSlice } from 'features/system/store/configSlice';
+import { useMemo } from 'react';
+import type { ImageDTO } from 'services/api/types';
+
+const createIsTooLargeToUpscaleSelector = (imageDTO?: ImageDTO) =>
+  createMemoizedSelector(selectUpscalelice, selectConfigSlice, (upscale, config) => {
+    const { upscaleModel, scale } = upscale;
+    const { maxUpscaleDimension } = config;
+
+    if (!maxUpscaleDimension || !upscaleModel || !imageDTO) {
+      // When these are missing, another warning will be shown
+      return false;
+    }
+
+    const { width, height } = imageDTO;
+
+    const maxPixels = maxUpscaleDimension ** 2;
+    const upscaledPixels = width * scale * height * scale;
+
+    return upscaledPixels > maxPixels;
+  });
+
+export const useIsTooLargeToUpscale = (imageDTO?: ImageDTO) => {
+  const selectIsTooLargeToUpscale = useMemo(() => createIsTooLargeToUpscaleSelector(imageDTO), [imageDTO]);
+  return useAppSelector(selectIsTooLargeToUpscale);
+};

invokeai/frontend/web/src/features/parameters/types/constants.ts

@@ -52,28 +52,34 @@ export const CLIP_SKIP_MAP = {
  * Mapping of schedulers to human readable name
  */
 export const SCHEDULER_OPTIONS: ComboboxOption[] = [
-  { value: 'euler', label: 'Euler' },
-  { value: 'deis', label: 'DEIS' },
   { value: 'ddim', label: 'DDIM' },
   { value: 'ddpm', label: 'DDPM' },
-  { value: 'dpmpp_sde', label: 'DPM++ SDE' },
+  { value: 'deis', label: 'DEIS' },
+  { value: 'deis_k', label: 'DEIS Karras' },
   { value: 'dpmpp_2s', label: 'DPM++ 2S' },
-  { value: 'dpmpp_2m', label: 'DPM++ 2M' },
-  { value: 'dpmpp_2m_sde', label: 'DPM++ 2M SDE' },
-  { value: 'heun', label: 'Heun' },
-  { value: 'kdpm_2', label: 'KDPM 2' },
-  { value: 'lms', label: 'LMS' },
-  { value: 'pndm', label: 'PNDM' },
-  { value: 'unipc', label: 'UniPC' },
-  { value: 'euler_k', label: 'Euler Karras' },
-  { value: 'dpmpp_sde_k', label: 'DPM++ SDE Karras' },
   { value: 'dpmpp_2s_k', label: 'DPM++ 2S Karras' },
+  { value: 'dpmpp_2m', label: 'DPM++ 2M' },
   { value: 'dpmpp_2m_k', label: 'DPM++ 2M Karras' },
+  { value: 'dpmpp_2m_sde', label: 'DPM++ 2M SDE' },
   { value: 'dpmpp_2m_sde_k', label: 'DPM++ 2M SDE Karras' },
-  { value: 'heun_k', label: 'Heun Karras' },
-  { value: 'lms_k', label: 'LMS Karras' },
+  { value: 'dpmpp_3m', label: 'DPM++ 3M' },
+  { value: 'dpmpp_3m_k', label: 'DPM++ 3M Karras' },
+  { value: 'dpmpp_sde', label: 'DPM++ SDE' },
+  { value: 'dpmpp_sde_k', label: 'DPM++ SDE Karras' },
+  { value: 'euler', label: 'Euler' },
+  { value: 'euler_k', label: 'Euler Karras' },
   { value: 'euler_a', label: 'Euler Ancestral' },
+  { value: 'heun', label: 'Heun' },
+  { value: 'heun_k', label: 'Heun Karras' },
+  { value: 'kdpm_2', label: 'KDPM 2' },
+  { value: 'kdpm_2_k', label: 'KDPM 2 Karras' },
   { value: 'kdpm_2_a', label: 'KDPM 2 Ancestral' },
+  { value: 'kdpm_2_a_k', label: 'KDPM 2 Ancestral Karras' },
   { value: 'lcm', label: 'LCM' },
+  { value: 'lms', label: 'LMS' },
+  { value: 'lms_k', label: 'LMS Karras' },
+  { value: 'pndm', label: 'PNDM' },
   { value: 'tcd', label: 'TCD' },
-].sort((a, b) => a.label.localeCompare(b.label));
+  { value: 'unipc', label: 'UniPC' },
+  { value: 'unipc_k', label: 'UniPC Karras' },
+];

invokeai/frontend/web/src/features/settingsAccordions/components/UpscaleSettingsAccordion/UpscaleInitialImage.tsx

@@ -1,4 +1,4 @@
-import { Flex } from '@invoke-ai/ui-library';
+import { Flex, Text } from '@invoke-ai/ui-library';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import IAIDndImage from 'common/components/IAIDndImage';
 import IAIDndImageIcon from 'common/components/IAIDndImageIcon';
@@ -41,13 +41,30 @@ export const UpscaleInitialImage = () => {
           postUploadAction={postUploadAction}
         />
         {imageDTO && (
-          <Flex position="absolute" flexDir="column" top={1} insetInlineEnd={1} gap={1}>
-            <IAIDndImageIcon
-              onClick={onReset}
-              icon={<PiArrowCounterClockwiseBold size={16} />}
-              tooltip={t('controlnet.resetControlImage')}
-            />
-          </Flex>
+          <>
+            <Flex position="absolute" flexDir="column" top={1} insetInlineEnd={1} gap={1}>
+              <IAIDndImageIcon
+                onClick={onReset}
+                icon={<PiArrowCounterClockwiseBold size={16} />}
+                tooltip={t('controlnet.resetControlImage')}
+              />
+            </Flex>
+            <Text
+              position="absolute"
+              background="base.900"
+              color="base.50"
+              fontSize="sm"
+              fontWeight="semibold"
+              bottom={0}
+              left={0}
+              opacity={0.7}
+              px={2}
+              lineHeight={1.25}
+              borderTopEndRadius="base"
+              borderBottomStartRadius="base"
+              pointerEvents="none"
+            >{`${imageDTO.width}x${imageDTO.height}`}</Text>
+          </>
         )}
       </Flex>
     </Flex>

invokeai/frontend/web/src/features/settingsAccordions/components/UpscaleSettingsAccordion/UpscaleWarning.tsx

@@ -1,6 +1,7 @@
 import { Button, Flex, ListItem, Text, UnorderedList } from '@invoke-ai/ui-library';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { $installModelsTab } from 'features/modelManagerV2/subpanels/InstallModels';
+import { useIsTooLargeToUpscale } from 'features/parameters/hooks/useIsTooLargeToUpscale';
 import { tileControlnetModelChanged } from 'features/parameters/store/upscaleSlice';
 import { setActiveTab } from 'features/ui/store/uiSlice';
 import { useCallback, useEffect, useMemo } from 'react';
@@ -12,10 +13,13 @@ export const UpscaleWarning = () => {
   const model = useAppSelector((s) => s.generation.model);
   const upscaleModel = useAppSelector((s) => s.upscale.upscaleModel);
   const tileControlnetModel = useAppSelector((s) => s.upscale.tileControlnetModel);
+  const upscaleInitialImage = useAppSelector((s) => s.upscale.upscaleInitialImage);
   const dispatch = useAppDispatch();
   const [modelConfigs, { isLoading }] = useControlNetModels();
   const disabledTabs = useAppSelector((s) => s.config.disabledTabs);
   const shouldShowButton = useMemo(() => !disabledTabs.includes('models'), [disabledTabs]);
+  const maxUpscaleDimension = useAppSelector((s) => s.config.maxUpscaleDimension);
+  const isTooLargeToUpscale = useIsTooLargeToUpscale(upscaleInitialImage || undefined);
 
   useEffect(() => {
     const validModel = modelConfigs.find((cnetModel) => {
@@ -24,7 +28,7 @@ export const UpscaleWarning = () => {
     dispatch(tileControlnetModelChanged(validModel || null));
   }, [model?.base, modelConfigs, dispatch]);
 
-  const warnings = useMemo(() => {
+  const modelWarnings = useMemo(() => {
     const _warnings: string[] = [];
     if (!model) {
       _warnings.push(t('upscaling.mainModelDesc'));
@@ -35,33 +39,44 @@ export const UpscaleWarning = () => {
     if (!upscaleModel) {
       _warnings.push(t('upscaling.upscaleModelDesc'));
     }
-
     return _warnings;
   }, [model, tileControlnetModel, upscaleModel, t]);
 
+  const otherWarnings = useMemo(() => {
+    const _warnings: string[] = [];
+    if (isTooLargeToUpscale && maxUpscaleDimension) {
+      _warnings.push(
+        t('upscaling.exceedsMaxSizeDetails', { maxUpscaleDimension: maxUpscaleDimension.toLocaleString() })
+      );
+    }
+    return _warnings;
+  }, [isTooLargeToUpscale, t, maxUpscaleDimension]);
+
   const handleGoToModelManager = useCallback(() => {
     dispatch(setActiveTab('models'));
     $installModelsTab.set(3);
   }, [dispatch]);
 
-  if (!warnings.length || isLoading || !shouldShowButton) {
+  if ((!modelWarnings.length && !otherWarnings.length) || isLoading || !shouldShowButton) {
     return null;
   }
 
   return (
     <Flex bg="error.500" borderRadius="base" padding={4} direction="column" fontSize="sm" gap={2}>
-      <Text>
-        <Trans
-          i18nKey="upscaling.missingModelsWarning"
-          components={{
-            LinkComponent: (
-              <Button size="sm" flexGrow={0} variant="link" color="base.50" onClick={handleGoToModelManager} />
-            ),
-          }}
-        />
-      </Text>
+      {!!modelWarnings.length && (
+        <Text>
+          <Trans
+            i18nKey="upscaling.missingModelsWarning"
+            components={{
+              LinkComponent: (
+                <Button size="sm" flexGrow={0} variant="link" color="base.50" onClick={handleGoToModelManager} />
+              ),
+            }}
+          />
+        </Text>
+      )}
       <UnorderedList>
-        {warnings.map((warning) => (
+        {[...modelWarnings, ...otherWarnings].map((warning) => (
           <ListItem key={warning}>{warning}</ListItem>
         ))}
       </UnorderedList>

invokeai/frontend/web/src/features/system/store/configSlice.ts

@@ -24,7 +24,6 @@ const initialConfigState: AppConfig = {
   disabledSDFeatures: ['variation', 'symmetry', 'hires', 'perlinNoise', 'noiseThreshold'],
   nodesAllowlist: undefined,
   nodesDenylist: undefined,
-  canRestoreDeletedImagesFromBin: true,
   sd: {
     disabledControlNetModels: [],
     disabledControlNetProcessors: [],

invokeai/frontend/web/src/services/api/schema.ts

@@ -3553,7 +3553,7 @@ export type components = {
        * @default euler
        * @enum {string}
        */
-      scheduler?: "ddim" | "ddpm" | "deis" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_a" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "lcm" | "tcd";
+      scheduler?: "ddim" | "ddpm" | "deis" | "deis_k" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_k" | "kdpm_2_a" | "kdpm_2_a_k" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_3m" | "dpmpp_3m_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "unipc_k" | "lcm" | "tcd";
       /**
        * UNet
        * @description UNet (scheduler, LoRAs)
@@ -8553,7 +8553,7 @@ export type components = {
        * Scheduler
        * @description Default scheduler for this model
        */
-      scheduler?: ("ddim" | "ddpm" | "deis" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_a" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "lcm" | "tcd") | null;
+      scheduler?: ("ddim" | "ddpm" | "deis" | "deis_k" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_k" | "kdpm_2_a" | "kdpm_2_a_k" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_3m" | "dpmpp_3m_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "unipc_k" | "lcm" | "tcd") | null;
       /**
        * Steps
        * @description Default number of steps for this model
@@ -11467,7 +11467,7 @@ export type components = {
        * @default euler
        * @enum {string}
        */
-      scheduler?: "ddim" | "ddpm" | "deis" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_a" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "lcm" | "tcd";
+      scheduler?: "ddim" | "ddpm" | "deis" | "deis_k" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_k" | "kdpm_2_a" | "kdpm_2_a_k" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_3m" | "dpmpp_3m_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "unipc_k" | "lcm" | "tcd";
       /**
        * type
        * @default scheduler
@@ -11483,7 +11483,7 @@ export type components = {
        * @description Scheduler to use during inference
        * @enum {string}
        */
-      scheduler: "ddim" | "ddpm" | "deis" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_a" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "lcm" | "tcd";
+      scheduler: "ddim" | "ddpm" | "deis" | "deis_k" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_k" | "kdpm_2_a" | "kdpm_2_a_k" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_3m" | "dpmpp_3m_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "unipc_k" | "lcm" | "tcd";
       /**
        * type
        * @default scheduler_output
@@ -13261,7 +13261,7 @@ export type components = {
        * @default euler
        * @enum {string}
        */
-      scheduler?: "ddim" | "ddpm" | "deis" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_a" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "lcm" | "tcd";
+      scheduler?: "ddim" | "ddpm" | "deis" | "deis_k" | "lms" | "lms_k" | "pndm" | "heun" | "heun_k" | "euler" | "euler_k" | "euler_a" | "kdpm_2" | "kdpm_2_k" | "kdpm_2_a" | "kdpm_2_a_k" | "dpmpp_2s" | "dpmpp_2s_k" | "dpmpp_2m" | "dpmpp_2m_k" | "dpmpp_2m_sde" | "dpmpp_2m_sde_k" | "dpmpp_3m" | "dpmpp_3m_k" | "dpmpp_sde" | "dpmpp_sde_k" | "unipc" | "unipc_k" | "lcm" | "tcd";
       /**
        * UNet
        * @description UNet (scheduler, LoRAs)

pyproject.toml

@@ -33,31 +33,35 @@ classifiers = [
 ]
 dependencies = [
   # Core generation dependencies, pinned for reproducible builds.
-  "accelerate==0.30.1",
+  "accelerate==0.33.0",
   "clip_anytorch==2.6.0",       # replacing "clip @ https://github.com/openai/CLIP/archive/eaa22acb90a5876642d0507623e859909230a52d.zip",
   "compel==2.0.2",
   "controlnet-aux==0.0.7",
-  "diffusers[torch]==0.27.2",
+  # TODO(ryand): Bump this once the next diffusers release is ready.
+  "diffusers[torch] @ git+https://github.com/huggingface/diffusers.git@4c6152c2fb0ade468aadb417102605a07a8635d3",
   "invisible-watermark==0.2.0", # needed to install SDXL base and refiner using their repo_ids
   "mediapipe==0.10.7",          # needed for "mediapipeface" controlnet model
   "numpy==1.26.4",              # >1.24.0 is needed to use the 'strict' argument to np.testing.assert_array_equal()
   "onnx==1.15.0",
   "onnxruntime==1.16.3",
   "opencv-python==4.9.0.80",
+  "optimum-quanto==0.2.4",
   "pytorch-lightning==2.1.3",
   "safetensors==0.4.3",
+  # sentencepiece is required to load T5TokenizerFast (used by FLUX).
+  "sentencepiece==0.2.0",
   "spandrel==0.3.4",
   "timm==0.6.13",               # needed to override timm latest in controlnet_aux, see  https://github.com/isl-org/ZoeDepth/issues/26
-  "torch==2.2.2",
+  "torch==2.4.0",
   "torchmetrics==0.11.4",
   "torchsde==0.2.6",
-  "torchvision==0.17.2",
+  "torchvision==0.19.0",
   "transformers==4.41.1",
 
   # Core application dependencies, pinned for reproducible builds.
   "fastapi-events==0.11.1",
   "fastapi==0.111.0",
-  "huggingface-hub==0.23.1",
+  "huggingface-hub==0.24.5",
   "pydantic-settings==2.2.1",
   "pydantic==2.7.2",
   "python-socketio==5.11.1",
@@ -74,7 +78,8 @@ dependencies = [
   "easing-functions",
   "einops",
   "facexlib",
-  "matplotlib",       # needed for plotting of Penner easing functions
+  # Exclude 3.9.1 which has a problem on windows, see https://github.com/matplotlib/matplotlib/issues/28551
+  "matplotlib!=3.9.1",
   "npyscreen",
   "omegaconf",
   "picklescan",
@@ -89,7 +94,6 @@ dependencies = [
   "rich~=13.3",
   "scikit-image~=0.21.0",
   "semver~=3.0.1",
-  "send2trash",
   "test-tube~=0.7.5",
   "windows-curses; sys_platform=='win32'",
 ]

tests/backend/model_manager/util/test_hf_model_select.py

@@ -326,3 +326,80 @@ def test_select_multiple_weights(
 ) -> None:
     filtered_files = filter_files(sd15_test_files, variant)
     assert set(filtered_files) == {Path(f) for f in expected_files}
+
+
+@pytest.fixture
+def flux_schnell_test_files() -> list[Path]:
+    return [
+        Path(f)
+        for f in [
+            "FLUX.1-schnell/.gitattributes",
+            "FLUX.1-schnell/README.md",
+            "FLUX.1-schnell/ae.safetensors",
+            "FLUX.1-schnell/flux1-schnell.safetensors",
+            "FLUX.1-schnell/model_index.json",
+            "FLUX.1-schnell/scheduler/scheduler_config.json",
+            "FLUX.1-schnell/schnell_grid.jpeg",
+            "FLUX.1-schnell/text_encoder/config.json",
+            "FLUX.1-schnell/text_encoder/model.safetensors",
+            "FLUX.1-schnell/text_encoder_2/config.json",
+            "FLUX.1-schnell/text_encoder_2/model-00001-of-00002.safetensors",
+            "FLUX.1-schnell/text_encoder_2/model-00002-of-00002.safetensors",
+            "FLUX.1-schnell/text_encoder_2/model.safetensors.index.json",
+            "FLUX.1-schnell/tokenizer/merges.txt",
+            "FLUX.1-schnell/tokenizer/special_tokens_map.json",
+            "FLUX.1-schnell/tokenizer/tokenizer_config.json",
+            "FLUX.1-schnell/tokenizer/vocab.json",
+            "FLUX.1-schnell/tokenizer_2/special_tokens_map.json",
+            "FLUX.1-schnell/tokenizer_2/spiece.model",
+            "FLUX.1-schnell/tokenizer_2/tokenizer.json",
+            "FLUX.1-schnell/tokenizer_2/tokenizer_config.json",
+            "FLUX.1-schnell/transformer/config.json",
+            "FLUX.1-schnell/transformer/diffusion_pytorch_model-00001-of-00003.safetensors",
+            "FLUX.1-schnell/transformer/diffusion_pytorch_model-00002-of-00003.safetensors",
+            "FLUX.1-schnell/transformer/diffusion_pytorch_model-00003-of-00003.safetensors",
+            "FLUX.1-schnell/transformer/diffusion_pytorch_model.safetensors.index.json",
+            "FLUX.1-schnell/vae/config.json",
+            "FLUX.1-schnell/vae/diffusion_pytorch_model.safetensors",
+        ]
+    ]
+
+
+@pytest.mark.parametrize(
+    ["variant", "expected_files"],
+    [
+        (
+            ModelRepoVariant.Default,
+            [
+                "FLUX.1-schnell/model_index.json",
+                "FLUX.1-schnell/scheduler/scheduler_config.json",
+                "FLUX.1-schnell/text_encoder/config.json",
+                "FLUX.1-schnell/text_encoder/model.safetensors",
+                "FLUX.1-schnell/text_encoder_2/config.json",
+                "FLUX.1-schnell/text_encoder_2/model-00001-of-00002.safetensors",
+                "FLUX.1-schnell/text_encoder_2/model-00002-of-00002.safetensors",
+                "FLUX.1-schnell/text_encoder_2/model.safetensors.index.json",
+                "FLUX.1-schnell/tokenizer/merges.txt",
+                "FLUX.1-schnell/tokenizer/special_tokens_map.json",
+                "FLUX.1-schnell/tokenizer/tokenizer_config.json",
+                "FLUX.1-schnell/tokenizer/vocab.json",
+                "FLUX.1-schnell/tokenizer_2/special_tokens_map.json",
+                "FLUX.1-schnell/tokenizer_2/spiece.model",
+                "FLUX.1-schnell/tokenizer_2/tokenizer.json",
+                "FLUX.1-schnell/tokenizer_2/tokenizer_config.json",
+                "FLUX.1-schnell/transformer/config.json",
+                "FLUX.1-schnell/transformer/diffusion_pytorch_model-00001-of-00003.safetensors",
+                "FLUX.1-schnell/transformer/diffusion_pytorch_model-00002-of-00003.safetensors",
+                "FLUX.1-schnell/transformer/diffusion_pytorch_model-00003-of-00003.safetensors",
+                "FLUX.1-schnell/transformer/diffusion_pytorch_model.safetensors.index.json",
+                "FLUX.1-schnell/vae/config.json",
+                "FLUX.1-schnell/vae/diffusion_pytorch_model.safetensors",
+            ],
+        ),
+    ],
+)
+def test_select_flux_schnell_files(
+    flux_schnell_test_files: list[Path], variant: ModelRepoVariant, expected_files: list[str]
+) -> None:
+    filtered_files = filter_files(flux_schnell_test_files, variant)
+    assert set(filtered_files) == {Path(f) for f in expected_files}