From 1bb5b4ab322da5a0021a8804687db6c68388cbe1 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Fri, 27 Jan 2023 11:52:05 -0800 Subject: [PATCH 1/5] fix dimension errors when inpainting model is used with hires-fix --- ldm/invoke/generator/txt2img2img.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ldm/invoke/generator/txt2img2img.py b/ldm/invoke/generator/txt2img2img.py index 47692a6bbb..4923e7daf5 100644 --- a/ldm/invoke/generator/txt2img2img.py +++ b/ldm/invoke/generator/txt2img2img.py @@ -3,10 +3,10 @@ ldm.invoke.generator.txt2img inherits from ldm.invoke.generator ''' import math -from diffusers.utils.logging import get_verbosity, set_verbosity, set_verbosity_error from typing import Callable, Optional import torch +from diffusers.utils.logging import get_verbosity, set_verbosity, set_verbosity_error from ldm.invoke.generator.base import Generator from ldm.invoke.generator.diffusers_pipeline import trim_to_multiple_of, StableDiffusionGeneratorPipeline, \ @@ -116,16 +116,20 @@ class Txt2Img2Img(Generator): scaled_height = height device = self.model.device + + channels = self.latent_channels + if channels == 9: + channels = 4 # we don't really want noise for all the mask channels if self.use_mps_noise or device.type == 'mps': return torch.randn([1, - self.latent_channels, + channels, scaled_height // self.downsampling_factor, scaled_width // self.downsampling_factor], dtype=self.torch_dtype(), device='cpu').to(device) else: return torch.randn([1, - self.latent_channels, + channels, scaled_height // self.downsampling_factor, scaled_width // self.downsampling_factor], dtype=self.torch_dtype(), From 09b6104bfd80212fe0ab528b9354b49094d1248b Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:04:12 -0800 Subject: [PATCH 2/5] refactor(txt2img2img): factor out tensor shape --- ldm/invoke/generator/txt2img2img.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/ldm/invoke/generator/txt2img2img.py b/ldm/invoke/generator/txt2img2img.py index 4923e7daf5..058628ba1c 100644 --- a/ldm/invoke/generator/txt2img2img.py +++ b/ldm/invoke/generator/txt2img2img.py @@ -115,22 +115,13 @@ class Txt2Img2Img(Generator): scaled_width = width scaled_height = height - device = self.model.device - + device = self.model.device channels = self.latent_channels if channels == 9: channels = 4 # we don't really want noise for all the mask channels + shape = (1, channels, + scaled_height // self.downsampling_factor, scaled_width // self.downsampling_factor) if self.use_mps_noise or device.type == 'mps': - return torch.randn([1, - channels, - scaled_height // self.downsampling_factor, - scaled_width // self.downsampling_factor], - dtype=self.torch_dtype(), - device='cpu').to(device) + return torch.randn(shape, dtype=self.torch_dtype(), device='cpu').to(device) else: - return torch.randn([1, - channels, - scaled_height // self.downsampling_factor, - scaled_width // self.downsampling_factor], - dtype=self.torch_dtype(), - device=device) + return torch.randn(shape, dtype=self.torch_dtype(), device=device) From 71b6ddf5fb305206985ec8dda6cb4487508d7b05 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Sat, 28 Jan 2023 11:10:32 -0800 Subject: [PATCH 3/5] fix(inpainting model): blank areas to be repainted in the masked image Otherwise the model seems too reluctant to change these areas, even though the mask channel should allow it to. --- ldm/invoke/generator/diffusers_pipeline.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py index 54e9d555af..763704928c 100644 --- a/ldm/invoke/generator/diffusers_pipeline.py +++ b/ldm/invoke/generator/diffusers_pipeline.py @@ -531,6 +531,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): init_image = image_resized_to_grid_as_tensor(init_image.convert('RGB')) init_image = init_image.to(device=device, dtype=latents_dtype) + mask = mask.to(device=device, dtype=latents_dtype) if init_image.dim() == 3: init_image = init_image.unsqueeze(0) @@ -549,17 +550,22 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): if mask.dim() == 3: mask = mask.unsqueeze(0) - mask = tv_resize(mask, init_image_latents.shape[-2:], T.InterpolationMode.BILINEAR) \ + latent_mask = tv_resize(mask, init_image_latents.shape[-2:], T.InterpolationMode.BILINEAR) \ .to(device=device, dtype=latents_dtype) guidance: List[Callable] = [] if is_inpainting_model(self.unet): # TODO: we should probably pass this in so we don't have to try/finally around setting it. + + # You'd think the inpainting model wouldn't be paying attention to the area it is going to repaint + # (that's why there's a mask!) but it seems to really want that blanked out. + masked_latents = self.non_noised_latents_from_image( + init_image * (1 - mask), device=device, dtype=latents_dtype) self.invokeai_diffuser.model_forward_callback = \ - AddsMaskLatents(self._unet_forward, mask, init_image_latents) + AddsMaskLatents(self._unet_forward, latent_mask, masked_latents) else: - guidance.append(AddsMaskGuidance(mask, init_image_latents, self.scheduler, noise)) + guidance.append(AddsMaskGuidance(latent_mask, init_image_latents, self.scheduler, noise)) try: result_latents, result_attention_maps = self.latents_from_embeddings( From b6d37a70ca12511b65d6f16970d7a945f88c1357 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Sat, 28 Jan 2023 13:34:22 -0800 Subject: [PATCH 4/5] fix(inpainting model): threshold mask to avoid gray blurry seam --- ldm/invoke/generator/diffusers_pipeline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py index 763704928c..d290ac8e8e 100644 --- a/ldm/invoke/generator/diffusers_pipeline.py +++ b/ldm/invoke/generator/diffusers_pipeline.py @@ -556,12 +556,12 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): guidance: List[Callable] = [] if is_inpainting_model(self.unet): - # TODO: we should probably pass this in so we don't have to try/finally around setting it. - # You'd think the inpainting model wouldn't be paying attention to the area it is going to repaint # (that's why there's a mask!) but it seems to really want that blanked out. - masked_latents = self.non_noised_latents_from_image( - init_image * (1 - mask), device=device, dtype=latents_dtype) + masked_init_image = init_image * torch.where(mask < 0.5, 1, 0) + masked_latents = self.non_noised_latents_from_image(masked_init_image, device=device, dtype=latents_dtype) + + # TODO: we should probably pass this in so we don't have to try/finally around setting it. self.invokeai_diffuser.model_forward_callback = \ AddsMaskLatents(self._unet_forward, latent_mask, masked_latents) else: From fc8e3dbcd33adf92b6930284b32e9e56bf0d1e6c Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 31 Jan 2023 09:59:58 -0500 Subject: [PATCH 5/5] fix crash when editing name of model - fixes a spurious "unknown model name" error when trying to edit the short name of an existing model. - relaxes naming requirements to include the ':' and '/' characters in model names --- ldm/invoke/CLI.py | 4 ++-- ldm/invoke/model_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ldm/invoke/CLI.py b/ldm/invoke/CLI.py index f5f0d5b86b..4a7b56eb5c 100644 --- a/ldm/invoke/CLI.py +++ b/ldm/invoke/CLI.py @@ -786,8 +786,8 @@ def _get_model_name(existing_names,completer,default_name:str='')->str: model_name = input(f'Short name for this model [{default_name}]: ').strip() if len(model_name)==0: model_name = default_name - if not re.match('^[\w._+-]+$',model_name): - print('** model name must contain only words, digits and the characters "._+-" **') + if not re.match('^[\w._+:/-]+$',model_name): + print('** model name must contain only words, digits and the characters "._+:/-" **') elif model_name != default_name and model_name in existing_names: print(f'** the name {model_name} is already in use. Pick another.') else: diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index 811e768764..dbc690ec54 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -125,7 +125,7 @@ class ModelManager(object): Set the default model. The change will not take effect until you call model_manager.commit() ''' - assert model_name in self.models,f"unknown model '{model_name}'" + assert model_name in self.model_names(), f"unknown model '{model_name}'" config = self.config for model in config: