From b6d1a17a1eee8d7c8e7e1dde69fae1a826895242 Mon Sep 17 00:00:00 2001 From: Edward Johan <53343081+younyokel@users.noreply.github.com> Date: Mon, 9 Jan 2023 23:53:55 +0600 Subject: [PATCH 01/13] tip fix --- docs/installation/020_INSTALL_MANUAL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/020_INSTALL_MANUAL.md b/docs/installation/020_INSTALL_MANUAL.md index 7f1e152945..fb7c1b0304 100644 --- a/docs/installation/020_INSTALL_MANUAL.md +++ b/docs/installation/020_INSTALL_MANUAL.md @@ -216,7 +216,7 @@ manager, please follow these steps: !!! tip - Do not move the source code repository after installation. The virtual environment directory has absolute paths in it that get confused if the directory is moved. + Do not move the source code repository after installation. The virtual environment directory has absolute paths in it that get confused if the directory is moved. --- From f86c8b043c0cdd61b70a6a95f83cc4aa2a56c8d6 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 15 Jan 2023 15:08:59 -0500 Subject: [PATCH 02/13] further improvements to initial load - Migration process will not crash if duplicate model files are found, one in legacy location and the other in new location. The model in the legacy location will be deleted in this case. - Added a hint to stable-diffusion-2.1 telling people it will work best with 768 pixel images. - Added the anything-4.0 model. --- configs/INITIAL_MODELS.yaml | 45 ++++++++++++++++++++++--------------- ldm/invoke/model_manager.py | 19 +++++++++++----- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/configs/INITIAL_MODELS.yaml b/configs/INITIAL_MODELS.yaml index 5df64d742d..fa55b16ef4 100644 --- a/configs/INITIAL_MODELS.yaml +++ b/configs/INITIAL_MODELS.yaml @@ -1,8 +1,13 @@ -stable-diffusion-2.1: - description: Stable Diffusion version 2.1 diffusers model (5.21 GB) +stable-diffusion-2.1-768: + description: Stable Diffusion version 2.1 diffusers model, trained on 768x768 images (5.21 GB) repo_id: stabilityai/stable-diffusion-2-1 format: diffusers recommended: True +stable-diffusion-2.1-base: + description: Stable Diffusion version 2.1 diffusers base model, trained on 512x512 images (5.21 GB) + repo_id: stabilityai/stable-diffusion-2-1-base + format: diffusers + recommended: False stable-diffusion-1.5: description: Stable Diffusion version 1.5 weight file (4.27 GB) repo_id: runwayml/stable-diffusion-v1-5 @@ -10,10 +15,17 @@ stable-diffusion-1.5: recommended: True vae: repo_id: stabilityai/sd-vae-ft-mse +stable-diffusion-1.4: + description: The original Stable Diffusion version 1.4 weight file (4.27 GB) + repo_id: CompVis/stable-diffusion-v1-4 + recommended: False + format: diffusers + vae: + repo_id: stabilityai/sd-vae-ft-mse width: 512 height: 512 inpainting-1.5: - description: RunwayML SD 1.5 model optimized for inpainting (4.27 GB) + description: RunwayML SD 1.5 model optimized for inpainting (ckpt version) (4.27 GB) repo_id: runwayml/stable-diffusion-inpainting config: v1-inpainting-inference.yaml file: sd-v1-5-inpainting.ckpt @@ -24,17 +36,13 @@ inpainting-1.5: recommended: True width: 512 height: 512 -stable-diffusion-1.4: - description: The original Stable Diffusion version 1.4 weight file (4.27 GB) - repo_id: CompVis/stable-diffusion-v1-4 - recommended: False - format: diffusers - vae: - repo_id: stabilityai/sd-vae-ft-mse - width: 512 - height: 512 +waifu-diffusion-1.4: + description: Latest waifu diffusion 1.4 (diffusers version) + format: diffusers + repo_id: hakurei/waifu-diffusion + recommended: True waifu-diffusion-1.3: - description: Stable Diffusion 1.4 fine tuned on anime-styled images (4.27 GB) + description: Stable Diffusion 1.4 fine tuned on anime-styled images (ckpt version) (4.27 GB) repo_id: hakurei/waifu-diffusion-v1-3 config: v1-inference.yaml file: model-epoch09-float32.ckpt @@ -52,10 +60,8 @@ trinart-2.0: recommended: False vae: repo_id: stabilityai/sd-vae-ft-mse - width: 512 - height: 512 trinart_characters-2.0: - description: An SD model finetuned with 19.2M anime/manga style images (4.27 GB) + description: An SD model finetuned with 19.2M anime/manga style images (ckpt version) (4.27 GB) repo_id: naclbit/trinart_derrida_characters_v2_stable_diffusion config: v1-inference.yaml file: derrida_final.ckpt @@ -66,6 +72,11 @@ trinart_characters-2.0: recommended: False width: 512 height: 512 +anything-4.0: + description: High-quality, highly detailed anime style images with just a few prompts + format: diffusers + repo_id: andite/anything-v4.0 + recommended: False papercut-1.0: description: SD 1.5 fine-tuned for papercut art (use "PaperCut" in your prompts) (2.13 GB) repo_id: Fictiverse/Stable_Diffusion_PaperCut_Model @@ -73,8 +84,6 @@ papercut-1.0: vae: repo_id: stabilityai/sd-vae-ft-mse recommended: False - width: 512 - height: 512 voxel_art-1.0: description: Stable Diffusion trained on voxel art (use "VoxelArt" in your prompts) (4.27 GB) repo_id: Fictiverse/Stable_Diffusion_VoxelArt_Model diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index 880d75476f..a00c4836e8 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -753,16 +753,20 @@ class ModelManager(object): print('** Legacy version <= 2.2.5 model directory layout detected. Reorganizing.') print('** This is a quick one-time operation.') - from shutil import move + from shutil import move, rmtree # transformer files get moved into the hub directory hub = models_dir / 'hub' os.makedirs(hub, exist_ok=True) for model in legacy_locations: - source = models_dir /model + source = models_dir / model + dest = hub / model.stem + print(f'** {source} => {dest}') if source.exists(): - print(f'DEBUG: Moving {models_dir / model} into hub') - move(models_dir / model, hub) + if dest.exists(): + rmtree(source) + else: + move(source, dest) # anything else gets moved into the diffusers directory diffusers = models_dir / 'diffusers' @@ -773,7 +777,12 @@ class ModelManager(object): if full_path.is_relative_to(hub) or full_path.is_relative_to(diffusers): continue if Path(dir).match('models--*--*'): - move(full_path,diffusers) + dest = diffusers / dir + print(f'** {full_path} => {dest}') + if dest.exists(): + rmtree(full_path) + else: + move(full_path,dest) # now clean up by removing any empty directories empty = [root for root, dirs, files, in os.walk(models_dir) if not len(dirs) and not len(files)] From 3a1724652e2d2d6ebf070f39664393fb2ffa3a15 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 14:19:27 -0500 Subject: [PATCH 03/13] upgrade requirements to CUDA 11.7, torch 1.13 (#2331) * upgrade requirements to CUDA 11.7, torch 1.13 * fix ROCm version number Co-authored-by: Lincoln Stein --- environments-and-requirements/requirements-lin-amd.txt | 2 +- environments-and-requirements/requirements-lin-cuda.txt | 2 +- environments-and-requirements/requirements-win-colab-cuda.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environments-and-requirements/requirements-lin-amd.txt b/environments-and-requirements/requirements-lin-amd.txt index 0aa6092b30..0e707be483 100644 --- a/environments-and-requirements/requirements-lin-amd.txt +++ b/environments-and-requirements/requirements-lin-amd.txt @@ -1,6 +1,6 @@ -r environments-and-requirements/requirements-base.txt # Get hardware-appropriate torch/torchvision ---extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org +--extra-index-url https://download.pytorch.org/whl/rocm5.2 --trusted-host https://download.pytorch.org torch>=1.13.1 torchvision>=0.14.1 -e . diff --git a/environments-and-requirements/requirements-lin-cuda.txt b/environments-and-requirements/requirements-lin-cuda.txt index d7ea4b4542..1f1ebcf947 100644 --- a/environments-and-requirements/requirements-lin-cuda.txt +++ b/environments-and-requirements/requirements-lin-cuda.txt @@ -1,4 +1,4 @@ ---extra-index-url https://download.pytorch.org/whl/cu116 --trusted-host https://download.pytorch.org +--trusted-host https://download.pytorch.org -r environments-and-requirements/requirements-base.txt torch>=1.13.1 torchvision>=0.14.1 diff --git a/environments-and-requirements/requirements-win-colab-cuda.txt b/environments-and-requirements/requirements-win-colab-cuda.txt index 01b25144b6..7fc55c538a 100644 --- a/environments-and-requirements/requirements-win-colab-cuda.txt +++ b/environments-and-requirements/requirements-win-colab-cuda.txt @@ -1,6 +1,6 @@ -r environments-and-requirements/requirements-base.txt # Get hardware-appropriate torch/torchvision ---extra-index-url https://download.pytorch.org/whl/cu116 --trusted-host https://download.pytorch.org +--extra-index-url https://download.pytorch.org/whl/cu117 --trusted-host https://download.pytorch.org torch==1.13.1 torchvision==0.14.1 -e . From d6ac0eeffd372171329fc9ce515f675298bc706a Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 14:21:34 -0500 Subject: [PATCH 04/13] make SD-1.5 the default again --- configs/INITIAL_MODELS.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/INITIAL_MODELS.yaml b/configs/INITIAL_MODELS.yaml index 6411a64779..cc3746224e 100644 --- a/configs/INITIAL_MODELS.yaml +++ b/configs/INITIAL_MODELS.yaml @@ -13,6 +13,7 @@ stable-diffusion-1_5: repo_id: runwayml/stable-diffusion-v1-5 format: diffusers recommended: True + default: True vae: repo_id: stabilityai/sd-vae-ft-mse stable-diffusion-1_4: From 563196bd03398fbc1d18cefe2077b72eba6b651e Mon Sep 17 00:00:00 2001 From: Damian Stewart Date: Mon, 16 Jan 2023 20:56:54 +0100 Subject: [PATCH 05/13] pass step count and step index to diffusion step func (#2342) --- ldm/invoke/generator/diffusers_pipeline.py | 8 ++++++-- ldm/models/diffusion/shared_invokeai_diffusion.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py index b4a0703f18..5e62abf9df 100644 --- a/ldm/invoke/generator/diffusers_pipeline.py +++ b/ldm/invoke/generator/diffusers_pipeline.py @@ -391,7 +391,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): for i, t in enumerate(self.progress_bar(timesteps)): batched_t.fill_(t) step_output = self.step(batched_t, latents, conditioning_data, - i, additional_guidance=additional_guidance) + step_index=i, + total_step_count=len(timesteps), + additional_guidance=additional_guidance) latents = step_output.prev_sample predicted_original = getattr(step_output, 'pred_original_sample', None) @@ -410,7 +412,8 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): @torch.inference_mode() def step(self, t: torch.Tensor, latents: torch.Tensor, conditioning_data: ConditioningData, - step_index:int | None = None, additional_guidance: List[Callable] = None): + step_index:int, total_step_count:int, + additional_guidance: List[Callable] = None): # invokeai_diffuser has batched timesteps, but diffusers schedulers expect a single value timestep = t[0] @@ -427,6 +430,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline): conditioning_data.unconditioned_embeddings, conditioning_data.text_embeddings, conditioning_data.guidance_scale, step_index=step_index, + total_step_count=total_step_count, threshold=conditioning_data.threshold ) diff --git a/ldm/models/diffusion/shared_invokeai_diffusion.py b/ldm/models/diffusion/shared_invokeai_diffusion.py index 0630546394..3be6b10170 100644 --- a/ldm/models/diffusion/shared_invokeai_diffusion.py +++ b/ldm/models/diffusion/shared_invokeai_diffusion.py @@ -89,6 +89,7 @@ class InvokeAIDiffuserComponent: conditioning: Union[torch.Tensor,dict], unconditional_guidance_scale: float, step_index: Optional[int]=None, + total_step_count: Optional[int]=None, threshold: Optional[ThresholdSettings]=None, ): """ @@ -106,7 +107,15 @@ class InvokeAIDiffuserComponent: cross_attention_control_types_to_do = [] context: Context = self.cross_attention_control_context if self.cross_attention_control_context is not None: - percent_through = self.estimate_percent_through(step_index, sigma) + if step_index is not None and total_step_count is not None: + # 🧨diffusers codepath + percent_through = step_index / total_step_count # will never reach 1.0 - this is deliberate + else: + # legacy compvis codepath + # TODO remove when compvis codepath support is dropped + if step_index is None and sigma is None: + raise ValueError(f"Either step_index or sigma is required when doing cross attention control, but both are None.") + percent_through = self.estimate_percent_through(step_index, sigma) cross_attention_control_types_to_do = context.get_active_cross_attention_control_types_for_step(percent_through) wants_cross_attention_control = (len(cross_attention_control_types_to_do) > 0) From 088cd2c4dddea15f6dd87e97e2f5a227d7258735 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 17:11:59 -0500 Subject: [PATCH 06/13] further tweaks to model management - Work around problem with OmegaConf.update() that prevented model names from containing periods. - Fix logic bug in !delete_model that didn't check for existence of model in config file. --- configs/INITIAL_MODELS.yaml | 22 +++++++++++----------- ldm/invoke/CLI.py | 7 +++---- ldm/invoke/model_manager.py | 8 +++++--- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/configs/INITIAL_MODELS.yaml b/configs/INITIAL_MODELS.yaml index cc3746224e..94502855da 100644 --- a/configs/INITIAL_MODELS.yaml +++ b/configs/INITIAL_MODELS.yaml @@ -1,14 +1,14 @@ -stable-diffusion-2_1-768: +stable-diffusion-2.1-768: description: Stable Diffusion version 2.1 diffusers model, trained on 768x768 images (5.21 GB) repo_id: stabilityai/stable-diffusion-2-1 format: diffusers recommended: True -stable-diffusion-2_1-base: +stable-diffusion-2.1-base: description: Stable Diffusion version 2.1 diffusers base model, trained on 512x512 images (5.21 GB) repo_id: stabilityai/stable-diffusion-2-1-base format: diffusers recommended: False -stable-diffusion-1_5: +stable-diffusion-1.5: description: Stable Diffusion version 1.5 weight file (4.27 GB) repo_id: runwayml/stable-diffusion-v1-5 format: diffusers @@ -16,7 +16,7 @@ stable-diffusion-1_5: default: True vae: repo_id: stabilityai/sd-vae-ft-mse -stable-diffusion-1_4: +stable-diffusion-1.4: description: The original Stable Diffusion version 1.4 weight file (4.27 GB) repo_id: CompVis/stable-diffusion-v1-4 recommended: False @@ -25,7 +25,7 @@ stable-diffusion-1_4: repo_id: stabilityai/sd-vae-ft-mse width: 512 height: 512 -inpainting-1_5: +inpainting-1.5: description: RunwayML SD 1.5 model optimized for inpainting (ckpt version) (4.27 GB) repo_id: runwayml/stable-diffusion-inpainting config: v1-inpainting-inference.yaml @@ -37,12 +37,12 @@ inpainting-1_5: recommended: True width: 512 height: 512 -waifu-diffusion-1_4: +waifu-diffusion-1.4: description: Latest waifu diffusion 1.4 (diffusers version) format: diffusers repo_id: hakurei/waifu-diffusion recommended: True -waifu-diffusion-1_3: +waifu-diffusion-1.3: description: Stable Diffusion 1.4 fine tuned on anime-styled images (ckpt version) (4.27 GB) repo_id: hakurei/waifu-diffusion-v1-3 config: v1-inference.yaml @@ -54,7 +54,7 @@ waifu-diffusion-1_3: recommended: False width: 512 height: 512 -trinart-2_0: +trinart-2.0: description: An SD model finetuned with ~40,000 assorted high resolution manga/anime-style pictures (2.13 GB) repo_id: naclbit/trinart_stable_diffusion_v2 format: diffusers @@ -73,19 +73,19 @@ trinart_characters-2_0: recommended: False width: 512 height: 512 -anything-4_0: +anything-4.0: description: High-quality, highly detailed anime style images with just a few prompts format: diffusers repo_id: andite/anything-v4.0 recommended: False -papercut-1_0: +papercut-1.0: description: SD 1.5 fine-tuned for papercut art (use "PaperCut" in your prompts) (2.13 GB) repo_id: Fictiverse/Stable_Diffusion_PaperCut_Model format: diffusers vae: repo_id: stabilityai/sd-vae-ft-mse recommended: False -voxel_art-1_0: +voxel_art-1.0: description: Stable Diffusion trained on voxel art (use "VoxelArt" in your prompts) (4.27 GB) repo_id: Fictiverse/Stable_Diffusion_VoxelArt_Model config: v1-inference.yaml diff --git a/ldm/invoke/CLI.py b/ldm/invoke/CLI.py index cfe9a64ed5..6fb0efeb8d 100644 --- a/ldm/invoke/CLI.py +++ b/ldm/invoke/CLI.py @@ -613,8 +613,6 @@ def import_diffuser_model(path_or_repo:str, gen, opt, completer)->str: description = model_description): print('** model failed to import') return None - if input('Make this the default model? [n] ').startswith(('y','Y')): - manager.set_default_model(model_name) return model_name def import_ckpt_model(path_or_url:str, gen, opt, completer)->str: @@ -647,8 +645,6 @@ def import_ckpt_model(path_or_url:str, gen, opt, completer)->str: print('** model failed to import') return None - if input('Make this the default model? [n] ').startswith(('y','Y')): - manager.set_model_default(model_name) return model_name def _verify_load(model_name:str, gen)->bool: @@ -726,6 +722,9 @@ def del_config(model_name:str, gen, opt, completer): if model_name == current_model: print("** Can't delete active model. !switch to another model first. **") return + if model_name not in gen.model_manager.config: + print(f"** Unknown model {model_name}") + return gen.model_manager.del_model(model_name) gen.model_manager.commit(opt.conf) print(f'** {model_name} deleted') diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index a00c4836e8..0e7f052456 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -230,6 +230,9 @@ class ModelManager(object): Delete the named model. ''' omega = self.config + if model_name not in omega: + print(f'** Unknown model {model_name}') + return del omega[model_name] if model_name in self.stack: self.stack.remove(model_name) @@ -253,9 +256,8 @@ class ModelManager(object): assert (clobber or model_name not in omega), f'attempt to overwrite existing model definition "{model_name}"' - if model_name not in omega: - omega[model_name] = dict() - OmegaConf.update(omega,model_name,model_attributes,merge=False) + omega[model_name] = model_attributes + if 'weights' in omega[model_name]: omega[model_name]['weights'].replace('\\','/') From 7e8f364d8db4af04c745c3f324090d98712dd5bd Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 19:32:06 -0500 Subject: [PATCH 07/13] do not use autocast for diffusers - All tensors in diffusers code path are now set explicitly to float32 or float16, depending on the --precision flag. - autocast is still used in the ckpt path, since it is being deprecated. --- ldm/generate.py | 3 ++- ldm/invoke/ckpt_generator/base.py | 3 ++- ldm/invoke/ckpt_generator/txt2img.py | 18 +++++++++------- ldm/invoke/conditioning.py | 3 +-- ldm/invoke/devices.py | 9 ++++++++ ldm/invoke/generator/base.py | 6 +++++- ldm/invoke/generator/txt2img.py | 21 ++++++++++--------- ldm/invoke/globals.py | 3 +++ ldm/invoke/model_manager.py | 2 +- ldm/modules/prompt_to_embeddings_converter.py | 4 ++-- ldm/modules/textual_inversion_manager.py | 1 - 11 files changed, 46 insertions(+), 27 deletions(-) diff --git a/ldm/generate.py b/ldm/generate.py index 7670448a93..63eaf79b50 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -29,7 +29,7 @@ from ldm.invoke.concepts_lib import HuggingFaceConceptsLibrary from ldm.invoke.conditioning import get_uc_and_c_and_ec from ldm.invoke.devices import choose_torch_device, choose_precision from ldm.invoke.generator.inpaint import infill_methods -from ldm.invoke.globals import global_cache_dir +from ldm.invoke.globals import global_cache_dir, Globals from ldm.invoke.image_util import InitImageResizer from ldm.invoke.model_manager import ModelManager from ldm.invoke.pngwriter import PngWriter @@ -201,6 +201,7 @@ class Generate: self.precision = 'float32' if self.precision == 'auto': self.precision = choose_precision(self.device) + Globals.full_precision = self.precision=='float32' # model caching system for fast switching self.model_manager = ModelManager(mconfig,self.device,self.precision,max_loaded_models=max_loaded_models) diff --git a/ldm/invoke/ckpt_generator/base.py b/ldm/invoke/ckpt_generator/base.py index c73bb50c9b..c84550a6e3 100644 --- a/ldm/invoke/ckpt_generator/base.py +++ b/ldm/invoke/ckpt_generator/base.py @@ -335,4 +335,5 @@ class CkptGenerator(): os.makedirs(dirname, exist_ok=True) image.save(filepath,'PNG') - + def torch_dtype(self)->torch.dtype: + return torch.float16 if self.precision == 'float16' else torch.float32 diff --git a/ldm/invoke/ckpt_generator/txt2img.py b/ldm/invoke/ckpt_generator/txt2img.py index 48b83be2ed..825b8583b9 100644 --- a/ldm/invoke/ckpt_generator/txt2img.py +++ b/ldm/invoke/ckpt_generator/txt2img.py @@ -72,16 +72,18 @@ class CkptTxt2Img(CkptGenerator): device = self.model.device if self.use_mps_noise or device.type == 'mps': x = torch.randn([1, - self.latent_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device='cpu').to(device) + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + dtype=self.torch_dtype(), + device='cpu').to(device) else: x = torch.randn([1, - self.latent_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device=device) + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + dtype=self.torch_dtype(), + device=device) if self.perlin > 0.0: x = (1-self.perlin)*x + self.perlin*self.get_perlin_noise(width // self.downsampling_factor, height // self.downsampling_factor) return x diff --git a/ldm/invoke/conditioning.py b/ldm/invoke/conditioning.py index fec3c7e7b1..2a687d4c51 100644 --- a/ldm/invoke/conditioning.py +++ b/ldm/invoke/conditioning.py @@ -17,7 +17,7 @@ from ..models.diffusion import cross_attention_control from ..models.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent from ..modules.encoders.modules import WeightedFrozenCLIPEmbedder from ..modules.prompt_to_embeddings_converter import WeightedPromptFragmentsToEmbeddingsConverter - +from ldm.invoke.devices import torch_dtype def get_uc_and_c_and_ec(prompt_string, model, log_tokens=False, skip_normalize_legacy_blend=False): @@ -238,7 +238,6 @@ def _get_embeddings_and_tokens_for_prompt(model, flattened_prompt: FlattenedProm if log_tokens: text = " ".join(fragments) log_tokenization(text, model, display_label=log_display_label) - return embeddings, tokens diff --git a/ldm/invoke/devices.py b/ldm/invoke/devices.py index 0fc749c4a4..94ddb74b24 100644 --- a/ldm/invoke/devices.py +++ b/ldm/invoke/devices.py @@ -21,10 +21,19 @@ def choose_precision(device) -> str: return 'float16' return 'float32' +def torch_dtype(device) -> torch.dtype: + if Globals.full_precision: + return torch.float32 + if choose_precision(device) == 'float16': + return torch.float16 + else: + return torch.float32 + def choose_autocast(precision): '''Returns an autocast context or nullcontext for the given precision string''' # float16 currently requires autocast to avoid errors like: # 'expected scalar type Half but found Float' + print(f'DEBUG: choose_autocast() called') if precision == 'autocast' or precision == 'float16': return autocast return nullcontext diff --git a/ldm/invoke/generator/base.py b/ldm/invoke/generator/base.py index 68c5ccdeff..bac7bbb333 100644 --- a/ldm/invoke/generator/base.py +++ b/ldm/invoke/generator/base.py @@ -22,6 +22,7 @@ from ldm.invoke.devices import choose_autocast from ldm.models.diffusion.cross_attention_map_saving import AttentionMapSaver from ldm.models.diffusion.ddpm import DiffusionWrapper from ldm.util import rand_perlin_2d +from contextlib import nullcontext downsampling = 8 CAUTION_IMG = 'assets/caution.png' @@ -64,7 +65,7 @@ class Generator: image_callback=None, step_callback=None, threshold=0.0, perlin=0.0, safety_checker:dict=None, **kwargs): - scope = choose_autocast(self.precision) + scope = nullcontext self.safety_checker = safety_checker attention_maps_images = [] attention_maps_callback = lambda saver: attention_maps_images.append(saver.get_stacked_maps_image()) @@ -341,3 +342,6 @@ class Generator: image.save(filepath,'PNG') + def torch_dtype(self)->torch.dtype: + return torch.float16 if self.precision == 'float16' else torch.float32 + diff --git a/ldm/invoke/generator/txt2img.py b/ldm/invoke/generator/txt2img.py index 174c1e469d..77b16a734e 100644 --- a/ldm/invoke/generator/txt2img.py +++ b/ldm/invoke/generator/txt2img.py @@ -36,10 +36,9 @@ class Txt2Img(Generator): threshold = ThresholdSettings(threshold, warmup=0.2) if threshold else None) .add_scheduler_args_if_applicable(pipeline.scheduler, eta=ddim_eta)) - def make_image(x_T) -> PIL.Image.Image: pipeline_output = pipeline.image_from_embeddings( - latents=torch.zeros_like(x_T), + latents=torch.zeros_like(x_T,dtype=self.torch_dtype()), noise=x_T, num_inference_steps=steps, conditioning_data=conditioning_data, @@ -59,16 +58,18 @@ class Txt2Img(Generator): input_channels = min(self.latent_channels, 4) if self.use_mps_noise or device.type == 'mps': x = torch.randn([1, - input_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device='cpu').to(device) + input_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + dtype=self.torch_dtype(), + device='cpu').to(device) else: x = torch.randn([1, - input_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device=device) + input_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + dtype=self.torch_dtype(), + device=device) if self.perlin > 0.0: x = (1-self.perlin)*x + self.perlin*self.get_perlin_noise(width // self.downsampling_factor, height // self.downsampling_factor) return x diff --git a/ldm/invoke/globals.py b/ldm/invoke/globals.py index c67dbac145..897bf5e204 100644 --- a/ldm/invoke/globals.py +++ b/ldm/invoke/globals.py @@ -43,6 +43,9 @@ Globals.always_use_cpu = False # The CLI will test connectivity at startup time. Globals.internet_available = True +# whether we are forcing full precision +Globals.full_precision = False + def global_config_dir()->Path: return Path(Globals.root, Globals.config_dir) diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index 880d75476f..bc19ba1449 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -761,7 +761,7 @@ class ModelManager(object): for model in legacy_locations: source = models_dir /model if source.exists(): - print(f'DEBUG: Moving {models_dir / model} into hub') + print(f'** Moving {models_dir / model} into hub') move(models_dir / model, hub) # anything else gets moved into the diffusers directory diff --git a/ldm/modules/prompt_to_embeddings_converter.py b/ldm/modules/prompt_to_embeddings_converter.py index b52577c83c..ab989e4892 100644 --- a/ldm/modules/prompt_to_embeddings_converter.py +++ b/ldm/modules/prompt_to_embeddings_converter.py @@ -4,7 +4,7 @@ import torch from transformers import CLIPTokenizer, CLIPTextModel from ldm.modules.textual_inversion_manager import TextualInversionManager - +from ldm.invoke.devices import torch_dtype class WeightedPromptFragmentsToEmbeddingsConverter(): @@ -207,7 +207,7 @@ class WeightedPromptFragmentsToEmbeddingsConverter(): per_token_weights += [1.0] * pad_length all_token_ids_tensor = torch.tensor(all_token_ids, dtype=torch.long, device=device) - per_token_weights_tensor = torch.tensor(per_token_weights, dtype=torch.float32, device=device) + per_token_weights_tensor = torch.tensor(per_token_weights, dtype=torch_dtype(self.text_encoder.device), device=device) #print(f"assembled all_token_ids_tensor with shape {all_token_ids_tensor.shape}") return all_token_ids_tensor, per_token_weights_tensor diff --git a/ldm/modules/textual_inversion_manager.py b/ldm/modules/textual_inversion_manager.py index 471a8ee07b..f7ced79a52 100644 --- a/ldm/modules/textual_inversion_manager.py +++ b/ldm/modules/textual_inversion_manager.py @@ -111,7 +111,6 @@ class TextualInversionManager(): if ti.trigger_token_id is not None: raise ValueError(f"Tokens already injected for textual inversion with trigger '{ti.trigger_string}'") - print(f'DEBUG: Injecting token {ti.trigger_string}') trigger_token_id = self._get_or_create_token_id_and_assign_embedding(ti.trigger_string, ti.embedding[0]) if ti.embedding_vector_length > 1: From ce00c9856fdf077f115d2468e9c11da907a4a3a2 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 22:50:13 -0500 Subject: [PATCH 08/13] fix perlin noise and txt2img2img --- ldm/invoke/generator/base.py | 3 ++- ldm/invoke/generator/txt2img2img.py | 10 ++++++---- ldm/invoke/model_manager.py | 2 +- ldm/util.py | 4 +++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ldm/invoke/generator/base.py b/ldm/invoke/generator/base.py index bac7bbb333..25cd281cfe 100644 --- a/ldm/invoke/generator/base.py +++ b/ldm/invoke/generator/base.py @@ -237,7 +237,8 @@ class Generator: def get_perlin_noise(self,width,height): fixdevice = 'cpu' if (self.model.device.type == 'mps') else self.model.device - return torch.stack([rand_perlin_2d((height, width), (8, 8), device = self.model.device).to(fixdevice) for _ in range(self.latent_channels)], dim=0).to(self.model.device) + noise = torch.stack([rand_perlin_2d((height, width), (8, 8), device = self.model.device).to(fixdevice) for _ in range(self.latent_channels)], dim=0).to(self.model.device) + return noise def new_seed(self): self.seed = random.randrange(0, np.iinfo(np.uint32).max) diff --git a/ldm/invoke/generator/txt2img2img.py b/ldm/invoke/generator/txt2img2img.py index e356f719c4..1dba0cfafb 100644 --- a/ldm/invoke/generator/txt2img2img.py +++ b/ldm/invoke/generator/txt2img2img.py @@ -90,9 +90,9 @@ class Txt2Img2Img(Generator): def get_noise_like(self, like: torch.Tensor): device = like.device if device.type == 'mps': - x = torch.randn_like(like, device='cpu').to(device) + x = torch.randn_like(like, device='cpu', dtype=self.torch_dtype()).to(device) else: - x = torch.randn_like(like, device=device) + x = torch.randn_like(like, device=device, dtype=self.torch_dtype()) if self.perlin > 0.0: shape = like.shape x = (1-self.perlin)*x + self.perlin*self.get_perlin_noise(shape[3], shape[2]) @@ -117,10 +117,12 @@ class Txt2Img2Img(Generator): self.latent_channels, scaled_height // self.downsampling_factor, scaled_width // self.downsampling_factor], - device='cpu').to(device) + dtype=self.torch_dtype(), + device='cpu').to(device) else: return torch.randn([1, self.latent_channels, scaled_height // self.downsampling_factor, scaled_width // self.downsampling_factor], - device=device) + dtype=self.torch_dtype(), + device=device) diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index bc19ba1449..f74706aaef 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -349,7 +349,7 @@ class ModelManager(object): if self.precision == 'float16': print(' | Using faster float16 precision') - model.to(torch.float16) + model = model.to(torch.float16) else: print(' | Using more accurate float32 precision') diff --git a/ldm/util.py b/ldm/util.py index 282a56c3e5..7d44dcd266 100644 --- a/ldm/util.py +++ b/ldm/util.py @@ -8,6 +8,7 @@ from threading import Thread from urllib import request from tqdm import tqdm from pathlib import Path +from ldm.invoke.devices import torch_dtype import numpy as np import torch @@ -235,7 +236,8 @@ def rand_perlin_2d(shape, res, device, fade = lambda t: 6*t**5 - 15*t**4 + 10*t* n01 = dot(tile_grads([0, -1],[1, None]), [0, -1]).to(device) n11 = dot(tile_grads([1, None], [1, None]), [-1,-1]).to(device) t = fade(grid[:shape[0], :shape[1]]) - return math.sqrt(2) * torch.lerp(torch.lerp(n00, n10, t[..., 0]), torch.lerp(n01, n11, t[..., 0]), t[..., 1]).to(device) + noise = math.sqrt(2) * torch.lerp(torch.lerp(n00, n10, t[..., 0]), torch.lerp(n01, n11, t[..., 0]), t[..., 1]).to(device) + return noise.to(dtype=torch_dtype(device)) def ask_user(question: str, answers: list): from itertools import chain, repeat From ce1c5e70b8f260955592b837db78332db0cf70f5 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 16 Jan 2023 23:18:43 -0500 Subject: [PATCH 09/13] fix autocast dependency in cross_attention_control --- ldm/models/diffusion/cross_attention_control.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldm/models/diffusion/cross_attention_control.py b/ldm/models/diffusion/cross_attention_control.py index 7415f1435b..03d5a5bcec 100644 --- a/ldm/models/diffusion/cross_attention_control.py +++ b/ldm/models/diffusion/cross_attention_control.py @@ -7,6 +7,7 @@ import torch import diffusers from torch import nn from diffusers.models.unet_2d_condition import UNet2DConditionModel +from ldm.invoke.devices import torch_dtype # adapted from bloc97's CrossAttentionControl colab # https://github.com/bloc97/CrossAttentionControl @@ -383,7 +384,7 @@ def inject_attention_function(unet, context: Context): remapped_saved_attention_slice = torch.index_select(saved_attention_slice, -1, index_map) this_attention_slice = suggested_attention_slice - mask = context.cross_attention_mask + mask = context.cross_attention_mask.to(torch_dtype(suggested_attention_slice.device)) saved_mask = mask this_mask = 1 - mask attention_slice = remapped_saved_attention_slice * saved_mask + \ From 212fec669a478b7ed5d0b0662d7e85251ca3d715 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 17 Jan 2023 12:45:04 -0500 Subject: [PATCH 10/13] add --default_only arg to configure_invokeai.py for CI use Added a --default_only argument that limits model downloads to the single default model, for use in continuous integration. New behavior - switch - --yes --default_only Behavior ----- -------------- -------- interactive download --yes non-interactively download all recommended models --yes --default_only non-interactively download the default model --- scripts/configure_invokeai.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/configure_invokeai.py b/scripts/configure_invokeai.py index 22ade19cdb..9d17a73317 100755 --- a/scripts/configure_invokeai.py +++ b/scripts/configure_invokeai.py @@ -197,6 +197,14 @@ def recommended_datasets()->dict: datasets[ds]=True return datasets +#--------------------------------------------- +def default_dataset()->dict: + datasets = dict() + for ds in Datasets.keys(): + if Datasets[ds].get('default',False): + datasets[ds]=True + return datasets + #--------------------------------------------- def all_datasets()->dict: datasets = dict() @@ -646,7 +654,7 @@ def download_weights(opt:dict) -> Union[str, None]: precision = 'float32' if opt.full_precision else choose_precision(torch.device(choose_torch_device())) if opt.yes_to_all: - models = recommended_datasets() + models = default_dataset() if opt.default_only else recommended_datasets() access_token = authenticate(opt.yes_to_all) if len(models)>0: successfully_downloaded = download_weight_datasets(models, access_token, precision=precision) @@ -808,6 +816,9 @@ def main(): dest='yes_to_all', action='store_true', help='answer "yes" to all prompts') + parser.add_argument('--default_only', + action='store_true', + help='when --yes specified, only install the default model') parser.add_argument('--config_file', '-c', dest='config_file', From 858ddffab6a1c8c30af350033e6718d7220bb416 Mon Sep 17 00:00:00 2001 From: mauwii Date: Tue, 17 Jan 2023 20:10:37 +0100 Subject: [PATCH 11/13] add `--default_only` to run-preload-models step --- .github/workflows/test-invoke-pip.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-invoke-pip.yml b/.github/workflows/test-invoke-pip.yml index 34c90f10c5..4985202782 100644 --- a/.github/workflows/test-invoke-pip.yml +++ b/.github/workflows/test-invoke-pip.yml @@ -119,6 +119,7 @@ jobs: run: > configure_invokeai.py --yes + --default_only --full-precision # can't use fp16 weights without a GPU - name: Run the tests From 3c919f0337d38e72c8e007a33458dbb070e4e697 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:37:14 -0800 Subject: [PATCH 12/13] Restore ldm/invoke/conditioning.py --- ldm/invoke/conditioning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldm/invoke/conditioning.py b/ldm/invoke/conditioning.py index 2a687d4c51..fec3c7e7b1 100644 --- a/ldm/invoke/conditioning.py +++ b/ldm/invoke/conditioning.py @@ -17,7 +17,7 @@ from ..models.diffusion import cross_attention_control from ..models.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent from ..modules.encoders.modules import WeightedFrozenCLIPEmbedder from ..modules.prompt_to_embeddings_converter import WeightedPromptFragmentsToEmbeddingsConverter -from ldm.invoke.devices import torch_dtype + def get_uc_and_c_and_ec(prompt_string, model, log_tokens=False, skip_normalize_legacy_blend=False): @@ -238,6 +238,7 @@ def _get_embeddings_and_tokens_for_prompt(model, flattened_prompt: FlattenedProm if log_tokens: text = " ".join(fragments) log_tokenization(text, model, display_label=log_display_label) + return embeddings, tokens From 5aec48735e0bb2fbb19f4b69469bce3c46035e28 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:44:45 -0800 Subject: [PATCH 13/13] =?UTF-8?q?lint(generator):=20=F0=9F=9A=AE=20remove?= =?UTF-8?q?=20unused=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ldm/invoke/generator/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ldm/invoke/generator/base.py b/ldm/invoke/generator/base.py index 25cd281cfe..3fd34765c6 100644 --- a/ldm/invoke/generator/base.py +++ b/ldm/invoke/generator/base.py @@ -8,6 +8,7 @@ import os import os.path as osp import random import traceback +from contextlib import nullcontext import cv2 import numpy as np @@ -18,11 +19,8 @@ from einops import rearrange from pytorch_lightning import seed_everything from tqdm import trange -from ldm.invoke.devices import choose_autocast -from ldm.models.diffusion.cross_attention_map_saving import AttentionMapSaver from ldm.models.diffusion.ddpm import DiffusionWrapper from ldm.util import rand_perlin_2d -from contextlib import nullcontext downsampling = 8 CAUTION_IMG = 'assets/caution.png'