From 91f7abb398d87a89e7698363807404d38c0b8921 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Mon, 20 Feb 2023 22:33:13 -0500 Subject: [PATCH 1/4] replace repeated triggers with --- ldm/modules/textual_inversion_manager.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ldm/modules/textual_inversion_manager.py b/ldm/modules/textual_inversion_manager.py index bf0d3ed8b9..be44c4c857 100644 --- a/ldm/modules/textual_inversion_manager.py +++ b/ldm/modules/textual_inversion_manager.py @@ -102,6 +102,7 @@ class TextualInversionManager: embedding_info["name"], embedding_info["embedding"], defer_injecting_tokens=defer_injecting_tokens, + source_file=ckpt_path, ) except ValueError as e: print(f' | Ignoring incompatible embedding {embedding_info["name"]}') @@ -112,7 +113,7 @@ class TextualInversionManager: ) def _add_textual_inversion( - self, trigger_str, embedding, defer_injecting_tokens=False + self, trigger_str, embedding, defer_injecting_tokens=False, source_file=Path ) -> TextualInversion: """ Add a textual inversion to be recognised. @@ -120,11 +121,13 @@ class TextualInversionManager: :param embedding: The actual embedding data that will be inserted into the conditioning at the point where the token_str appears. :return: The token id for the added embedding, either existing or newly-added. """ - if trigger_str in [ti.trigger_string for ti in self.textual_inversions]: + triggers = [ti.trigger_string for ti in self.textual_inversions] + if trigger_str in triggers: + new_trigger_str = f'<{source_file.stem}>' print( - f">> TextualInversionManager refusing to overwrite already-loaded token '{trigger_str}'" + f">> {source_file.parents[0].name}/{source_file.name}: Trigger token '{trigger_str}' already in use. Trigger with {new_trigger_str}" ) - return + trigger_str = new_trigger_str if not self.full_precision: embedding = embedding.half() if len(embedding.shape) == 1: From 9436f2e3d1de962470c3d54379b8e629e1014310 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 21 Feb 2023 06:23:34 -0500 Subject: [PATCH 2/4] alphabetize trigger strings --- ldm/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/generate.py b/ldm/generate.py index 76629ff2f9..5379d49fa4 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -972,7 +972,7 @@ class Generate: ti_path, defer_injecting_tokens=True ) print( - f'>> Textual inversion triggers: {", ".join(self.model.textual_inversion_manager.get_all_trigger_strings())}' + f'>> Textual inversion triggers: {", ".join(sorted(self.model.textual_inversion_manager.get_all_trigger_strings()))}' ) self.model_name = model_name From 5e41811fb5e378cd9bd0ccecc348a0536d7122a0 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 21 Feb 2023 17:04:42 -0500 Subject: [PATCH 3/4] move trigger text munging to upper level per review --- ldm/modules/textual_inversion_manager.py | 79 +++++++++++++++--------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/ldm/modules/textual_inversion_manager.py b/ldm/modules/textual_inversion_manager.py index be44c4c857..72ae8979e2 100644 --- a/ldm/modules/textual_inversion_manager.py +++ b/ldm/modules/textual_inversion_manager.py @@ -34,6 +34,7 @@ class TextualInversionManager: self.text_encoder = text_encoder self.full_precision = full_precision self.hf_concepts_library = HuggingFaceConceptsLibrary() + self.trigger_to_sourcefile = dict() default_textual_inversions: list[TextualInversion] = [] self.textual_inversions = default_textual_inversions @@ -59,15 +60,17 @@ class TextualInversionManager: def get_all_trigger_strings(self) -> list[str]: return [ti.trigger_string for ti in self.textual_inversions] - def load_textual_inversion(self, ckpt_path: Union[str,Path], defer_injecting_tokens: bool = False): + def load_textual_inversion( + self, ckpt_path: Union[str, Path], defer_injecting_tokens: bool = False + ): ckpt_path = Path(ckpt_path) if not ckpt_path.is_file(): return - + if str(ckpt_path).endswith(".DS_Store"): return - + try: scan_result = scan_file_path(str(ckpt_path)) if scan_result.infected_files == 1: @@ -89,31 +92,48 @@ class TextualInversionManager: return elif ( self.text_encoder.get_input_embeddings().weight.data[0].shape[0] - != embedding_info['token_dim'] + != embedding_info["token_dim"] ): print( f"** Notice: {ckpt_path.parents[0].name}/{ckpt_path.name} was trained on a model with an incompatible token dimension: {self.text_encoder.get_input_embeddings().weight.data[0].shape[0]} vs {embedding_info['token_dim']}." ) return - if embedding_info: - try: - self._add_textual_inversion( - embedding_info["name"], - embedding_info["embedding"], - defer_injecting_tokens=defer_injecting_tokens, - source_file=ckpt_path, - ) - except ValueError as e: - print(f' | Ignoring incompatible embedding {embedding_info["name"]}') - print(f" | The error was {str(e)}") - else: - print( - f">> Failed to load embedding located at {str(ckpt_path)}. Unsupported file." + # Resolve the situation in which an earlier embedding has claimed the same + # trigger string. We replace the trigger with '', as we used to. + trigger_str = embedding_info["name"] + sourcefile = ( + f"{ckpt_path.parent.name}/{ckpt_path.name}" + if ckpt_path.name == "learned_embeds.bin" + else ckpt_path.name + ) + + if trigger_str in self.trigger_to_sourcefile: + replacement_trigger_str = ( + f"<{ckpt_path.parent.name}>" + if ckpt_path.name == "learned_embeds.bin" + else f"<{ckpt_path.stem}>" ) + print( + f">> {sourcefile}: Trigger token '{trigger_str}' is already claimed by '{self.trigger_to_sourcefile[trigger_str]}'. Trigger this concept with {replacement_trigger_str}" + ) + trigger_str = replacement_trigger_str + + try: + self._add_textual_inversion( + trigger_str, + embedding_info["embedding"], + defer_injecting_tokens=defer_injecting_tokens, + ) + except ValueError as e: + print(f' | Ignoring incompatible embedding {embedding_info["name"]}') + print(f" | The error was {str(e)}") + + # remember which source file claims this trigger + self.trigger_to_sourcefile[trigger_str] = sourcefile def _add_textual_inversion( - self, trigger_str, embedding, defer_injecting_tokens=False, source_file=Path + self, trigger_str, embedding, defer_injecting_tokens=False ) -> TextualInversion: """ Add a textual inversion to be recognised. @@ -121,20 +141,18 @@ class TextualInversionManager: :param embedding: The actual embedding data that will be inserted into the conditioning at the point where the token_str appears. :return: The token id for the added embedding, either existing or newly-added. """ - triggers = [ti.trigger_string for ti in self.textual_inversions] - if trigger_str in triggers: - new_trigger_str = f'<{source_file.stem}>' + if trigger_str in [ti.trigger_string for ti in self.textual_inversions]: print( - f">> {source_file.parents[0].name}/{source_file.name}: Trigger token '{trigger_str}' already in use. Trigger with {new_trigger_str}" + f"** TextualInversionManager refusing to overwrite already-loaded token '{trigger_str}'" ) - trigger_str = new_trigger_str + return if not self.full_precision: embedding = embedding.half() if len(embedding.shape) == 1: embedding = embedding.unsqueeze(0) elif len(embedding.shape) > 2: raise ValueError( - f"TextualInversionManager cannot add {trigger_str} because the embedding shape {embedding.shape} is incorrect. The embedding must have shape [token_dim] or [V, token_dim] where V is vector length and token_dim is 768 for SD1 or 1280 for SD2." + f"** TextualInversionManager cannot add {trigger_str} because the embedding shape {embedding.shape} is incorrect. The embedding must have shape [token_dim] or [V, token_dim] where V is vector length and token_dim is 768 for SD1 or 1280 for SD2." ) try: @@ -150,7 +168,7 @@ class TextualInversionManager: else: traceback.print_exc() print( - f">> TextualInversionManager was unable to add a textual inversion with trigger string {trigger_str}." + f"** TextualInversionManager was unable to add a textual inversion with trigger string {trigger_str}." ) raise @@ -297,7 +315,7 @@ class TextualInversionManager: elif file_type == "bin": return self._parse_embedding_bin(embedding_file) else: - print(f">> Not a recognized embedding file: {embedding_file}") + print(f"** Notice: unrecognized embedding file format: {embedding_file}") return None def _parse_embedding_pt(self, embedding_file): @@ -358,8 +376,9 @@ class TextualInversionManager: embedding_info = None else: for token in list(embedding_ckpt.keys()): - embedding_info["name"] = token or os.path.basename( - os.path.splitext(embedding_file)[0] + embedding_info["name"] = ( + token + or f"<{os.path.basename(os.path.splitext(embedding_file)[0])}>" ) embedding_info["embedding"] = embedding_ckpt[token] embedding_info[ @@ -383,7 +402,7 @@ class TextualInversionManager: embedding_info["name"] = ( token if token != "*" - else os.path.basename(os.path.splitext(embedding_file)[0]) + else f"<{os.path.basename(os.path.splitext(embedding_file)[0])}>" ) embedding_info["embedding"] = embedding_ckpt[ "string_to_param" From d41dcdfc46c15de36a3f44c177af77f2698099ec Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 21 Feb 2023 21:38:42 -0500 Subject: [PATCH 4/4] move trigger_str registration into try block --- ldm/modules/textual_inversion_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ldm/modules/textual_inversion_manager.py b/ldm/modules/textual_inversion_manager.py index 72ae8979e2..ee86c9d226 100644 --- a/ldm/modules/textual_inversion_manager.py +++ b/ldm/modules/textual_inversion_manager.py @@ -125,13 +125,13 @@ class TextualInversionManager: embedding_info["embedding"], defer_injecting_tokens=defer_injecting_tokens, ) + # remember which source file claims this trigger + self.trigger_to_sourcefile[trigger_str] = sourcefile + except ValueError as e: print(f' | Ignoring incompatible embedding {embedding_info["name"]}') print(f" | The error was {str(e)}") - # remember which source file claims this trigger - self.trigger_to_sourcefile[trigger_str] = sourcefile - def _add_textual_inversion( self, trigger_str, embedding, defer_injecting_tokens=False ) -> TextualInversion: