From 04b57c408f4d3b5befaf8c1c2c1afbe789f911ca Mon Sep 17 00:00:00 2001 From: Sergey Borisov Date: Thu, 6 Jul 2023 16:09:40 +0300 Subject: [PATCH] Add clip skip option to prompt node --- invokeai/app/invocations/compel.py | 2 ++ invokeai/backend/model_management/lora.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/invokeai/app/invocations/compel.py b/invokeai/app/invocations/compel.py index 4850b9670d..4941f9ee8a 100644 --- a/invokeai/app/invocations/compel.py +++ b/invokeai/app/invocations/compel.py @@ -44,6 +44,7 @@ class CompelInvocation(BaseInvocation): prompt: str = Field(default="", description="Prompt") clip: ClipField = Field(None, description="Clip to use") + clip_skip: int = Field(0, description="Layers to skip in text_encoder") # Schema customisation class Config(InvocationConfig): @@ -95,6 +96,7 @@ class CompelInvocation(BaseInvocation): with ModelPatcher.apply_lora_text_encoder(text_encoder_info.context.model, _lora_loader()),\ ModelPatcher.apply_ti(tokenizer_info.context.model, text_encoder_info.context.model, ti_list) as (tokenizer, ti_manager),\ + ModelPatcher.apply_clip_skip(text_encoder_info.context.model, self.clip_skip),\ text_encoder_info as text_encoder: compel = Compel( diff --git a/invokeai/backend/model_management/lora.py b/invokeai/backend/model_management/lora.py index d8ecdf81c2..3d2fc327f4 100644 --- a/invokeai/backend/model_management/lora.py +++ b/invokeai/backend/model_management/lora.py @@ -615,6 +615,24 @@ class ModelPatcher: text_encoder.resize_token_embeddings(init_tokens_count) + @classmethod + @contextmanager + def apply_clip_skip( + cls, + text_encoder: CLIPTextModel, + clip_skip: int, + ): + skipped_layers = [] + try: + for i in range(clip_skip): + skipped_layers.append(text_encoder.text_model.encoder.layers.pop(-1)) + + yield + + finally: + while len(skipped_layers) > 0: + text_encoder.text_model.encoder.layers.append(skipped_layers.pop()) + class TextualInversionModel: name: str embedding: torch.Tensor # [n, 768]|[n, 1280]