Matt3o base changes

invokeai/app/invocations/denoise_latents.py

@@ -608,6 +608,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
                     end_step_percent=single_ip_adapter.end_step_percent,
                     ip_adapter_conditioning=IPAdapterConditioningInfo(image_prompt_embeds, uncond_image_prompt_embeds),
                     mask=mask,
+                    method=single_ip_adapter.method
                 )
             )
 

invokeai/app/invocations/ip_adapter.py

@@ -31,6 +31,7 @@ class IPAdapterField(BaseModel):
     image_encoder_model: ModelIdentifierField = Field(description="The name of the CLIP image encoder model.")
     weight: Union[float, List[float]] = Field(default=1, description="The weight given to the IP-Adapter.")
     target_blocks: List[str] = Field(default=[], description="The IP Adapter blocks to apply")
+    method: str = Field(default='full', description="Weight apply method")
     begin_step_percent: float = Field(
         default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)"
     )
@@ -137,7 +138,7 @@ class IPAdapterInvocation(BaseInvocation):
             if ip_adapter_info.base == "sd-1":
                 target_blocks = ["up_blocks.1"]
             elif ip_adapter_info.base == "sdxl":
-                target_blocks = ["up_blocks.0.attentions.1"]
+                target_blocks = ["up_blocks.0.attentions.1", "down_blocks.2.attentions.1"]
             else:
                 raise ValueError(f"Unsupported IP-Adapter base type: '{ip_adapter_info.base}'.")
         elif self.method == "composition":
@@ -162,6 +163,7 @@ class IPAdapterInvocation(BaseInvocation):
                 begin_step_percent=self.begin_step_percent,
                 end_step_percent=self.end_step_percent,
                 mask=self.mask,
+                method=self.method
             ),
         )
 

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -371,7 +371,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
 
         if use_ip_adapter or use_regional_prompting:
             ip_adapters: Optional[List[UNetIPAdapterData]] = (
-                [{"ip_adapter": ipa.ip_adapter_model, "target_blocks": ipa.target_blocks} for ipa in ip_adapter_data]
+                [{"ip_adapter": ipa.ip_adapter_model, "target_blocks": ipa.target_blocks, "method": ipa.method} for ipa in ip_adapter_data]
                 if use_ip_adapter
                 else None
             )

invokeai/backend/stable_diffusion/diffusion/conditioning_data.py

@@ -113,6 +113,7 @@ class IPAdapterData:
     weight: Union[float, List[float]] = 1.0
     begin_step_percent: float = 0.0
     end_step_percent: float = 1.0
+    method: str = 'full'
 
     def scale_for_step(self, step_index: int, total_steps: int) -> float:
         first_adapter_step = math.floor(self.begin_step_percent * total_steps)

invokeai/backend/stable_diffusion/diffusion/custom_atttention.py

@@ -14,7 +14,7 @@ from invokeai.backend.stable_diffusion.diffusion.regional_prompt_data import Reg
 class IPAdapterAttentionWeights:
     ip_adapter_weights: IPAttentionProcessorWeights
     skip: bool
-
+    negative: bool
 
 class CustomAttnProcessor2_0(AttnProcessor2_0):
     """A custom implementation of AttnProcessor2_0 that supports additional Invoke features.
@@ -134,6 +134,7 @@ class CustomAttnProcessor2_0(AttnProcessor2_0):
         # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         # End unmodified block from AttnProcessor2_0.
 
+
         # Apply IP-Adapter conditioning.
         if is_cross_attention:
             if self._ip_adapter_attention_weights:
@@ -162,6 +163,10 @@ class CustomAttnProcessor2_0(AttnProcessor2_0):
                     # Expected ip_hidden_state shape: (batch_size, num_ip_images, ip_seq_len, ip_image_embedding)
 
                     if not self._ip_adapter_attention_weights[ipa_index].skip:
+                        # apply the IP-Adapter weights to the negative embeds
+                        if self._ip_adapter_attention_weights[ipa_index].negative:
+                            ip_hidden_states = torch.cat([ip_hidden_states[1], ip_hidden_states[0]*0], dim=0)
+
                         ip_key = ipa_weights.to_k_ip(ip_hidden_states)
                         ip_value = ipa_weights.to_v_ip(ip_hidden_states)
 

invokeai/backend/stable_diffusion/diffusion/unet_attention_patcher.py

@@ -13,6 +13,7 @@ from invokeai.backend.stable_diffusion.diffusion.custom_atttention import (
 class UNetIPAdapterData(TypedDict):
     ip_adapter: IPAdapter
     target_blocks: List[str]
+    method: str
 
 
 class UNetAttentionPatcher:
@@ -39,12 +40,14 @@ class UNetAttentionPatcher:
                 for ip_adapter in self._ip_adapters:
                     ip_adapter_weights = ip_adapter["ip_adapter"].attn_weights.get_attention_processor_weights(idx)
                     skip = True
+                    negative = False
                     for block in ip_adapter["target_blocks"]:
                         if block in name:
                             skip = False
+                            negative = (ip_adapter["method"] == 'style' and block == "down_blocks.2.attentions.1")
                             break
                     ip_adapter_attention_weights: IPAdapterAttentionWeights = IPAdapterAttentionWeights(
-                        ip_adapter_weights=ip_adapter_weights, skip=skip
+                        ip_adapter_weights=ip_adapter_weights, skip=skip, negative=negative
                     )
                     ip_adapter_attention_weights_collection.append(ip_adapter_attention_weights)