github/InvokeAI
1
1
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI.git synced 2026-02-16 23:01:28 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make FLUX controlnet node API more like SD API and get it working with linear UI.

Browse Source
This commit is contained in:
Ryan Dick
2024-10-09 23:38:31 +00:00
parent 3953e60a4f
commit b1567fe0e4
5 changed files with 25 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
invokeai/app/invocations/flux_controlnet.py
View File

@@ -16,7 +16,7 @@ from invokeai.app.util.controlnet_utils import CONTROLNET_RESIZE_VALUES
class FluxControlNetField(BaseModel):
image: ImageField = Field(description="The control image")
controlnet_model: ModelIdentifierField = Field(description="The ControlNet model to use")
control_model: ModelIdentifierField = Field(description="The ControlNet model to use")
control_weight: float | list[float] = Field(default=1, description="The weight given to the ControlNet")
begin_step_percent: float = Field(
default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
@@ -42,7 +42,7 @@ class FluxControlNetField(BaseModel):
class FluxControlNetOutput(BaseInvocationOutput):
"""FLUX ControlNet info"""
controlnet: FluxControlNetField = OutputField(description=FieldDescriptions.control)
control: FluxControlNetField = OutputField(description=FieldDescriptions.control)
@invocation(
@@ -57,7 +57,7 @@ class FluxControlNetInvocation(BaseInvocation):
"""Collect FLUX ControlNet info to pass to other nodes."""
image: ImageField = InputField(description="The control image")
controlnet_model: ModelIdentifierField = InputField(
control_model: ModelIdentifierField = InputField(
description=FieldDescriptions.controlnet_model, ui_type=UIType.ControlNetModel
)
control_weight: float | list[float] = InputField(
@@ -84,9 +84,9 @@ class FluxControlNetInvocation(BaseInvocation):
def invoke(self, context: InvocationContext) -> FluxControlNetOutput:
return FluxControlNetOutput(
controlnet=FluxControlNetField(
control=FluxControlNetField(
image=self.image,
controlnet_model=self.controlnet_model,
control_model=self.control_model,
control_weight=self.control_weight,
begin_step_percent=self.begin_step_percent,
end_step_percent=self.end_step_percent,

16
invokeai/app/invocations/flux_denoise.py
View File

@@ -92,7 +92,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
description="The guidance strength. Higher values adhere more strictly to the prompt, and will produce less diverse images. FLUX dev only, ignored for schnell.",
)
seed: int = InputField(default=0, description="Randomness seed for reproducibility.")
controlnet: FluxControlNetField | list[FluxControlNetField] | None = InputField(
control: FluxControlNetField | list[FluxControlNetField] | None = InputField(
default=None, input=Input.Connection, description="ControlNet models."
)
controlnet_vae: VAEField | None = InputField(
@@ -322,14 +322,14 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
) -> list[XLabsControlNetExtension | InstantXControlNetExtension]:
# Normalize the controlnet input to list[ControlField].
controlnets: list[FluxControlNetField]
if self.controlnet is None:
if self.control is None:
controlnets = []
elif isinstance(self.controlnet, FluxControlNetField):
controlnets = [self.controlnet]
elif isinstance(self.controlnet, list):
controlnets = self.controlnet
elif isinstance(self.control, FluxControlNetField):
controlnets = [self.control]
elif isinstance(self.control, list):
controlnets = self.control
else:
raise ValueError(f"Unsupported controlnet type: {type(self.controlnet)}")
raise ValueError(f"Unsupported controlnet type: {type(self.control)}")
# TODO(ryand): Add a field to the model config so that we can distinguish between XLabs and InstantX ControlNets
# before loading the models. Then make sure that all VAE encoding is done before loading the ControlNets to
@@ -337,7 +337,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
controlnet_extensions: list[XLabsControlNetExtension | InstantXControlNetExtension] = []
for controlnet in controlnets:
model = exit_stack.enter_context(context.models.load(controlnet.controlnet_model))
model = exit_stack.enter_context(context.models.load(controlnet.control_model))
image = context.images.get_pil(controlnet.image.image_name)
if isinstance(model, XLabsControlNetFlux):

7
invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts
View File

@@ -119,12 +119,7 @@ const addControlNetToGraph = (
control_weight: weight,
image: { image_name },
});
if (controlNet.type === 'flux_controlnet') {
g.addEdge(controlNet, 'controlnet', collector, 'item');
} else {
g.addEdge(controlNet, 'control', collector, 'item');
}
g.addEdge(controlNet, 'control', collector, 'item');
};
const addT2IAdapterToGraph = (

3
invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts
View File

@@ -95,6 +95,7 @@ export const buildFLUXGraph = async (
> = l2i;
g.addEdge(modelLoader, 'transformer', noise, 'transformer');
g.addEdge(modelLoader, 'vae', noise, 'controlnet_vae');
g.addEdge(modelLoader, 'vae', l2i, 'vae');
g.addEdge(modelLoader, 'clip', posCond, 'clip');
@@ -192,7 +193,7 @@ export const buildFLUXGraph = async (
modelConfig.base
);
if (controlNetResult.addedControlNets > 0) {
g.addEdge(controlNetCollector, 'collection', noise, 'controlnet');
g.addEdge(controlNetCollector, 'collection', noise, 'control');
} else {
g.deleteNode(controlNetCollector.id);
}

30
invokeai/frontend/web/src/services/api/schema.ts
View File

@@ -4622,7 +4622,7 @@ export type components = {
/**
* Fp32
* @description Whether or not to use full float32 precision
* @default true
* @default false
*/
fp32?: boolean;
/**
@@ -4705,7 +4705,7 @@ export type components = {
/**
* Fp32
* @description Whether or not to use full float32 precision
* @default true
* @default false
*/
fp32?: boolean;
/**
@@ -6297,7 +6297,7 @@ export type components = {
/** @description The control image */
image: components["schemas"]["ImageField"];
/** @description The ControlNet model to use */
controlnet_model: components["schemas"]["ModelIdentifierField"];
control_model: components["schemas"]["ModelIdentifierField"];
/**
* Control Weight
* @description The weight given to the ControlNet
@@ -6323,12 +6323,6 @@ export type components = {
* @enum {string}
*/
resize_mode?: "just_resize" | "crop_resize" | "fill_resize" | "just_resize_simple";
/**
* Instantx Control Mode
* @description The control mode for InstantX ControlNet union models. Ignored for other ControlNet models.
* @default 0
*/
instantx_control_mode?: number;
};
/**
* FLUX ControlNet
@@ -6361,7 +6355,7 @@ export type components = {
* @description ControlNet model to load
* @default null
*/
controlnet_model?: components["schemas"]["ModelIdentifierField"];
control_model?: components["schemas"]["ModelIdentifierField"];
/**
* Control Weight
* @description The weight given to the ControlNet
@@ -6387,12 +6381,6 @@ export type components = {
* @enum {string}
*/
resize_mode?: "just_resize" | "crop_resize" | "fill_resize" | "just_resize_simple";
/**
* Instantx Control Mode
* @description The control mode for InstantX ControlNet union models. Ignored for other ControlNet models.
* @default 0
*/
instantx_control_mode?: number;
/**
* type
* @default flux_controlnet
@@ -6407,7 +6395,7 @@ export type components = {
*/
FluxControlNetOutput: {
/** @description ControlNet(s) to apply */
controlnet: components["schemas"]["FluxControlNetField"];
control: components["schemas"]["FluxControlNetField"];
/**
* type
* @default flux_controlnet_output
@@ -6512,11 +6500,11 @@ export type components = {
*/
seed?: number;
/**
* Controlnet
* Control
* @description ControlNet models.
* @default null
*/
controlnet?: components["schemas"]["FluxControlNetField"] | components["schemas"]["FluxControlNetField"][] | null;
control?: components["schemas"]["FluxControlNetField"] | components["schemas"]["FluxControlNetField"][] | null;
/**
* @description VAE
* @default null
@@ -8863,7 +8851,7 @@ export type components = {
/**
* Fp32
* @description Whether or not to use full float32 precision
* @default true
* @default false
*/
fp32?: boolean;
/**
@@ -10146,7 +10134,7 @@ export type components = {
/**
* Fp32
* @description Whether or not to use full float32 precision
* @default true
* @default false
*/
fp32?: boolean;
/**