From b1567fe0e4d1ad462758d44802d082f9c0c6e7dc Mon Sep 17 00:00:00 2001 From: Ryan Dick Date: Wed, 9 Oct 2024 23:38:31 +0000 Subject: [PATCH] Make FLUX controlnet node API more like SD API and get it working with linear UI. --- invokeai/app/invocations/flux_controlnet.py | 10 +++---- invokeai/app/invocations/flux_denoise.py | 16 +++++----- .../graph/generation/addControlAdapters.ts | 7 +---- .../util/graph/generation/buildFLUXGraph.ts | 3 +- .../frontend/web/src/services/api/schema.ts | 30 ++++++------------- 5 files changed, 25 insertions(+), 41 deletions(-) diff --git a/invokeai/app/invocations/flux_controlnet.py b/invokeai/app/invocations/flux_controlnet.py index 2daded9287..6dd3738c80 100644 --- a/invokeai/app/invocations/flux_controlnet.py +++ b/invokeai/app/invocations/flux_controlnet.py @@ -16,7 +16,7 @@ from invokeai.app.util.controlnet_utils import CONTROLNET_RESIZE_VALUES class FluxControlNetField(BaseModel): image: ImageField = Field(description="The control image") - controlnet_model: ModelIdentifierField = Field(description="The ControlNet model to use") + control_model: ModelIdentifierField = Field(description="The ControlNet model to use") control_weight: float | list[float] = Field(default=1, description="The weight given to the ControlNet") begin_step_percent: float = Field( default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)" @@ -42,7 +42,7 @@ class FluxControlNetField(BaseModel): class FluxControlNetOutput(BaseInvocationOutput): """FLUX ControlNet info""" - controlnet: FluxControlNetField = OutputField(description=FieldDescriptions.control) + control: FluxControlNetField = OutputField(description=FieldDescriptions.control) @invocation( @@ -57,7 +57,7 @@ class FluxControlNetInvocation(BaseInvocation): """Collect FLUX ControlNet info to pass to other nodes.""" image: ImageField = InputField(description="The control image") - controlnet_model: ModelIdentifierField = InputField( + control_model: ModelIdentifierField = InputField( description=FieldDescriptions.controlnet_model, ui_type=UIType.ControlNetModel ) control_weight: float | list[float] = InputField( @@ -84,9 +84,9 @@ class FluxControlNetInvocation(BaseInvocation): def invoke(self, context: InvocationContext) -> FluxControlNetOutput: return FluxControlNetOutput( - controlnet=FluxControlNetField( + control=FluxControlNetField( image=self.image, - controlnet_model=self.controlnet_model, + control_model=self.control_model, control_weight=self.control_weight, begin_step_percent=self.begin_step_percent, end_step_percent=self.end_step_percent, diff --git a/invokeai/app/invocations/flux_denoise.py b/invokeai/app/invocations/flux_denoise.py index fc3d3cb8fd..5e295f9950 100644 --- a/invokeai/app/invocations/flux_denoise.py +++ b/invokeai/app/invocations/flux_denoise.py @@ -92,7 +92,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard): description="The guidance strength. Higher values adhere more strictly to the prompt, and will produce less diverse images. FLUX dev only, ignored for schnell.", ) seed: int = InputField(default=0, description="Randomness seed for reproducibility.") - controlnet: FluxControlNetField | list[FluxControlNetField] | None = InputField( + control: FluxControlNetField | list[FluxControlNetField] | None = InputField( default=None, input=Input.Connection, description="ControlNet models." ) controlnet_vae: VAEField | None = InputField( @@ -322,14 +322,14 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard): ) -> list[XLabsControlNetExtension | InstantXControlNetExtension]: # Normalize the controlnet input to list[ControlField]. controlnets: list[FluxControlNetField] - if self.controlnet is None: + if self.control is None: controlnets = [] - elif isinstance(self.controlnet, FluxControlNetField): - controlnets = [self.controlnet] - elif isinstance(self.controlnet, list): - controlnets = self.controlnet + elif isinstance(self.control, FluxControlNetField): + controlnets = [self.control] + elif isinstance(self.control, list): + controlnets = self.control else: - raise ValueError(f"Unsupported controlnet type: {type(self.controlnet)}") + raise ValueError(f"Unsupported controlnet type: {type(self.control)}") # TODO(ryand): Add a field to the model config so that we can distinguish between XLabs and InstantX ControlNets # before loading the models. Then make sure that all VAE encoding is done before loading the ControlNets to @@ -337,7 +337,7 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard): controlnet_extensions: list[XLabsControlNetExtension | InstantXControlNetExtension] = [] for controlnet in controlnets: - model = exit_stack.enter_context(context.models.load(controlnet.controlnet_model)) + model = exit_stack.enter_context(context.models.load(controlnet.control_model)) image = context.images.get_pil(controlnet.image.image_name) if isinstance(model, XLabsControlNetFlux): diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts index 908f912358..aacea5de8b 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/addControlAdapters.ts @@ -119,12 +119,7 @@ const addControlNetToGraph = ( control_weight: weight, image: { image_name }, }); - - if (controlNet.type === 'flux_controlnet') { - g.addEdge(controlNet, 'controlnet', collector, 'item'); - } else { - g.addEdge(controlNet, 'control', collector, 'item'); - } + g.addEdge(controlNet, 'control', collector, 'item'); }; const addT2IAdapterToGraph = ( diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts index d58f38b8c4..cc8a7347fe 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts @@ -95,6 +95,7 @@ export const buildFLUXGraph = async ( > = l2i; g.addEdge(modelLoader, 'transformer', noise, 'transformer'); + g.addEdge(modelLoader, 'vae', noise, 'controlnet_vae'); g.addEdge(modelLoader, 'vae', l2i, 'vae'); g.addEdge(modelLoader, 'clip', posCond, 'clip'); @@ -192,7 +193,7 @@ export const buildFLUXGraph = async ( modelConfig.base ); if (controlNetResult.addedControlNets > 0) { - g.addEdge(controlNetCollector, 'collection', noise, 'controlnet'); + g.addEdge(controlNetCollector, 'collection', noise, 'control'); } else { g.deleteNode(controlNetCollector.id); } diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 36d4b525b1..f41fff7581 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -4622,7 +4622,7 @@ export type components = { /** * Fp32 * @description Whether or not to use full float32 precision - * @default true + * @default false */ fp32?: boolean; /** @@ -4705,7 +4705,7 @@ export type components = { /** * Fp32 * @description Whether or not to use full float32 precision - * @default true + * @default false */ fp32?: boolean; /** @@ -6297,7 +6297,7 @@ export type components = { /** @description The control image */ image: components["schemas"]["ImageField"]; /** @description The ControlNet model to use */ - controlnet_model: components["schemas"]["ModelIdentifierField"]; + control_model: components["schemas"]["ModelIdentifierField"]; /** * Control Weight * @description The weight given to the ControlNet @@ -6323,12 +6323,6 @@ export type components = { * @enum {string} */ resize_mode?: "just_resize" | "crop_resize" | "fill_resize" | "just_resize_simple"; - /** - * Instantx Control Mode - * @description The control mode for InstantX ControlNet union models. Ignored for other ControlNet models. - * @default 0 - */ - instantx_control_mode?: number; }; /** * FLUX ControlNet @@ -6361,7 +6355,7 @@ export type components = { * @description ControlNet model to load * @default null */ - controlnet_model?: components["schemas"]["ModelIdentifierField"]; + control_model?: components["schemas"]["ModelIdentifierField"]; /** * Control Weight * @description The weight given to the ControlNet @@ -6387,12 +6381,6 @@ export type components = { * @enum {string} */ resize_mode?: "just_resize" | "crop_resize" | "fill_resize" | "just_resize_simple"; - /** - * Instantx Control Mode - * @description The control mode for InstantX ControlNet union models. Ignored for other ControlNet models. - * @default 0 - */ - instantx_control_mode?: number; /** * type * @default flux_controlnet @@ -6407,7 +6395,7 @@ export type components = { */ FluxControlNetOutput: { /** @description ControlNet(s) to apply */ - controlnet: components["schemas"]["FluxControlNetField"]; + control: components["schemas"]["FluxControlNetField"]; /** * type * @default flux_controlnet_output @@ -6512,11 +6500,11 @@ export type components = { */ seed?: number; /** - * Controlnet + * Control * @description ControlNet models. * @default null */ - controlnet?: components["schemas"]["FluxControlNetField"] | components["schemas"]["FluxControlNetField"][] | null; + control?: components["schemas"]["FluxControlNetField"] | components["schemas"]["FluxControlNetField"][] | null; /** * @description VAE * @default null @@ -8863,7 +8851,7 @@ export type components = { /** * Fp32 * @description Whether or not to use full float32 precision - * @default true + * @default false */ fp32?: boolean; /** @@ -10146,7 +10134,7 @@ export type components = { /** * Fp32 * @description Whether or not to use full float32 precision - * @default true + * @default false */ fp32?: boolean; /**