From 00f36cb491b3f67a7bfa4a1c604c01b23d4e162d Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Wed, 8 May 2024 15:21:51 +1000 Subject: [PATCH] tidy(ui): clean up control layers graph builder --- .../util/graph/addControlLayersToGraph.ts | 768 +++++++++--------- 1 file changed, 370 insertions(+), 398 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addControlLayersToGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addControlLayersToGraph.ts index de3456f371..5abbce028b 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addControlLayersToGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addControlLayersToGraph.ts @@ -8,14 +8,12 @@ import { rgLayerMaskImageUploaded, } from 'features/controlLayers/store/controlLayersSlice'; import type { InitialImageLayer, Layer, RegionalGuidanceLayer } from 'features/controlLayers/store/types'; -import { - type ControlNetConfigV2, - type ImageWithDims, - type IPAdapterConfigV2, - isControlNetConfigV2, - isT2IAdapterConfigV2, - type ProcessorConfig, - type T2IAdapterConfigV2, +import type { + ControlNetConfigV2, + ImageWithDims, + IPAdapterConfigV2, + ProcessorConfig, + T2IAdapterConfigV2, } from 'features/controlLayers/util/controlAdapters'; import { getRegionalPromptLayerBlobs } from 'features/controlLayers/util/getLayerBlobs'; import type { ImageField } from 'features/nodes/types/common'; @@ -40,6 +38,7 @@ import { upsertMetadata } from 'features/nodes/util/graph/metadata'; import { size } from 'lodash-es'; import { getImageDTO, imagesApi } from 'services/api/endpoints/images'; import type { + BaseModelType, CollectInvocation, ControlNetInvocation, Edge, @@ -53,324 +52,6 @@ import type { } from 'services/api/types'; import { assert } from 'tsafe'; -const buildControlImage = ( - image: ImageWithDims | null, - processedImage: ImageWithDims | null, - processorConfig: ProcessorConfig | null -): ImageField => { - if (processedImage && processorConfig) { - // We've processed the image in the app - use it for the control image. - return { - image_name: processedImage.name, - }; - } else if (image) { - // No processor selected, and we have an image - the user provided a processed image, use it for the control image. - return { - image_name: image.name, - }; - } - assert(false, 'Attempted to add unprocessed control image'); -}; - -const addControlNetCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { - if (graph.nodes[CONTROL_NET_COLLECT]) { - // You see, we've already got one! - return; - } - // Add the ControlNet collector - const controlNetIterateNode: CollectInvocation = { - id: CONTROL_NET_COLLECT, - type: 'collect', - is_intermediate: true, - }; - graph.nodes[CONTROL_NET_COLLECT] = controlNetIterateNode; - graph.edges.push({ - source: { node_id: CONTROL_NET_COLLECT, field: 'collection' }, - destination: { - node_id: denoiseNodeId, - field: 'control', - }, - }); -}; - -const addGlobalControlNetsToGraph = async ( - controlNets: ControlNetConfigV2[], - graph: NonNullableGraph, - denoiseNodeId: string -) => { - if (controlNets.length === 0) { - return; - } - addControlNetCollectorSafe(graph, denoiseNodeId); - - for (const controlNet of controlNets) { - if (!controlNet.model) { - return; - } - const { id, beginEndStepPct, controlMode, image, model, processedImage, processorConfig, weight } = controlNet; - - const controlNetNode: ControlNetInvocation = { - id: `control_net_${id}`, - type: 'controlnet', - is_intermediate: true, - begin_step_percent: beginEndStepPct[0], - end_step_percent: beginEndStepPct[1], - control_mode: controlMode, - resize_mode: 'just_resize', - control_model: model, - control_weight: weight, - image: buildControlImage(image, processedImage, processorConfig), - }; - - graph.nodes[controlNetNode.id] = controlNetNode; - - graph.edges.push({ - source: { node_id: controlNetNode.id, field: 'control' }, - destination: { - node_id: CONTROL_NET_COLLECT, - field: 'item', - }, - }); - } -}; - -const addT2IAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { - if (graph.nodes[T2I_ADAPTER_COLLECT]) { - // You see, we've already got one! - return; - } - // Even though denoise_latents' t2i adapter input is collection or scalar, keep it simple and always use a collect - const t2iAdapterCollectNode: CollectInvocation = { - id: T2I_ADAPTER_COLLECT, - type: 'collect', - is_intermediate: true, - }; - graph.nodes[T2I_ADAPTER_COLLECT] = t2iAdapterCollectNode; - graph.edges.push({ - source: { node_id: T2I_ADAPTER_COLLECT, field: 'collection' }, - destination: { - node_id: denoiseNodeId, - field: 't2i_adapter', - }, - }); -}; - -const addGlobalT2IAdaptersToGraph = async ( - t2iAdapters: T2IAdapterConfigV2[], - graph: NonNullableGraph, - denoiseNodeId: string -) => { - if (t2iAdapters.length === 0) { - return; - } - addT2IAdapterCollectorSafe(graph, denoiseNodeId); - - for (const t2iAdapter of t2iAdapters) { - if (!t2iAdapter.model) { - return; - } - const { id, beginEndStepPct, image, model, processedImage, processorConfig, weight } = t2iAdapter; - - const t2iAdapterNode: T2IAdapterInvocation = { - id: `t2i_adapter_${id}`, - type: 't2i_adapter', - is_intermediate: true, - begin_step_percent: beginEndStepPct[0], - end_step_percent: beginEndStepPct[1], - resize_mode: 'just_resize', - t2i_adapter_model: model, - weight: weight, - image: buildControlImage(image, processedImage, processorConfig), - }; - - graph.nodes[t2iAdapterNode.id] = t2iAdapterNode; - - graph.edges.push({ - source: { node_id: t2iAdapterNode.id, field: 't2i_adapter' }, - destination: { - node_id: T2I_ADAPTER_COLLECT, - field: 'item', - }, - }); - } -}; - -const addIPAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { - if (graph.nodes[IP_ADAPTER_COLLECT]) { - // You see, we've already got one! - return; - } - - const ipAdapterCollectNode: CollectInvocation = { - id: IP_ADAPTER_COLLECT, - type: 'collect', - is_intermediate: true, - }; - graph.nodes[IP_ADAPTER_COLLECT] = ipAdapterCollectNode; - graph.edges.push({ - source: { node_id: IP_ADAPTER_COLLECT, field: 'collection' }, - destination: { - node_id: denoiseNodeId, - field: 'ip_adapter', - }, - }); -}; - -const addGlobalIPAdaptersToGraph = async ( - ipAdapters: IPAdapterConfigV2[], - graph: NonNullableGraph, - denoiseNodeId: string -) => { - if (ipAdapters.length === 0) { - return; - } - addIPAdapterCollectorSafe(graph, denoiseNodeId); - - for (const ipAdapter of ipAdapters) { - const { id, weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter; - assert(image, 'IP Adapter image is required'); - assert(model, 'IP Adapter model is required'); - - const ipAdapterNode: IPAdapterInvocation = { - id: `ip_adapter_${id}`, - type: 'ip_adapter', - is_intermediate: true, - weight, - method, - ip_adapter_model: model, - clip_vision_model: clipVisionModel, - begin_step_percent: beginEndStepPct[0], - end_step_percent: beginEndStepPct[1], - image: { - image_name: image.name, - }, - }; - - graph.nodes[ipAdapterNode.id] = ipAdapterNode; - - graph.edges.push({ - source: { node_id: ipAdapterNode.id, field: 'ip_adapter' }, - destination: { - node_id: IP_ADAPTER_COLLECT, - field: 'item', - }, - }); - } -}; - -const addInitialImageLayerToGraph = ( - state: RootState, - graph: NonNullableGraph, - denoiseNodeId: string, - layer: InitialImageLayer -) => { - const { vaePrecision, model } = state.generation; - const { refinerModel, refinerStart } = state.sdxl; - const { width, height } = state.controlLayers.present.size; - assert(layer.isEnabled, 'Initial image layer is not enabled'); - assert(layer.image, 'Initial image layer has no image'); - - const isSDXL = model?.base === 'sdxl'; - const useRefinerStartEnd = isSDXL && Boolean(refinerModel); - - const denoiseNode = graph.nodes[denoiseNodeId]; - assert(denoiseNode?.type === 'denoise_latents', `Missing denoise node or incorrect type: ${denoiseNode?.type}`); - - const { denoisingStrength } = layer; - denoiseNode.denoising_start = useRefinerStartEnd - ? Math.min(refinerStart, 1 - denoisingStrength) - : 1 - denoisingStrength; - denoiseNode.denoising_end = useRefinerStartEnd ? refinerStart : 1; - - // We conditionally hook the image in depending on if a resize is needed - const i2lNode: ImageToLatentsInvocation = { - type: 'i2l', - id: IMAGE_TO_LATENTS, - is_intermediate: true, - use_cache: true, - fp32: vaePrecision === 'fp32', - }; - - graph.nodes[i2lNode.id] = i2lNode; - graph.edges.push({ - source: { - node_id: IMAGE_TO_LATENTS, - field: 'latents', - }, - destination: { - node_id: denoiseNode.id, - field: 'latents', - }, - }); - - if (layer.image.width !== width || layer.image.height !== height) { - // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` - - // Create a resize node, explicitly setting its image - const resizeNode: ImageResizeInvocation = { - id: RESIZE, - type: 'img_resize', - image: { - image_name: layer.image.name, - }, - is_intermediate: true, - width, - height, - }; - - graph.nodes[RESIZE] = resizeNode; - - // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` - graph.edges.push({ - source: { node_id: RESIZE, field: 'image' }, - destination: { - node_id: IMAGE_TO_LATENTS, - field: 'image', - }, - }); - - // The `RESIZE` node also passes its width and height to `NOISE` - graph.edges.push({ - source: { node_id: RESIZE, field: 'width' }, - destination: { - node_id: NOISE, - field: 'width', - }, - }); - - graph.edges.push({ - source: { node_id: RESIZE, field: 'height' }, - destination: { - node_id: NOISE, - field: 'height', - }, - }); - } else { - // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly - i2lNode.image = { - image_name: layer.image.name, - }; - - // Pass the image's dimensions to the `NOISE` node - graph.edges.push({ - source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, - destination: { - node_id: NOISE, - field: 'width', - }, - }); - graph.edges.push({ - source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, - destination: { - node_id: NOISE, - field: 'height', - }, - }); - } - - upsertMetadata(graph, { generation_mode: isSDXL ? 'sdxl_img2img' : 'img2img' }); -}; - export const addControlLayersToGraph = async ( state: RootState, graph: NonNullableGraph, @@ -380,74 +61,24 @@ export const addControlLayersToGraph = async ( assert(mainModel, 'Missing main model when building graph'); const isSDXL = mainModel.base === 'sdxl'; - const validLayers: Layer[] = []; + // Filter out layers with incompatible base model, missing control image + const validLayers = state.controlLayers.present.layers.filter((l) => isValidLayer(l, mainModel.base)); - // Add global control adapters - const validControlAdapterLayers = state.controlLayers.present.layers - // Must be a Control Adapter layer - .filter(isControlAdapterLayer) - // Must be enabled - .filter((l) => l.isEnabled) - .filter((l) => { - const ca = l.controlAdapter; - // Must be have a model that matches the current base and must have a control image - const hasModel = Boolean(ca.model); - const modelMatchesBase = ca.model?.base === mainModel.base; - const hasControlImage = ca.image || (ca.processedImage && ca.processorConfig); - return hasModel && modelMatchesBase && hasControlImage; - }); - const validControlNets = validControlAdapterLayers.map((l) => l.controlAdapter).filter(isControlNetConfigV2); - addGlobalControlNetsToGraph(validControlNets, graph, denoiseNodeId); - - const validT2IAdapters = validControlAdapterLayers.map((l) => l.controlAdapter).filter(isT2IAdapterConfigV2); - addGlobalT2IAdaptersToGraph(validT2IAdapters, graph, denoiseNodeId); - - validLayers.push(...validControlAdapterLayers); - - const validIPAdapterLayers = state.controlLayers.present.layers - // Must be an IP Adapter layer - .filter(isIPAdapterLayer) - // Must be enabled - .filter((l) => l.isEnabled) - // We want the IP Adapters themselves - .filter((l) => { - const ipa = l.ipAdapter; - const hasModel = Boolean(ipa.model); - const modelMatchesBase = ipa.model?.base === mainModel.base; - const hasImage = Boolean(ipa.image); - return hasModel && modelMatchesBase && hasImage; - }); - const validIPAdapters = validIPAdapterLayers.map((l) => l.ipAdapter); - addGlobalIPAdaptersToGraph(validIPAdapters, graph, denoiseNodeId); - validLayers.push(...validIPAdapterLayers); - - const initialImageLayer = state.controlLayers.present.layers.filter(isInitialImageLayer).find((l) => { - if (!l.isEnabled) { - return false; - } - if (!l.image) { - return false; - } - return true; - }); - if (initialImageLayer) { - addInitialImageLayerToGraph(state, graph, denoiseNodeId, initialImageLayer); - validLayers.push(initialImageLayer); + const validControlAdapters = validLayers.filter(isControlAdapterLayer).map((l) => l.controlAdapter); + for (const ca of validControlAdapters) { + addGlobalControlAdapterToGraph(ca, graph, denoiseNodeId); } - const validRGLayers = state.controlLayers.present.layers - // Only RG layers are get masks - .filter(isRegionalGuidanceLayer) - // Only visible layers are rendered on the canvas - .filter((l) => l.isEnabled) - // Only layers with prompts get added to the graph - .filter((l) => { - const hasTextPrompt = Boolean(l.positivePrompt || l.negativePrompt); - const hasIPAdapter = l.ipAdapters.filter((ipa) => ipa.image).length > 0; - return hasTextPrompt || hasIPAdapter; - }); - validLayers.push(...validRGLayers); + const validIPAdapters = validLayers.filter(isIPAdapterLayer).map((l) => l.ipAdapter); + for (const ipAdapter of validIPAdapters) { + addGlobalIPAdapterToGraph(ipAdapter, graph, denoiseNodeId); + } + const initialImageLayers = validLayers.filter(isInitialImageLayer); + assert(initialImageLayers.length <= 1, 'Only one initial image layer allowed'); + if (initialImageLayers[0]) { + addInitialImageLayerToGraph(state, graph, denoiseNodeId, initialImageLayers[0]); + } // TODO: We should probably just use conditioning collectors by default, and skip all this fanagling with re-routing // the existing conditioning nodes. @@ -510,6 +141,7 @@ export const addControlLayersToGraph = async ( }, }); + const validRGLayers = validLayers.filter(isRegionalGuidanceLayer); const layerIds = validRGLayers.map((l) => l.id); const blobs = await getRegionalPromptLayerBlobs(layerIds); assert(size(blobs) === size(layerIds), 'Mismatch between layer IDs and blobs'); @@ -665,15 +297,11 @@ export const addControlLayersToGraph = async ( } } - // TODO(psyche): For some reason, I have to explicitly annotate regionalIPAdapters here. Not sure why. - const regionalIPAdapters: IPAdapterConfigV2[] = layer.ipAdapters.filter((ipAdapter) => { - const hasModel = Boolean(ipAdapter.model); - const modelMatchesBase = ipAdapter.model?.base === mainModel.base; - const hasControlImage = Boolean(ipAdapter.image); - return hasModel && modelMatchesBase && hasControlImage; - }); + const validRegionalIPAdapters: IPAdapterConfigV2[] = layer.ipAdapters.filter((ipa) => + isValidIPAdapter(ipa, mainModel.base) + ); - for (const ipAdapter of regionalIPAdapters) { + for (const ipAdapter of validRegionalIPAdapters) { addIPAdapterCollectorSafe(graph, denoiseNodeId); const { id, weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter; assert(model, 'IP Adapter model is required'); @@ -735,3 +363,347 @@ const getMaskImage = async (layer: RegionalGuidanceLayer, blob: Blob): Promise { + if (processedImage && processorConfig) { + // We've processed the image in the app - use it for the control image. + return { + image_name: processedImage.name, + }; + } else if (image) { + // No processor selected, and we have an image - the user provided a processed image, use it for the control image. + return { + image_name: image.name, + }; + } + assert(false, 'Attempted to add unprocessed control image'); +}; + +const addGlobalControlAdapterToGraph = ( + controlAdapter: ControlNetConfigV2 | T2IAdapterConfigV2, + graph: NonNullableGraph, + denoiseNodeId: string +) => { + if (controlAdapter.type === 'controlnet') { + addGlobalControlNetToGraph(controlAdapter, graph, denoiseNodeId); + } + if (controlAdapter.type === 't2i_adapter') { + addGlobalT2IAdapterToGraph(controlAdapter, graph, denoiseNodeId); + } +}; + +const addControlNetCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { + if (graph.nodes[CONTROL_NET_COLLECT]) { + // You see, we've already got one! + return; + } + // Add the ControlNet collector + const controlNetIterateNode: CollectInvocation = { + id: CONTROL_NET_COLLECT, + type: 'collect', + is_intermediate: true, + }; + graph.nodes[CONTROL_NET_COLLECT] = controlNetIterateNode; + graph.edges.push({ + source: { node_id: CONTROL_NET_COLLECT, field: 'collection' }, + destination: { + node_id: denoiseNodeId, + field: 'control', + }, + }); +}; + +const addGlobalControlNetToGraph = (controlNet: ControlNetConfigV2, graph: NonNullableGraph, denoiseNodeId: string) => { + const { id, beginEndStepPct, controlMode, image, model, processedImage, processorConfig, weight } = controlNet; + assert(model, 'ControlNet model is required'); + const controlImage = buildControlImage(image, processedImage, processorConfig); + addControlNetCollectorSafe(graph, denoiseNodeId); + + const controlNetNode: ControlNetInvocation = { + id: `control_net_${id}`, + type: 'controlnet', + is_intermediate: true, + begin_step_percent: beginEndStepPct[0], + end_step_percent: beginEndStepPct[1], + control_mode: controlMode, + resize_mode: 'just_resize', + control_model: model, + control_weight: weight, + image: controlImage, + }; + + graph.nodes[controlNetNode.id] = controlNetNode; + + graph.edges.push({ + source: { node_id: controlNetNode.id, field: 'control' }, + destination: { + node_id: CONTROL_NET_COLLECT, + field: 'item', + }, + }); +}; + +const addT2IAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { + if (graph.nodes[T2I_ADAPTER_COLLECT]) { + // You see, we've already got one! + return; + } + // Even though denoise_latents' t2i adapter input is collection or scalar, keep it simple and always use a collect + const t2iAdapterCollectNode: CollectInvocation = { + id: T2I_ADAPTER_COLLECT, + type: 'collect', + is_intermediate: true, + }; + graph.nodes[T2I_ADAPTER_COLLECT] = t2iAdapterCollectNode; + graph.edges.push({ + source: { node_id: T2I_ADAPTER_COLLECT, field: 'collection' }, + destination: { + node_id: denoiseNodeId, + field: 't2i_adapter', + }, + }); +}; + +const addGlobalT2IAdapterToGraph = (t2iAdapter: T2IAdapterConfigV2, graph: NonNullableGraph, denoiseNodeId: string) => { + const { id, beginEndStepPct, image, model, processedImage, processorConfig, weight } = t2iAdapter; + assert(model, 'T2I Adapter model is required'); + const controlImage = buildControlImage(image, processedImage, processorConfig); + addT2IAdapterCollectorSafe(graph, denoiseNodeId); + + const t2iAdapterNode: T2IAdapterInvocation = { + id: `t2i_adapter_${id}`, + type: 't2i_adapter', + is_intermediate: true, + begin_step_percent: beginEndStepPct[0], + end_step_percent: beginEndStepPct[1], + resize_mode: 'just_resize', + t2i_adapter_model: model, + weight: weight, + image: controlImage, + }; + + graph.nodes[t2iAdapterNode.id] = t2iAdapterNode; + + graph.edges.push({ + source: { node_id: t2iAdapterNode.id, field: 't2i_adapter' }, + destination: { + node_id: T2I_ADAPTER_COLLECT, + field: 'item', + }, + }); +}; + +const addIPAdapterCollectorSafe = (graph: NonNullableGraph, denoiseNodeId: string) => { + if (graph.nodes[IP_ADAPTER_COLLECT]) { + // You see, we've already got one! + return; + } + + const ipAdapterCollectNode: CollectInvocation = { + id: IP_ADAPTER_COLLECT, + type: 'collect', + is_intermediate: true, + }; + graph.nodes[IP_ADAPTER_COLLECT] = ipAdapterCollectNode; + graph.edges.push({ + source: { node_id: IP_ADAPTER_COLLECT, field: 'collection' }, + destination: { + node_id: denoiseNodeId, + field: 'ip_adapter', + }, + }); +}; + +const addGlobalIPAdapterToGraph = (ipAdapter: IPAdapterConfigV2, graph: NonNullableGraph, denoiseNodeId: string) => { + addIPAdapterCollectorSafe(graph, denoiseNodeId); + const { id, weight, model, clipVisionModel, method, beginEndStepPct, image } = ipAdapter; + assert(image, 'IP Adapter image is required'); + assert(model, 'IP Adapter model is required'); + + const ipAdapterNode: IPAdapterInvocation = { + id: `ip_adapter_${id}`, + type: 'ip_adapter', + is_intermediate: true, + weight, + method, + ip_adapter_model: model, + clip_vision_model: clipVisionModel, + begin_step_percent: beginEndStepPct[0], + end_step_percent: beginEndStepPct[1], + image: { + image_name: image.name, + }, + }; + + graph.nodes[ipAdapterNode.id] = ipAdapterNode; + + graph.edges.push({ + source: { node_id: ipAdapterNode.id, field: 'ip_adapter' }, + destination: { + node_id: IP_ADAPTER_COLLECT, + field: 'item', + }, + }); +}; + +const addInitialImageLayerToGraph = ( + state: RootState, + graph: NonNullableGraph, + denoiseNodeId: string, + layer: InitialImageLayer +) => { + const { vaePrecision, model } = state.generation; + const { refinerModel, refinerStart } = state.sdxl; + const { width, height } = state.controlLayers.present.size; + assert(layer.isEnabled, 'Initial image layer is not enabled'); + assert(layer.image, 'Initial image layer has no image'); + + const isSDXL = model?.base === 'sdxl'; + const useRefinerStartEnd = isSDXL && Boolean(refinerModel); + + const denoiseNode = graph.nodes[denoiseNodeId]; + assert(denoiseNode?.type === 'denoise_latents', `Missing denoise node or incorrect type: ${denoiseNode?.type}`); + + const { denoisingStrength } = layer; + denoiseNode.denoising_start = useRefinerStartEnd + ? Math.min(refinerStart, 1 - denoisingStrength) + : 1 - denoisingStrength; + denoiseNode.denoising_end = useRefinerStartEnd ? refinerStart : 1; + + const i2lNode: ImageToLatentsInvocation = { + type: 'i2l', + id: IMAGE_TO_LATENTS, + is_intermediate: true, + use_cache: true, + fp32: vaePrecision === 'fp32', + }; + + graph.nodes[i2lNode.id] = i2lNode; + graph.edges.push({ + source: { + node_id: IMAGE_TO_LATENTS, + field: 'latents', + }, + destination: { + node_id: denoiseNode.id, + field: 'latents', + }, + }); + + if (layer.image.width !== width || layer.image.height !== height) { + // The init image needs to be resized to the specified width and height before being passed to `IMAGE_TO_LATENTS` + + // Create a resize node, explicitly setting its image + const resizeNode: ImageResizeInvocation = { + id: RESIZE, + type: 'img_resize', + image: { + image_name: layer.image.name, + }, + is_intermediate: true, + width, + height, + }; + + graph.nodes[RESIZE] = resizeNode; + + // The `RESIZE` node then passes its image to `IMAGE_TO_LATENTS` + graph.edges.push({ + source: { node_id: RESIZE, field: 'image' }, + destination: { + node_id: IMAGE_TO_LATENTS, + field: 'image', + }, + }); + + // The `RESIZE` node also passes its width and height to `NOISE` + graph.edges.push({ + source: { node_id: RESIZE, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + + graph.edges.push({ + source: { node_id: RESIZE, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } else { + // We are not resizing, so we need to set the image on the `IMAGE_TO_LATENTS` node explicitly + i2lNode.image = { + image_name: layer.image.name, + }; + + // Pass the image's dimensions to the `NOISE` node + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'width' }, + destination: { + node_id: NOISE, + field: 'width', + }, + }); + graph.edges.push({ + source: { node_id: IMAGE_TO_LATENTS, field: 'height' }, + destination: { + node_id: NOISE, + field: 'height', + }, + }); + } + + upsertMetadata(graph, { generation_mode: isSDXL ? 'sdxl_img2img' : 'img2img' }); +}; + +const isValidControlAdapter = (ca: ControlNetConfigV2 | T2IAdapterConfigV2, base: BaseModelType): boolean => { + // Must be have a model that matches the current base and must have a control image + const hasModel = Boolean(ca.model); + const modelMatchesBase = ca.model?.base === base; + const hasControlImage = Boolean(ca.image || (ca.processedImage && ca.processorConfig)); + return hasModel && modelMatchesBase && hasControlImage; +}; + +const isValidIPAdapter = (ipa: IPAdapterConfigV2, base: BaseModelType): boolean => { + // Must be have a model that matches the current base and must have a control image + const hasModel = Boolean(ipa.model); + const modelMatchesBase = ipa.model?.base === base; + const hasImage = Boolean(ipa.image); + return hasModel && modelMatchesBase && hasImage; +}; + +const isValidLayer = (layer: Layer, base: BaseModelType) => { + if (isControlAdapterLayer(layer)) { + if (!layer.isEnabled) { + return false; + } + return isValidControlAdapter(layer.controlAdapter, base); + } + if (isIPAdapterLayer(layer)) { + if (!layer.isEnabled) { + return false; + } + return isValidIPAdapter(layer.ipAdapter, base); + } + if (isInitialImageLayer(layer)) { + if (!layer.isEnabled) { + return false; + } + if (!layer.image) { + return false; + } + return true; + } + if (isRegionalGuidanceLayer(layer)) { + const hasTextPrompt = Boolean(layer.positivePrompt || layer.negativePrompt); + const hasIPAdapter = layer.ipAdapters.filter((ipa) => isValidIPAdapter(ipa, base)).length > 0; + return hasTextPrompt || hasIPAdapter; + } + return false; +};