diff --git a/REFIMAGE_MODEL_SWITCHING_IMPLEMENTATION.md b/REFIMAGE_MODEL_SWITCHING_IMPLEMENTATION.md new file mode 100644 index 0000000000..72fe4af31f --- /dev/null +++ b/REFIMAGE_MODEL_SWITCHING_IMPLEMENTATION.md @@ -0,0 +1,114 @@ +# RefImage/IPAdapter Model Switching Implementation + +This document describes the implementation of automatic RefImage/IPAdapter model switching when the base model architecture changes in InvokeAI. + +## Overview + +When a user changes the base model architecture (e.g., from SDXL to FLUX), the system now automatically switches RefImage/IPAdapter models to the first compatible model available for the new architecture. If no compatible models are available, the RefImage/IPAdapter models are cleared. + +## Implementation Details + +### Files Modified + +1. **`invokeai/frontend/web/src/services/api/types.ts`** + - Added `isApiModelConfig` type guard function + - This function identifies API-based models (ChatGPT-4o, Imagen3, Imagen4, FLUX-Kontext API) + +2. **`invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelSelected.ts`** + - Enhanced the `addModelSelectedListener` function to handle RefImage/IPAdapter model switching + - Added logic to automatically switch both global and regional reference image models + +### Key Components + +#### Type Guard Function +```typescript +export const isApiModelConfig = (config: AnyModelConfig): config is ApiModelConfig => { + return ( + isChatGPT4oModelConfig(config) || + isImagen3ModelConfig(config) || + isImagen4ModelConfig(config) || + isFluxKontextApiModelConfig(config) + ); +}; +``` + +#### Model Selection Logic +The implementation includes a selector function that identifies all available global reference image models: + +```typescript +const selectGlobalReferenceImageModels = (state: RootState): AnyModelConfig[] => { + const allModels = selectIPAdapterModels(state); + return allModels.filter((model: AnyModelConfig) => + isIPAdapterModelConfig(model) || + isFluxReduxModelConfig(model) || + isChatGPT4oModelConfig(model) || + isFluxKontextApiModelConfig(model) || + isFluxKontextModelConfig(model) + ); +}; +``` + +### Functionality + +#### What Happens When Base Model Changes + +1. **Model Compatibility Check**: The system checks if existing RefImage/IPAdapter models are compatible with the new base model architecture + +2. **Automatic Switching**: If incompatible models are found: + - Finds the first compatible model for the new architecture + - Automatically switches to that model + - Logs the model switch for debugging + +3. **Fallback Behavior**: If no compatible models are available: + - Clears the incompatible model + - Increments the `modelsCleared` counter + - Shows a warning toast to the user + +#### Areas Affected + +1. **Global Reference Images**: Standalone reference images managed in the `refImages` slice +2. **Regional Guidance Reference Images**: Reference images associated with regional guidance entities on the canvas + +### User Experience + +- **Seamless Transition**: Users can switch between model architectures without manually reconfiguring RefImage/IPAdapter models +- **Intelligent Defaults**: The system automatically selects the first available compatible model +- **Clear Feedback**: Users receive notifications when models are cleared due to incompatibility +- **Preserved Configuration**: When switching to a compatible model, other settings (weight, method, etc.) are preserved when possible + +### Technical Benefits + +1. **Reduced Manual Work**: Users don't need to manually reconfigure RefImage/IPAdapter models when switching architectures +2. **Prevents Errors**: Automatically prevents incompatible model configurations that would cause generation failures +3. **Maintains Workflow**: Users can focus on creative work rather than technical model management +4. **Consistent Behavior**: The same logic applies to both global and regional reference images + +### Integration Points + +The implementation integrates with several existing systems: + +- **Model Management**: Uses existing model selection and compatibility checking systems +- **State Management**: Works with Redux slices for canvas, reference images, and parameters +- **User Feedback**: Integrates with the existing toast notification system +- **Logging**: Provides detailed debug logging for troubleshooting + +### Error Handling + +- Graceful fallback when no compatible models are available +- Comprehensive logging for debugging model switching decisions +- User notifications through toast messages +- Maintains system stability even when model switching fails + +## Usage + +This feature works automatically without any user configuration required. When users: + +1. Switch from one base model architecture to another (e.g., SDXL → FLUX) +2. Have existing RefImage/IPAdapter models configured +3. The system will automatically: + - Check compatibility of existing models + - Switch to the first compatible model for the new architecture + - Or clear the model if none are compatible + - Notify the user of any changes made + +This implementation ensures a smooth user experience when working with different model architectures while maintaining the integrity of the generation pipeline. \ No newline at end of file diff --git a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelSelected.ts b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelSelected.ts index 4be02fcc30..f4d68eb981 100644 --- a/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelSelected.ts +++ b/invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelSelected.ts @@ -4,14 +4,56 @@ import { bboxSyncedToOptimalDimension } from 'features/controlLayers/store/canva import { selectIsStaging } from 'features/controlLayers/store/canvasStagingAreaSlice'; import { loraDeleted } from 'features/controlLayers/store/lorasSlice'; import { modelChanged, syncedToOptimalDimension, vaeSelected } from 'features/controlLayers/store/paramsSlice'; -import { selectBboxModelBase } from 'features/controlLayers/store/selectors'; +import { + refImageModelChanged, + selectReferenceImageEntities +} from 'features/controlLayers/store/refImagesSlice'; +import { selectBboxModelBase, selectAllEntities } from 'features/controlLayers/store/selectors'; +import { + rgRefImageModelChanged +} from 'features/controlLayers/store/canvasSlice'; +import { + getEntityIdentifier, + isRegionalGuidanceEntityIdentifier +} from 'features/controlLayers/store/types'; +import type { + CanvasEntityState, + RefImageState +} from 'features/controlLayers/store/types'; import { modelSelected } from 'features/parameters/store/actions'; import { zParameterModel } from 'features/parameters/types/parameterSchemas'; import { toast } from 'features/toast/toast'; +import { + selectIPAdapterModels +} from 'services/api/hooks/modelsByType'; +import type { + AnyModelConfig +} from 'services/api/types'; +import { + isIPAdapterModelConfig, + isFluxReduxModelConfig, + isChatGPT4oModelConfig, + isFluxKontextApiModelConfig, + isFluxKontextModelConfig +} from 'services/api/types'; +import type { RootState } from 'app/store/store'; import { t } from 'i18next'; const log = logger('models'); +// Selector for global reference image models +const selectGlobalReferenceImageModels = (state: RootState): AnyModelConfig[] => { + const allModels = selectIPAdapterModels(state); + // Add other model types that can be used as reference images + return allModels.filter((model: AnyModelConfig) => + isIPAdapterModelConfig(model) || + isFluxReduxModelConfig(model) || + isChatGPT4oModelConfig(model) || + isFluxKontextApiModelConfig(model) || + isFluxKontextModelConfig(model) + ); +}; + export const addModelSelectedListener = (startAppListening: AppStartListening) => { startAppListening({ actionCreator: modelSelected, @@ -34,7 +76,7 @@ export const addModelSelectedListener = (startAppListening: AppStartListening) = let modelsCleared = 0; // handle incompatible loras - state.loras.loras.forEach((lora) => { + state.loras.loras.forEach((lora: any) => { if (lora.model.base !== newBaseModel) { dispatch(loraDeleted({ id: lora.id })); modelsCleared += 1; @@ -58,6 +100,61 @@ export const addModelSelectedListener = (startAppListening: AppStartListening) = // } // }); + // Handle incompatible reference image models - switch to first compatible model + const availableRefImageModels = selectGlobalReferenceImageModels(state).filter((model: AnyModelConfig) => model.base === newBaseModel); + const firstCompatibleModel = availableRefImageModels[0] || null; + + // Handle global reference images + const refImageEntities = selectReferenceImageEntities(state); + refImageEntities.forEach((entity: RefImageState) => { + if (entity.config.model && entity.config.model.base !== newBaseModel) { + dispatch(refImageModelChanged({ + id: entity.id, + modelConfig: firstCompatibleModel + })); + if (firstCompatibleModel) { + log.debug( + { oldModel: entity.config.model, newModel: firstCompatibleModel }, + 'Switched global reference image model to compatible model' + ); + } else { + log.debug( + { oldModel: entity.config.model }, + 'Cleared global reference image model - no compatible models available' + ); + modelsCleared += 1; + } + } + }); + + // Handle regional guidance reference images + const canvasEntities = selectAllEntities(state.canvas.present); + canvasEntities.forEach((entity: CanvasEntityState) => { + if (isRegionalGuidanceEntityIdentifier(getEntityIdentifier(entity))) { + entity.referenceImages.forEach((refImage: any) => { + if (refImage.config.model && refImage.config.model.base !== newBaseModel) { + dispatch(rgRefImageModelChanged({ + entityIdentifier: getEntityIdentifier(entity), + referenceImageId: refImage.id, + modelConfig: firstCompatibleModel + })); + if (firstCompatibleModel) { + log.debug( + { oldModel: refImage.config.model, newModel: firstCompatibleModel }, + 'Switched regional guidance reference image model to compatible model' + ); + } else { + log.debug( + { oldModel: refImage.config.model }, + 'Cleared regional guidance reference image model - no compatible models available' + ); + modelsCleared += 1; + } + } + }); + } + }); + if (modelsCleared > 0) { toast({ id: 'BASE_MODEL_CHANGED', diff --git a/invokeai/frontend/web/src/services/api/types.ts b/invokeai/frontend/web/src/services/api/types.ts index 725b0ebfe8..1dd5319355 100644 --- a/invokeai/frontend/web/src/services/api/types.ts +++ b/invokeai/frontend/web/src/services/api/types.ts @@ -400,3 +400,12 @@ export type UploadImageArg = { export type ImageUploadEntryResponse = S['ImageUploadEntry']; export type ImageUploadEntryRequest = paths['/api/v1/images/']['post']['requestBody']['content']['application/json']; + +export const isApiModelConfig = (config: AnyModelConfig): config is ApiModelConfig => { + return ( + isChatGPT4oModelConfig(config) || + isImagen3ModelConfig(config) || + isImagen4ModelConfig(config) || + isFluxKontextApiModelConfig(config) + ); +};