From 038010a1ca308e9f3a2067723eedd021f2897bda Mon Sep 17 00:00:00 2001 From: Mary Hipp Rogers Date: Wed, 2 Jul 2025 10:26:48 -0400 Subject: [PATCH] feat(ui): prompt expansion (#8140) * initializing prompt expansion and putting response in prompt box working for all methods * properly disable UI and show loading state on prompt box when there is a pending prompt expansion item * misc wrapup: disable apploying prompt templates, dont block textarea resize handle * update progress to differentiate between prompt expansion and non * cleanup * lint * more cleanup * add image to background of loading state * add allowPromptExpansion for front-end gating * updated readiness text for needing to accept or discard * fix tsc * lint * lint * refactor(ui): prompt expansion logic * tidy(ui): remove unnecessary changes * revert(ui): unused arg on useImageUploadButton * feat(ui): simplify prompt expansion state * set pending for dragndrop and context menu * add readiness logic for generate tab * missing translation * update error handling for prompt expansion --------- Co-authored-by: Mary Hipp Co-authored-by: Mary Hipp Co-authored-by: psychedelicious <4822129+psychedelicious@users.noreply.github.com> --- invokeai/frontend/web/public/locales/en.json | 30 ++++- .../frontend/web/src/app/types/invokeai.ts | 1 + .../src/common/hooks/useImageUploadButton.tsx | 28 ++++- .../konva/CanvasStateApiModule.ts | 2 +- .../src/features/controlLayers/store/util.ts | 3 + invokeai/frontend/web/src/features/dnd/dnd.ts | 41 ++++-- .../ImageMenuItemUseForPromptGeneration.tsx | 46 +++++++ .../SingleSelectionMenuItems.tsx | 2 + .../components/Core/ParamPositivePrompt.tsx | 95 ++++++++------ .../Prompts/PromptOverlayButtonWrapper.tsx | 4 +- .../PromptExpansion/PromptExpansionMenu.tsx | 80 ++++++++++++ .../PromptExpansionOverlay.tsx | 69 ++++++++++ .../PromptExpansionResultOverlay.tsx | 76 +++++++++++ .../features/prompt/PromptExpansion/expand.ts | 43 +++++++ .../features/prompt/PromptExpansion/graph.ts | 43 +++++++ .../features/prompt/PromptExpansion/state.ts | 98 +++++++++++++++ .../web/src/features/prompt/usePrompt.ts | 7 +- .../web/src/features/queue/store/readiness.ts | 118 +++++++++++++++++- .../src/features/system/store/configSlice.ts | 2 + .../web/src/services/api/run-graph.ts | 22 ++-- 20 files changed, 739 insertions(+), 71 deletions(-) create mode 100644 invokeai/frontend/web/src/features/gallery/components/ImageContextMenu/ImageMenuItemUseForPromptGeneration.tsx create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/PromptExpansionMenu.tsx create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/PromptExpansionOverlay.tsx create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/PromptExpansionResultOverlay.tsx create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/expand.ts create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/graph.ts create mode 100644 invokeai/frontend/web/src/features/prompt/PromptExpansion/state.ts diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 44cba5629d..fff29142fd 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -225,7 +225,16 @@ "prompt": { "addPromptTrigger": "Add Prompt Trigger", "compatibleEmbeddings": "Compatible Embeddings", - "noMatchingTriggers": "No matching triggers" + "noMatchingTriggers": "No matching triggers", + "generateFromImage": "Generate prompt from image", + "expandCurrentPrompt": "Expand Current Prompt", + "uploadImageForPromptGeneration": "Upload Image for Prompt Generation", + "expandingPrompt": "Expanding prompt...", + "resultTitle": "Prompt Expansion Complete", + "resultSubtitle": "Choose how to handle the expanded prompt:", + "replace": "Replace", + "insert": "Insert", + "discard": "Discard" }, "queue": { "queue": "Queue", @@ -342,7 +351,7 @@ "copy": "Copy", "currentlyInUse": "This image is currently in use in the following features:", "drop": "Drop", - "dropOrUpload": "$t(gallery.drop) or Upload", + "dropOrUpload": "Drop or Upload", "dropToUpload": "$t(gallery.drop) to Upload", "deleteImage_one": "Delete Image", "deleteImage_other": "Delete {{count}} Images", @@ -396,7 +405,8 @@ "compareHelp4": "Press Z or Esc to exit.", "openViewer": "Open Viewer", "closeViewer": "Close Viewer", - "move": "Move" + "move": "Move", + "useForPromptGeneration": "Use for Prompt Generation" }, "hotkeys": { "hotkeys": "Hotkeys", @@ -938,7 +948,8 @@ "selectModel": "Select a Model", "noLoRAsInstalled": "No LoRAs installed", "noRefinerModelsInstalled": "No SDXL Refiner models installed", - "defaultVAE": "Default VAE" + "defaultVAE": "Default VAE", + "noCompatibleLoRAs": "No Compatible LoRAs" }, "nodes": { "arithmeticSequence": "Arithmetic Sequence", @@ -1188,7 +1199,9 @@ "canvasIsSelectingObject": "Canvas is busy (selecting object)", "noPrompts": "No prompts generated", "noNodesInGraph": "No nodes in graph", - "systemDisconnected": "System disconnected" + "systemDisconnected": "System disconnected", + "promptExpansionPending": "Prompt expansion in progress", + "promptExpansionResultPending": "Please accept or discard your prompt expansion result" }, "maskBlur": "Mask Blur", "negativePromptPlaceholder": "Negative Prompt", @@ -1389,7 +1402,12 @@ "fluxKontextIncompatibleGenerationMode": "Flux Kontext supports Text to Image only. Use other models for Image to Image, Inpainting and Outpainting tasks.", "problemUnpublishingWorkflow": "Problem Unpublishing Workflow", "problemUnpublishingWorkflowDescription": "There was a problem unpublishing the workflow. Please try again.", - "workflowUnpublished": "Workflow Unpublished" + "workflowUnpublished": "Workflow Unpublished", + "sentToCanvas": "Sent to Canvas", + "sentToUpscale": "Sent to Upscale", + "promptGenerationStarted": "Prompt generation started", + "uploadAndPromptGenerationFailed": "Failed to upload image and generate prompt", + "promptExpansionFailed": "Prompt expansion failed" }, "popovers": { "clipSkip": { diff --git a/invokeai/frontend/web/src/app/types/invokeai.ts b/invokeai/frontend/web/src/app/types/invokeai.ts index 000b99a1c5..afa3a402aa 100644 --- a/invokeai/frontend/web/src/app/types/invokeai.ts +++ b/invokeai/frontend/web/src/app/types/invokeai.ts @@ -78,6 +78,7 @@ export type AppConfig = { allowPrivateStylePresets: boolean; allowClientSideUpload: boolean; allowPublishWorkflows: boolean; + allowPromptExpansion: boolean; disabledTabs: TabName[]; disabledFeatures: AppFeature[]; disabledSDFeatures: SDFeature[]; diff --git a/invokeai/frontend/web/src/common/hooks/useImageUploadButton.tsx b/invokeai/frontend/web/src/common/hooks/useImageUploadButton.tsx index 9c19a31e5f..ff53b8ec79 100644 --- a/invokeai/frontend/web/src/common/hooks/useImageUploadButton.tsx +++ b/invokeai/frontend/web/src/common/hooks/useImageUploadButton.tsx @@ -21,11 +21,15 @@ type UseImageUploadButtonArgs = isDisabled?: boolean; allowMultiple: false; onUpload?: (imageDTO: ImageDTO) => void; + onUploadStarted?: (files: File) => void; + onError?: (error: unknown) => void; } | { isDisabled?: boolean; allowMultiple: true; onUpload?: (imageDTOs: ImageDTO[]) => void; + onUploadStarted?: (files: File[]) => void; + onError?: (error: unknown) => void; }; const log = logger('gallery'); @@ -49,7 +53,13 @@ const log = logger('gallery'); *