feat(ui): add new img2img canvas from image functionality

This replicates the img2img flow:
- Reset the canvas
- Resize the bbox to the image's aspect ratio at the optimal size for the selected model
- Add the image as a raster layer
- Resizes the layer to fit the bbox using the 'fill' strategy

After this completes, the user can immediately click Invoke and it will do img2img.

invokeai/frontend/web/public/locales/en.json

@@ -1641,6 +1641,7 @@
         "sendToCanvas": "Send To Canvas",
         "newLayerFromImage": "New Layer from Image",
         "newCanvasFromImage": "New Canvas from Image",
+        "newImg2ImgCanvasFromImage": "New Img2Img from Image",
         "copyToClipboard": "Copy to Clipboard",
         "sendToCanvasDesc": "Pressing Invoke stages your work in progress on the canvas.",
         "viewProgressInViewer": "View progress and outputs in the <Btn>Image Viewer</Btn>.",

invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts

@@ -2,9 +2,11 @@ import { createSelector } from '@reduxjs/toolkit';
 import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { deepClone } from 'common/util/deepClone';
+import { CanvasEntityAdapterBase } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterBase';
 import { getPrefixedId } from 'features/controlLayers/konva/util';
 import { canvasReset } from 'features/controlLayers/store/actions';
 import {
+  bboxChangedFromCanvas,
   controlLayerAdded,
   inpaintMaskAdded,
   rasterLayerAdded,
@@ -15,7 +17,12 @@ import {
   rgPositivePromptChanged,
 } from 'features/controlLayers/store/canvasSlice';
 import { selectBase } from 'features/controlLayers/store/paramsSlice';
-import { selectBboxRect, selectCanvasSlice, selectEntityOrThrow } from 'features/controlLayers/store/selectors';
+import {
+  selectBboxModelBase,
+  selectBboxRect,
+  selectCanvasSlice,
+  selectEntityOrThrow,
+} from 'features/controlLayers/store/selectors';
 import type {
   CanvasEntityIdentifier,
   CanvasRasterLayerState,
@@ -30,7 +37,9 @@ import {
   initialIPAdapter,
   initialT2IAdapter,
 } from 'features/controlLayers/store/util';
+import { calculateNewSize } from 'features/controlLayers/util/getScaledBoundingBoxDimensions';
 import { zModelIdentifierField } from 'features/nodes/types/common';
+import { getOptimalDimension } from 'features/parameters/util/optimalDimension';
 import { useCallback } from 'react';
 import { modelConfigsAdapterSelectors, selectModelConfigsQuery } from 'services/api/endpoints/models';
 import type { ControlNetModelConfig, ImageDTO, IPAdapterModelConfig, T2IAdapterModelConfig } from 'services/api/types';
@@ -134,6 +143,56 @@ export const useNewCanvasFromImage = () => {
   return func;
 };
 
+/**
+ * Returns a function that adds a new canvas with the given image as the initial image, replicating the img2img flow:
+ * - Reset the canvas
+ * - Resize the bbox to the image's aspect ratio at the optimal size for the selected model
+ * - Add the image as a raster layer
+ * - Resizes the layer to fit the bbox using the 'fill' strategy
+ *
+ * This allows the user to immediately generate a new image from the given image without any additional steps.
+ */
+export const useNewImg2ImgCanvasFromImage = () => {
+  const dispatch = useAppDispatch();
+  const bboxRect = useAppSelector(selectBboxRect);
+  const base = useAppSelector(selectBboxModelBase);
+  const func = useCallback(
+    (imageDTO: ImageDTO) => {
+      // Calculate the new bbox dimensions to fit the image's aspect ratio at the optimal size
+      const ratio = imageDTO.width / imageDTO.height;
+      const optimalDimension = getOptimalDimension(base);
+      const { width, height } = calculateNewSize(ratio, optimalDimension ** 2, base);
+
+      // The overrides need to include the layer's ID so we can transform the layer it is initialized
+      const overrides = {
+        id: getPrefixedId('raster_layer'),
+        position: { x: bboxRect.x, y: bboxRect.y },
+        objects: [imageDTOToImageObject(imageDTO)],
+      } satisfies Partial<CanvasRasterLayerState>;
+
+      CanvasEntityAdapterBase.registerInitCallback(async (adapter) => {
+        // Skip the callback if the adapter is not the one we are creating
+        if (adapter.id !== overrides.id) {
+          return false;
+        }
+        // Fit the layer to the bbox w/ fill strategy
+        await adapter.transformer.startTransform({ silent: true });
+        adapter.transformer.fitToBboxFill();
+        await adapter.transformer.applyTransform();
+        return true;
+      });
+
+      dispatch(canvasReset());
+      // The `bboxChangedFromCanvas` reducer does no validation! Careful!
+      dispatch(bboxChangedFromCanvas({ x: 0, y: 0, width, height }));
+      dispatch(rasterLayerAdded({ overrides, isSelected: true }));
+    },
+    [base, bboxRect.x, bboxRect.y, dispatch]
+  );
+
+  return func;
+};
+
 export const useAddInpaintMask = () => {
   const dispatch = useAppDispatch();
   const func = useCallback(() => {

invokeai/frontend/web/src/features/gallery/components/ImageContextMenu/ImageMenuItemsNewImg2ImgCanvasFromImage.tsx

@@ -0,0 +1,37 @@
+import { MenuItem } from '@invoke-ai/ui-library';
+import { useAppDispatch } from 'app/store/storeHooks';
+import { useNewImg2ImgCanvasFromImage } from 'features/controlLayers/hooks/addLayerHooks';
+import { useImageViewer } from 'features/gallery/components/ImageViewer/useImageViewer';
+import { useImageDTOContext } from 'features/gallery/contexts/ImageDTOContext';
+import { toast } from 'features/toast/toast';
+import { setActiveTab } from 'features/ui/store/uiSlice';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiFileImageBold } from 'react-icons/pi';
+
+export const ImageMenuItemsNewImg2ImgCanvasFromImage = memo(() => {
+  const { t } = useTranslation();
+  const dispatch = useAppDispatch();
+  const imageDTO = useImageDTOContext();
+  const imageViewer = useImageViewer();
+  const newImg2ImgCanvasFromImage = useNewImg2ImgCanvasFromImage();
+
+  const onClick = useCallback(() => {
+    newImg2ImgCanvasFromImage(imageDTO);
+    dispatch(setActiveTab('canvas'));
+    imageViewer.close();
+    toast({
+      id: 'SENT_TO_CANVAS',
+      title: t('toast.sentToCanvas'),
+      status: 'success',
+    });
+  }, [dispatch, imageDTO, imageViewer, newImg2ImgCanvasFromImage, t]);
+
+  return (
+    <MenuItem icon={<PiFileImageBold />} onClickCapture={onClick}>
+      {t('controlLayers.newImg2ImgCanvasFromImage')}
+    </MenuItem>
+  );
+});
+
+ImageMenuItemsNewImg2ImgCanvasFromImage.displayName = 'ImageMenuItemsNewImg2ImgCanvasFromImage';

invokeai/frontend/web/src/features/gallery/components/ImageContextMenu/SingleSelectionMenuItems.tsx

@@ -1,5 +1,6 @@
 import { MenuDivider } from '@invoke-ai/ui-library';
 import { IconMenuItemGroup } from 'common/components/IconMenuItem';
+import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
 import { ImageMenuItemChangeBoard } from 'features/gallery/components/ImageContextMenu/ImageMenuItemChangeBoard';
 import { ImageMenuItemCopy } from 'features/gallery/components/ImageContextMenu/ImageMenuItemCopy';
 import { ImageMenuItemDelete } from 'features/gallery/components/ImageContextMenu/ImageMenuItemDelete';
@@ -12,6 +13,7 @@ import { ImageMenuItemOpenInNewTab } from 'features/gallery/components/ImageCont
 import { ImageMenuItemOpenInViewer } from 'features/gallery/components/ImageContextMenu/ImageMenuItemOpenInViewer';
 import { ImageMenuItemSelectForCompare } from 'features/gallery/components/ImageContextMenu/ImageMenuItemSelectForCompare';
 import { ImageMenuItemSendToUpscale } from 'features/gallery/components/ImageContextMenu/ImageMenuItemSendToUpscale';
+import { ImageMenuItemsNewImg2ImgCanvasFromImage } from 'features/gallery/components/ImageContextMenu/ImageMenuItemsNewImg2ImgCanvasFromImage';
 import { ImageMenuItemStarUnstar } from 'features/gallery/components/ImageContextMenu/ImageMenuItemStarUnstar';
 import { ImageDTOContextProvider } from 'features/gallery/contexts/ImageDTOContext';
 import { memo } from 'react';
@@ -37,8 +39,11 @@ const SingleSelectionMenuItems = ({ imageDTO }: SingleSelectionMenuItemsProps) =
       <ImageMenuItemMetadataRecallActions />
       <MenuDivider />
       <ImageMenuItemSendToUpscale />
-      <ImageMenuItemNewLayerFromImage />
-      <ImageMenuItemNewCanvasFromImage />
+      <CanvasManagerProviderGate>
+        <ImageMenuItemNewLayerFromImage />
+        <ImageMenuItemNewCanvasFromImage />
+        <ImageMenuItemsNewImg2ImgCanvasFromImage />
+      </CanvasManagerProviderGate>
       <MenuDivider />
       <ImageMenuItemChangeBoard />
       <ImageMenuItemStarUnstar />