refactor(ui): canvas flow (wip)

invokeai/frontend/web/src/features/controlLayers/components/CanvasAlerts/CanvasAlertsInvocationProgress.tsx

@@ -6,11 +6,11 @@ import { selectIsLocal } from 'features/system/store/configSlice';
 import { selectSystemShouldShowInvocationProgressDetail } from 'features/system/store/systemSlice';
 import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { $invocationProgressMessage } from 'services/events/stores';
+import { $lastProgressMessage } from 'services/events/stores';
 
 const CanvasAlertsInvocationProgressContentLocal = memo(() => {
   const { t } = useTranslation();
-  const invocationProgressMessage = useStore($invocationProgressMessage);
+  const invocationProgressMessage = useStore($lastProgressMessage);
 
   if (!invocationProgressMessage) {
     return null;

invokeai/frontend/web/src/features/controlLayers/components/CanvasMainPanelContent.tsx

@@ -1,8 +1,21 @@
 import type { SystemStyleObject } from '@invoke-ai/ui-library';
-import { Button, ContextMenu, Flex, IconButton, Image, Menu, MenuButton, MenuList, Text } from '@invoke-ai/ui-library';
+import {
+  Button,
+  ContextMenu,
+  Flex,
+  Heading,
+  IconButton,
+  Image,
+  Menu,
+  MenuButton,
+  MenuList,
+  Text,
+} from '@invoke-ai/ui-library';
 import { useStore } from '@nanostores/react';
+import { useAppStore } from 'app/store/nanostores/store';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { FocusRegionWrapper } from 'common/components/FocusRegionWrapper';
+import { useImageUploadButton } from 'common/hooks/useImageUploadButton';
 import { CanvasAlertsPreserveMask } from 'features/controlLayers/components/CanvasAlerts/CanvasAlertsPreserveMask';
 import { CanvasAlertsSelectedEntityStatus } from 'features/controlLayers/components/CanvasAlerts/CanvasAlertsSelectedEntityStatus';
 import { CanvasAlertsSendingToGallery } from 'features/controlLayers/components/CanvasAlerts/CanvasAlertsSendingTo';
@@ -32,13 +45,17 @@ import {
   stagingAreaNextStagedImageSelected,
   stagingAreaPrevStagedImageSelected,
 } from 'features/controlLayers/store/canvasStagingAreaSlice';
+import { newCanvasFromImageDndTarget } from 'features/dnd/dnd';
+import { DndDropTarget } from 'features/dnd/DndDropTarget';
+import { newCanvasFromImage } from 'features/imageActions/actions';
 import type { ProgressImage } from 'features/nodes/types/common';
-import { memo, useCallback, useEffect } from 'react';
+import { memo, useCallback, useEffect, useMemo } from 'react';
 import { useHotkeys } from 'react-hotkeys-hook';
-import { PiDotsThreeOutlineVerticalFill } from 'react-icons/pi';
+import { Trans, useTranslation } from 'react-i18next';
+import { PiDotsThreeOutlineVerticalFill, PiUploadBold } from 'react-icons/pi';
 import type { ImageDTO, S } from 'services/api/types';
 import { $lastCanvasProgressImage, $socket } from 'services/events/stores';
-import type { Equals } from 'tsafe';
+import type { Equals, Param0 } from 'tsafe';
 import { assert } from 'tsafe';
 
 import { CanvasAlertsInvocationProgress } from './CanvasAlerts/CanvasAlertsInvocationProgress';
@@ -80,45 +97,190 @@ export const CanvasMainPanelContent = memo(() => {
 
 CanvasMainPanelContent.displayName = 'CanvasMainPanelContent';
 
+const generateWithStartingImageDndTargetData = newCanvasFromImageDndTarget.getData({
+  type: 'raster_layer',
+  withResize: true,
+});
+const generateWithStartingImageAndInpaintMaskDndTargetData = newCanvasFromImageDndTarget.getData({
+  type: 'raster_layer',
+  withInpaintMask: true,
+});
+const generateWithControlImageDndTargetData = newCanvasFromImageDndTarget.getData({
+  type: 'control_layer',
+  withResize: true,
+});
+
 const NoActiveSession = memo(() => {
+  const { t } = useTranslation();
   const dispatch = useAppDispatch();
   const newSesh = useCallback(() => {
     dispatch(canvasSessionStarted({ sessionType: 'advanced' }));
   }, [dispatch]);
+
   return (
     <Flex flexDir="column" w="full" h="full" alignItems="center" justifyContent="center">
-      <Text fontSize="lg" fontWeight="bold">
-        No Active Session
-      </Text>
-      <Button display="flex" flexDir="column" gap={2} p={8} minH={0} minW={0} onClick={newSesh}>
-        <Text>New Canvas Session</Text>
-        <Text>- New Canvas Session</Text>
-        <Text>- 1 Inpaint mask layer added</Text>
+      <Heading>Get Started with Invoke</Heading>
+      <Button variant="ghost" onClick={newSesh}>
+        Start a new Canvas Session
       </Button>
-      <Flex flexDir="column" gap={2} p={8} border="dashed yellow 2px">
-        <Text>Generate with Starting Image</Text>
-        <Text>- New Canvas Session</Text>
-        <Text>- Dropped image as raster layer</Text>
-        <Text>- Bbox resized</Text>
-      </Flex>
-      <Flex flexDir="column" gap={2} p={8} border="dashed yellow 2px">
-        <Text>Generate with Control Image</Text>
-        <Text>- New Canvas Session</Text>
-        <Text>- Dropped image as control layer</Text>
-        <Text>- Bbox resized</Text>
-      </Flex>
-      <Flex flexDir="column" gap={2} p={8} border="dashed yellow 2px">
-        <Text>Edit Image</Text>
-        <Text>- New Canvas Session</Text>
-        <Text>- Dropped image as raster layer</Text>
-        <Text>- Bbox resized</Text>
-        <Text>- 1 Inpaint mask layer added</Text>
+      <Text>or</Text>
+      <Flex flexDir="column" maxW={512}>
+        <GenerateWithStartingImage />
+        <GenerateWithControlImage />
+        <GenerateWithStartingImageAndInpaintMask />
       </Flex>
     </Flex>
   );
 });
 NoActiveSession.displayName = 'NoActiveSession';
 
+const GenerateWithStartingImage = memo(() => {
+  const { t } = useTranslation();
+  const { getState, dispatch } = useAppStore();
+  const useImageUploadButtonOptions = useMemo<Param0<typeof useImageUploadButton>>(
+    () => ({
+      onUpload: (imageDTO: ImageDTO) => {
+        newCanvasFromImage({ imageDTO, type: 'raster_layer', withResize: true, getState, dispatch });
+      },
+      allowMultiple: false,
+    }),
+    [dispatch, getState]
+  );
+  const uploadApi = useImageUploadButton(useImageUploadButtonOptions);
+  const components = useMemo(
+    () => ({
+      UploadButton: (
+        <Button
+          size="sm"
+          variant="link"
+          color="base.300"
+          {...uploadApi.getUploadButtonProps()}
+          rightIcon={<PiUploadBold />}
+        />
+      ),
+    }),
+    [uploadApi]
+  );
+
+  return (
+    <Flex position="relative" flexDir="column">
+      <Text fontSize="lg" fontWeight="semibold">
+        Generate with a Starting Image
+      </Text>
+      <Text color="base.300">Regenerate the starting image using the model (Image to Image).</Text>
+      <Text color="base.300">
+        <Trans i18nKey="controlLayers.uploadOrDragAnImage" components={components} />
+        <input {...uploadApi.getUploadInputProps()} />
+      </Text>
+      <DndDropTarget
+        dndTarget={newCanvasFromImageDndTarget}
+        dndTargetData={generateWithStartingImageDndTargetData}
+        label="Drop"
+      />
+    </Flex>
+  );
+});
+GenerateWithStartingImage.displayName = 'GenerateWithStartingImage';
+
+const GenerateWithControlImage = memo(() => {
+  const { t } = useTranslation();
+  const { getState, dispatch } = useAppStore();
+  const useImageUploadButtonOptions = useMemo<Param0<typeof useImageUploadButton>>(
+    () => ({
+      onUpload: (imageDTO: ImageDTO) => {
+        newCanvasFromImage({ imageDTO, type: 'control_layer', withResize: true, getState, dispatch });
+      },
+      allowMultiple: false,
+    }),
+    [dispatch, getState]
+  );
+  const uploadApi = useImageUploadButton(useImageUploadButtonOptions);
+  const components = useMemo(
+    () => ({
+      UploadButton: (
+        <Button
+          size="sm"
+          variant="link"
+          color="base.300"
+          {...uploadApi.getUploadButtonProps()}
+          rightIcon={<PiUploadBold />}
+        />
+      ),
+    }),
+    [uploadApi]
+  );
+
+  return (
+    <Flex position="relative" flexDir="column">
+      <Text fontSize="lg" fontWeight="semibold">
+        Generate with a Control Image
+      </Text>
+      <Text color="base.300">
+        Generate a new image using the control image to guide the structure and composition (Text to Image with
+        Control).
+      </Text>
+      <Text color="base.300">
+        <Trans i18nKey="controlLayers.uploadOrDragAnImage" components={components} />
+        <input {...uploadApi.getUploadInputProps()} />
+      </Text>
+      <DndDropTarget
+        dndTarget={newCanvasFromImageDndTarget}
+        dndTargetData={generateWithControlImageDndTargetData}
+        label="Drop"
+      />
+    </Flex>
+  );
+});
+GenerateWithControlImage.displayName = 'GenerateWithControlImage';
+
+const GenerateWithStartingImageAndInpaintMask = memo(() => {
+  const { t } = useTranslation();
+  const { getState, dispatch } = useAppStore();
+  const useImageUploadButtonOptions = useMemo<Param0<typeof useImageUploadButton>>(
+    () => ({
+      onUpload: (imageDTO: ImageDTO) => {
+        newCanvasFromImage({ imageDTO, type: 'raster_layer', withInpaintMask: true, getState, dispatch });
+      },
+      allowMultiple: false,
+    }),
+    [dispatch, getState]
+  );
+  const uploadApi = useImageUploadButton(useImageUploadButtonOptions);
+  const components = useMemo(
+    () => ({
+      UploadButton: (
+        <Button
+          size="sm"
+          variant="link"
+          color="base.300"
+          {...uploadApi.getUploadButtonProps()}
+          rightIcon={<PiUploadBold />}
+        />
+      ),
+    }),
+    [uploadApi]
+  );
+
+  return (
+    <Flex position="relative" flexDir="column">
+      <Text fontSize="lg" fontWeight="semibold">
+        Edit Image
+      </Text>
+      <Text color="base.300">Edit the image by regenerating parts of it (Inpaint).</Text>
+      <Text color="base.300">
+        <Trans i18nKey="controlLayers.uploadOrDragAnImage" components={components} />
+        <input {...uploadApi.getUploadInputProps()} />
+      </Text>
+      <DndDropTarget
+        dndTarget={newCanvasFromImageDndTarget}
+        dndTargetData={generateWithStartingImageAndInpaintMaskDndTargetData}
+        label="Drop"
+      />
+    </Flex>
+  );
+});
+GenerateWithStartingImageAndInpaintMask.displayName = 'GenerateWithStartingImageAndInpaintMask';
+
 type EphemeralProgressImage = { sessionId: string; image: ProgressImage };
 
 const SimpleActiveSession = memo(() => {
@@ -213,6 +375,9 @@ const SelectedImage = memo(() => {
           src={selectedImage.imageDTO.image_url}
           width={selectedImage.imageDTO.width}
           height={selectedImage.imageDTO.height}
+          onLoad={() => {
+            console.log('onload');
+          }}
         />
       </Flex>
     );

invokeai/frontend/web/src/features/controlLayers/components/CanvasRightPanelStacked.tsx

@@ -0,0 +1,254 @@
+import { combine } from '@atlaskit/pragmatic-drag-and-drop/combine';
+import { dropTargetForElements, monitorForElements } from '@atlaskit/pragmatic-drag-and-drop/element/adapter';
+import { dropTargetForExternal, monitorForExternal } from '@atlaskit/pragmatic-drag-and-drop/external/adapter';
+import { Box, Tab } from '@invoke-ai/ui-library';
+import { useAppDispatch, useAppSelector, useAppStore } from 'app/store/storeHooks';
+import { CanvasLayersPanelContent } from 'features/controlLayers/components/CanvasLayersPanelContent';
+import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
+import { selectEntityCountActive } from 'features/controlLayers/store/selectors';
+import { multipleImageDndSource, singleImageDndSource } from 'features/dnd/dnd';
+import { DndDropOverlay } from 'features/dnd/DndDropOverlay';
+import type { DndTargetState } from 'features/dnd/types';
+import GalleryPanelContent from 'features/gallery/components/GalleryPanelContent';
+import { useImageViewer } from 'features/gallery/components/ImageViewer/useImageViewer';
+import { selectActiveTabCanvasRightPanel } from 'features/ui/store/uiSelectors';
+import { activeTabCanvasRightPanelChanged } from 'features/ui/store/uiSlice';
+import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Panel, PanelGroup, PanelResizeHandle } from 'react-resizable-panels';
+
+export const CanvasRightPanelStacked = memo(() => {
+  const { t } = useTranslation();
+  const activeTab = useAppSelector(selectActiveTabCanvasRightPanel);
+  const imageViewer = useImageViewer();
+  const dispatch = useAppDispatch();
+
+  const tabIndex = useMemo(() => {
+    if (activeTab === 'gallery') {
+      return 1;
+    } else {
+      return 0;
+    }
+  }, [activeTab]);
+
+  const onClickViewerToggleButton = useCallback(() => {
+    imageViewer.open();
+  }, [imageViewer]);
+
+  const onChangeTab = useCallback(
+    (index: number) => {
+      if (index === 0) {
+        dispatch(activeTabCanvasRightPanelChanged('layers'));
+      } else {
+        dispatch(activeTabCanvasRightPanelChanged('gallery'));
+      }
+    },
+    [dispatch]
+  );
+
+  return (
+    <PanelGroup direction="vertical">
+      <Panel>
+        <GalleryPanelContent />
+      </Panel>
+      <PanelResizeHandle />
+      <Panel>
+        <CanvasManagerProviderGate>
+          <CanvasLayersPanelContent />
+        </CanvasManagerProviderGate>
+      </Panel>
+    </PanelGroup>
+  );
+});
+
+CanvasRightPanelStacked.displayName = 'CanvasRightPanelStacked';
+
+const PanelTabs = memo(() => {
+  const { t } = useTranslation();
+  const store = useAppStore();
+  const activeEntityCount = useAppSelector(selectEntityCountActive);
+  const [layersTabDndState, setLayersTabDndState] = useState<DndTargetState>('idle');
+  const [galleryTabDndState, setGalleryTabDndState] = useState<DndTargetState>('idle');
+  const layersTabRef = useRef<HTMLDivElement>(null);
+  const galleryTabRef = useRef<HTMLDivElement>(null);
+  const timeoutRef = useRef<number | null>(null);
+
+  const layersTabLabel = useMemo(() => {
+    if (activeEntityCount === 0) {
+      return t('controlLayers.layer_other');
+    }
+    return `${t('controlLayers.layer_other')} (${activeEntityCount})`;
+  }, [activeEntityCount, t]);
+
+  useEffect(() => {
+    if (!layersTabRef.current) {
+      return;
+    }
+
+    const getIsOnLayersTab = () => selectActiveTabCanvasRightPanel(store.getState()) === 'layers';
+
+    const onDragEnter = () => {
+      // If we are already on the layers tab, do nothing
+      if (getIsOnLayersTab()) {
+        return;
+      }
+
+      // Else set the state to active and switch to the layers tab after a timeout
+      setLayersTabDndState('over');
+      timeoutRef.current = window.setTimeout(() => {
+        timeoutRef.current = null;
+        store.dispatch(activeTabCanvasRightPanelChanged('layers'));
+        // When we switch tabs, the other tab should be pending
+        setLayersTabDndState('idle');
+        setGalleryTabDndState('potential');
+      }, 300);
+    };
+    const onDragLeave = () => {
+      // Set the state to idle or pending depending on the current tab
+      if (getIsOnLayersTab()) {
+        setLayersTabDndState('idle');
+      } else {
+        setLayersTabDndState('potential');
+      }
+      // Abort the tab switch if it hasn't happened yet
+      if (timeoutRef.current !== null) {
+        clearTimeout(timeoutRef.current);
+      }
+    };
+    const onDragStart = () => {
+      // Set the state to pending when a drag starts
+      setLayersTabDndState('potential');
+    };
+    return combine(
+      dropTargetForElements({
+        element: layersTabRef.current,
+        onDragEnter,
+        onDragLeave,
+      }),
+      monitorForElements({
+        canMonitor: ({ source }) => {
+          if (!singleImageDndSource.typeGuard(source.data) && !multipleImageDndSource.typeGuard(source.data)) {
+            return false;
+          }
+          // Only monitor if we are not already on the gallery tab
+          return !getIsOnLayersTab();
+        },
+        onDragStart,
+      }),
+      dropTargetForExternal({
+        element: layersTabRef.current,
+        onDragEnter,
+        onDragLeave,
+      }),
+      monitorForExternal({
+        canMonitor: () => !getIsOnLayersTab(),
+        onDragStart,
+      })
+    );
+  }, [store]);
+
+  useEffect(() => {
+    if (!galleryTabRef.current) {
+      return;
+    }
+
+    const getIsOnGalleryTab = () => selectActiveTabCanvasRightPanel(store.getState()) === 'gallery';
+
+    const onDragEnter = () => {
+      // If we are already on the gallery tab, do nothing
+      if (getIsOnGalleryTab()) {
+        return;
+      }
+
+      // Else set the state to active and switch to the gallery tab after a timeout
+      setGalleryTabDndState('over');
+      timeoutRef.current = window.setTimeout(() => {
+        timeoutRef.current = null;
+        store.dispatch(activeTabCanvasRightPanelChanged('gallery'));
+        // When we switch tabs, the other tab should be pending
+        setGalleryTabDndState('idle');
+        setLayersTabDndState('potential');
+      }, 300);
+    };
+
+    const onDragLeave = () => {
+      // Set the state to idle or pending depending on the current tab
+      if (getIsOnGalleryTab()) {
+        setGalleryTabDndState('idle');
+      } else {
+        setGalleryTabDndState('potential');
+      }
+      // Abort the tab switch if it hasn't happened yet
+      if (timeoutRef.current !== null) {
+        clearTimeout(timeoutRef.current);
+      }
+    };
+
+    const onDragStart = () => {
+      // Set the state to pending when a drag starts
+      setGalleryTabDndState('potential');
+    };
+
+    return combine(
+      dropTargetForElements({
+        element: galleryTabRef.current,
+        onDragEnter,
+        onDragLeave,
+      }),
+      monitorForElements({
+        canMonitor: ({ source }) => {
+          if (!singleImageDndSource.typeGuard(source.data) && !multipleImageDndSource.typeGuard(source.data)) {
+            return false;
+          }
+          // Only monitor if we are not already on the gallery tab
+          return !getIsOnGalleryTab();
+        },
+        onDragStart,
+      }),
+      dropTargetForExternal({
+        element: galleryTabRef.current,
+        onDragEnter,
+        onDragLeave,
+      }),
+      monitorForExternal({
+        canMonitor: () => !getIsOnGalleryTab(),
+        onDragStart,
+      })
+    );
+  }, [store]);
+
+  useEffect(() => {
+    const onDrop = () => {
+      // Reset the dnd state when a drop happens
+      setGalleryTabDndState('idle');
+      setLayersTabDndState('idle');
+    };
+    const cleanup = combine(monitorForElements({ onDrop }), monitorForExternal({ onDrop }));
+
+    return () => {
+      cleanup();
+      if (timeoutRef.current !== null) {
+        clearTimeout(timeoutRef.current);
+      }
+    };
+  }, []);
+
+  return (
+    <>
+      <Tab ref={layersTabRef} position="relative" w={32}>
+        <Box as="span" w="full">
+          {layersTabLabel}
+        </Box>
+        <DndDropOverlay dndState={layersTabDndState} withBackdrop={false} />
+      </Tab>
+      <Tab ref={galleryTabRef} position="relative" w={32}>
+        <Box as="span" w="full">
+          {t('gallery.gallery')}
+        </Box>
+        <DndDropOverlay dndState={galleryTabDndState} withBackdrop={false} />
+      </Tab>
+    </>
+  );
+});
+
+PanelTabs.displayName = 'PanelTabs';

invokeai/frontend/web/src/features/controlLayers/components/IPAdapter/IPAdapterImagePreview.tsx

@@ -1,7 +1,7 @@
 import { Flex } from '@invoke-ai/ui-library';
 import { useStore } from '@nanostores/react';
 import { skipToken } from '@reduxjs/toolkit/query';
-import { UploadImageButton } from 'common/hooks/useImageUploadButton';
+import { UploadImageIconButton } from 'common/hooks/useImageUploadButton';
 import type { ImageWithDims } from 'features/controlLayers/store/types';
 import type { setGlobalReferenceImageDndTarget, setRegionalGuidanceReferenceImageDndTarget } from 'features/dnd/dnd';
 import { DndDropTarget } from 'features/dnd/DndDropTarget';
@@ -51,7 +51,7 @@ export const IPAdapterImagePreview = memo(
     return (
       <Flex position="relative" w="full" h="full" alignItems="center" data-error={!imageDTO && !image?.image_name}>
         {!imageDTO && (
-          <UploadImageButton
+          <UploadImageIconButton
             w="full"
             h="full"
             isError={!imageDTO && !image?.image_name}

invokeai/frontend/web/src/features/controlLayers/components/NewSessionConfirmationAlertDialog.tsx

@@ -20,7 +20,7 @@ export const useNewGallerySession = () => {
   const newSessionDialog = useNewGallerySessionDialog();
 
   const newGallerySessionImmediate = useCallback(() => {
-    dispatch(canvasSessionStarted({ sessionType: 'simple' }));
+    dispatch(canvasSessionStarted({ sessionType: null }));
     dispatch(activeTabCanvasRightPanelChanged('gallery'));
   }, [dispatch]);
 

invokeai/frontend/web/src/features/controlLayers/hooks/useDeferredModelLoadingInvocationProgressMessage.ts

@@ -1,11 +1,11 @@
 import { useStore } from '@nanostores/react';
 import { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { $invocationProgressMessage } from 'services/events/stores';
+import { $lastProgressMessage } from 'services/events/stores';
 
 export const useDeferredModelLoadingInvocationProgressMessage = () => {
   const { t } = useTranslation();
-  const invocationProgressMessage = useStore($invocationProgressMessage);
+  const invocationProgressMessage = useStore($lastProgressMessage);
   const [delayedMessage, setDelayedMessage] = useState<string | null>(null);
 
   useEffect(() => {

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -1,6 +1,7 @@
 import { deepClone } from 'common/util/deepClone';
 import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types';
 import { fetchModelConfigByIdentifier } from 'features/metadata/util/modelFetchingHelpers';
+import type { ProgressImage} from 'features/nodes/types/common';
 import { zMainModelBase, zModelIdentifierField } from 'features/nodes/types/common';
 import type { ParameterLoRAModel } from 'features/parameters/types/parameterSchemas';
 import {
@@ -437,10 +438,12 @@ export type LoRA = {
 };
 
 export type StagingAreaImage = {
+  sessionId: string;
   imageDTO: ImageDTO;
   offsetX: number;
   offsetY: number;
 };
+export type EphemeralProgressImage = { sessionId: string; image: ProgressImage };
 
 export const zAspectRatioID = z.enum(['Free', '21:9', '9:21', '16:9', '3:2', '4:3', '1:1', '3:4', '2:3', '9:16']);