fix(ui): compositor not setting processing flag when cleaning up

Perf boost and reduces the number of images we create on the backend.

invokeai/frontend/web/public/locales/en.json

@@ -1647,8 +1647,9 @@
         "pullBboxIntoReferenceImageError": "Problem Pulling BBox Into ReferenceImage",
         "regionIsEmpty": "Selected region is empty",
         "mergeVisible": "Merge Visible",
-        "mergeVisibleOk": "Merged visible layers",
-        "mergeVisibleError": "Error merging visible layers",
+        "mergeDown": "Merge Down",
+        "mergeVisibleOk": "Merged layers",
+        "mergeVisibleError": "Error merging layers",
         "clearHistory": "Clear History",
         "bboxOverlay": "Show Bbox Overlay",
         "resetCanvas": "Reset Canvas",

invokeai/frontend/web/src/features/controlLayers/components/ControlLayer/ControlLayerMenuItems.tsx

@@ -5,6 +5,7 @@ import { CanvasEntityMenuItemsCropToBbox } from 'features/controlLayers/componen
 import { CanvasEntityMenuItemsDelete } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDelete';
 import { CanvasEntityMenuItemsDuplicate } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDuplicate';
 import { CanvasEntityMenuItemsFilter } from 'features/controlLayers/components/common/CanvasEntityMenuItemsFilter';
+import { CanvasEntityMenuItemsMergeDown } from 'features/controlLayers/components/common/CanvasEntityMenuItemsMergeDown';
 import { CanvasEntityMenuItemsSave } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSave';
 import { CanvasEntityMenuItemsSelectObject } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSelectObject';
 import { CanvasEntityMenuItemsTransform } from 'features/controlLayers/components/common/CanvasEntityMenuItemsTransform';
@@ -27,6 +28,7 @@ export const ControlLayerMenuItems = memo(() => {
       <CanvasEntityMenuItemsSelectObject />
       <ControlLayerMenuItemsTransparencyEffect />
       <MenuDivider />
+      <CanvasEntityMenuItemsMergeDown />
       <ControlLayerMenuItemsCopyToSubMenu />
       <ControlLayerMenuItemsConvertToSubMenu />
       <CanvasEntityMenuItemsCropToBbox />

invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMaskMenuItems.tsx

@@ -4,6 +4,8 @@ import { CanvasEntityMenuItemsArrange } from 'features/controlLayers/components/
 import { CanvasEntityMenuItemsCropToBbox } from 'features/controlLayers/components/common/CanvasEntityMenuItemsCropToBbox';
 import { CanvasEntityMenuItemsDelete } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDelete';
 import { CanvasEntityMenuItemsDuplicate } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDuplicate';
+import { CanvasEntityMenuItemsMergeDown } from 'features/controlLayers/components/common/CanvasEntityMenuItemsMergeDown';
+import { CanvasEntityMenuItemsSave } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSave';
 import { CanvasEntityMenuItemsTransform } from 'features/controlLayers/components/common/CanvasEntityMenuItemsTransform';
 import { InpaintMaskMenuItemsConvertToSubMenu } from 'features/controlLayers/components/InpaintMask/InpaintMaskMenuItemsConvertToSubMenu';
 import { InpaintMaskMenuItemsCopyToSubMenu } from 'features/controlLayers/components/InpaintMask/InpaintMaskMenuItemsCopyToSubMenu';
@@ -20,9 +22,11 @@ export const InpaintMaskMenuItems = memo(() => {
       <MenuDivider />
       <CanvasEntityMenuItemsTransform />
       <MenuDivider />
+      <CanvasEntityMenuItemsMergeDown />
       <InpaintMaskMenuItemsCopyToSubMenu />
       <InpaintMaskMenuItemsConvertToSubMenu />
       <CanvasEntityMenuItemsCropToBbox />
+      <CanvasEntityMenuItemsSave />
     </>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/RasterLayer/RasterLayerMenuItems.tsx

@@ -5,6 +5,7 @@ import { CanvasEntityMenuItemsCropToBbox } from 'features/controlLayers/componen
 import { CanvasEntityMenuItemsDelete } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDelete';
 import { CanvasEntityMenuItemsDuplicate } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDuplicate';
 import { CanvasEntityMenuItemsFilter } from 'features/controlLayers/components/common/CanvasEntityMenuItemsFilter';
+import { CanvasEntityMenuItemsMergeDown } from 'features/controlLayers/components/common/CanvasEntityMenuItemsMergeDown';
 import { CanvasEntityMenuItemsSave } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSave';
 import { CanvasEntityMenuItemsSelectObject } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSelectObject';
 import { CanvasEntityMenuItemsTransform } from 'features/controlLayers/components/common/CanvasEntityMenuItemsTransform';
@@ -25,6 +26,7 @@ export const RasterLayerMenuItems = memo(() => {
       <CanvasEntityMenuItemsFilter />
       <CanvasEntityMenuItemsSelectObject />
       <MenuDivider />
+      <CanvasEntityMenuItemsMergeDown />
       <RasterLayerMenuItemsCopyToSubMenu />
       <RasterLayerMenuItemsConvertToSubMenu />
       <CanvasEntityMenuItemsCropToBbox />

invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceMenuItems.tsx

@@ -4,6 +4,8 @@ import { CanvasEntityMenuItemsArrange } from 'features/controlLayers/components/
 import { CanvasEntityMenuItemsCropToBbox } from 'features/controlLayers/components/common/CanvasEntityMenuItemsCropToBbox';
 import { CanvasEntityMenuItemsDelete } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDelete';
 import { CanvasEntityMenuItemsDuplicate } from 'features/controlLayers/components/common/CanvasEntityMenuItemsDuplicate';
+import { CanvasEntityMenuItemsMergeDown } from 'features/controlLayers/components/common/CanvasEntityMenuItemsMergeDown';
+import { CanvasEntityMenuItemsSave } from 'features/controlLayers/components/common/CanvasEntityMenuItemsSave';
 import { CanvasEntityMenuItemsTransform } from 'features/controlLayers/components/common/CanvasEntityMenuItemsTransform';
 import { RegionalGuidanceMenuItemsAddPromptsAndIPAdapter } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceMenuItemsAddPromptsAndIPAdapter';
 import { RegionalGuidanceMenuItemsAutoNegative } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceMenuItemsAutoNegative';
@@ -25,9 +27,11 @@ export const RegionalGuidanceMenuItems = memo(() => {
       <CanvasEntityMenuItemsTransform />
       <RegionalGuidanceMenuItemsAutoNegative />
       <MenuDivider />
+      <CanvasEntityMenuItemsMergeDown />
       <RegionalGuidanceMenuItemsCopyToSubMenu />
       <RegionalGuidanceMenuItemsConvertToSubMenu />
       <CanvasEntityMenuItemsCropToBbox />
+      <CanvasEntityMenuItemsSave />
     </>
   );
 });

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityGroupList.tsx

@@ -7,9 +7,9 @@ import { CanvasEntityMergeVisibleButton } from 'features/controlLayers/component
 import { CanvasEntityTypeIsHiddenToggle } from 'features/controlLayers/components/common/CanvasEntityTypeIsHiddenToggle';
 import { useEntityTypeInformationalPopover } from 'features/controlLayers/hooks/useEntityTypeInformationalPopover';
 import { useEntityTypeTitle } from 'features/controlLayers/hooks/useEntityTypeTitle';
-import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
+import { type CanvasEntityIdentifier, isRenderableEntityType } from 'features/controlLayers/store/types';
 import type { PropsWithChildren } from 'react';
-import { memo, useMemo } from 'react';
+import { memo } from 'react';
 import { PiCaretDownBold } from 'react-icons/pi';
 
 type Props = PropsWithChildren<{
@@ -25,8 +25,6 @@ export const CanvasEntityGroupList = memo(({ isSelected, type, children }: Props
   const title = useEntityTypeTitle(type);
   const informationalPopoverFeature = useEntityTypeInformationalPopover(type);
   const collapse = useBoolean(true);
-  const canMergeVisible = useMemo(() => type === 'raster_layer' || type === 'inpaint_mask', [type]);
-  const canHideAll = useMemo(() => type !== 'reference_image', [type]);
 
   return (
     <Flex flexDir="column" w="full">
@@ -76,8 +74,8 @@ export const CanvasEntityGroupList = memo(({ isSelected, type, children }: Props
 
           <Spacer />
         </Flex>
-        {canMergeVisible && <CanvasEntityMergeVisibleButton type={type} />}
-        {canHideAll && <CanvasEntityTypeIsHiddenToggle type={type} />}
+        {isRenderableEntityType(type) && <CanvasEntityMergeVisibleButton type={type} />}
+        {isRenderableEntityType(type) && <CanvasEntityTypeIsHiddenToggle type={type} />}
         <CanvasEntityAddOfTypeButton type={type} />
       </Flex>
       <Collapse in={collapse.isTrue}>

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityMenuItemsMergeDown.tsx

@@ -0,0 +1,35 @@
+import { MenuItem } from '@invoke-ai/ui-library';
+import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
+import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
+import { useCanvasIsBusy } from 'features/controlLayers/hooks/useCanvasIsBusy';
+import { useEntityIdentifierBelowThisOne } from 'features/controlLayers/hooks/useNextRenderableEntityIdentifier';
+import type { CanvasRenderableEntityType } from 'features/controlLayers/store/types';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { PiStackSimpleBold } from 'react-icons/pi';
+
+export const CanvasEntityMenuItemsMergeDown = memo(() => {
+  const { t } = useTranslation();
+  const canvasManager = useCanvasManager();
+  const isBusy = useCanvasIsBusy();
+  const entityIdentifier = useEntityIdentifierContext<CanvasRenderableEntityType>();
+  const entityIdentifierBelowThisOne = useEntityIdentifierBelowThisOne(entityIdentifier);
+  const mergeDown = useCallback(() => {
+    if (entityIdentifierBelowThisOne === null) {
+      return;
+    }
+    canvasManager.compositor.mergeByEntityIdentifiers([entityIdentifierBelowThisOne, entityIdentifier], true);
+  }, [canvasManager.compositor, entityIdentifier, entityIdentifierBelowThisOne]);
+
+  return (
+    <MenuItem
+      onClick={mergeDown}
+      icon={<PiStackSimpleBold />}
+      isDisabled={isBusy || entityIdentifierBelowThisOne === null}
+    >
+      {t('controlLayers.mergeDown')}
+    </MenuItem>
+  );
+});
+
+CanvasEntityMenuItemsMergeDown.displayName = 'CanvasEntityMenuItemsMergeDown';

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityMergeVisibleButton.tsx

@@ -1,80 +1,24 @@
 import { IconButton } from '@invoke-ai/ui-library';
-import { logger } from 'app/logging/logger';
-import { useAppDispatch } from 'app/store/storeHooks';
-import { withResultAsync } from 'common/util/result';
 import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
 import { useCanvasIsBusy } from 'features/controlLayers/hooks/useCanvasIsBusy';
-import { useEntityTypeCount } from 'features/controlLayers/hooks/useEntityTypeCount';
-import { inpaintMaskAdded, rasterLayerAdded } from 'features/controlLayers/store/canvasSlice';
-import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
-import { imageDTOToImageObject } from 'features/controlLayers/store/util';
-import { toast } from 'features/toast/toast';
+import { useVisibleEntityCountByType } from 'features/controlLayers/hooks/useVisibleEntityCountByType';
+import type { CanvasRenderableEntityType } from 'features/controlLayers/store/types';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import { PiStackBold } from 'react-icons/pi';
-import { serializeError } from 'serialize-error';
-
-const log = logger('canvas');
 
 type Props = {
-  type: CanvasEntityIdentifier['type'];
+  type: CanvasRenderableEntityType;
 };
 
 export const CanvasEntityMergeVisibleButton = memo(({ type }: Props) => {
   const { t } = useTranslation();
-  const dispatch = useAppDispatch();
   const canvasManager = useCanvasManager();
   const isBusy = useCanvasIsBusy();
-  const entityCount = useEntityTypeCount(type);
-  const onClick = useCallback(async () => {
-    if (type === 'raster_layer') {
-      const rect = canvasManager.stage.getVisibleRect('raster_layer');
-      const result = await withResultAsync(() =>
-        canvasManager.compositor.rasterizeAndUploadCompositeRasterLayer(rect, { is_intermediate: true })
-      );
-
-      if (result.isOk()) {
-        dispatch(
-          rasterLayerAdded({
-            isSelected: true,
-            overrides: {
-              objects: [imageDTOToImageObject(result.value)],
-              position: { x: Math.floor(rect.x), y: Math.floor(rect.y) },
-            },
-            isMergingVisible: true,
-          })
-        );
-        toast({ title: t('controlLayers.mergeVisibleOk') });
-      } else {
-        log.error({ error: serializeError(result.error) }, 'Failed to merge visible');
-        toast({ title: t('controlLayers.mergeVisibleError'), status: 'error' });
-      }
-    } else if (type === 'inpaint_mask') {
-      const rect = canvasManager.stage.getVisibleRect('inpaint_mask');
-      const result = await withResultAsync(() =>
-        canvasManager.compositor.rasterizeAndUploadCompositeInpaintMask(rect, false)
-      );
-
-      if (result.isOk()) {
-        dispatch(
-          inpaintMaskAdded({
-            isSelected: true,
-            overrides: {
-              objects: [imageDTOToImageObject(result.value)],
-              position: { x: Math.floor(rect.x), y: Math.floor(rect.y) },
-            },
-            isMergingVisible: true,
-          })
-        );
-        toast({ title: t('controlLayers.mergeVisibleOk') });
-      } else {
-        log.error({ error: serializeError(result.error) }, 'Failed to merge visible');
-        toast({ title: t('controlLayers.mergeVisibleError'), status: 'error' });
-      }
-    } else {
-      log.error({ type }, 'Unsupported type for merge visible');
-    }
-  }, [canvasManager.compositor, canvasManager.stage, dispatch, t, type]);
+  const entityCount = useVisibleEntityCountByType(type);
+  const mergeVisible = useCallback(() => {
+    canvasManager.compositor.mergeVisibleOfType(type);
+  }, [canvasManager.compositor, type]);
 
   return (
     <IconButton
@@ -83,7 +27,7 @@ export const CanvasEntityMergeVisibleButton = memo(({ type }: Props) => {
       tooltip={t('controlLayers.mergeVisible')}
       variant="link"
       icon={<PiStackBold />}
-      onClick={onClick}
+      onClick={mergeVisible}
       alignSelf="stretch"
       isDisabled={entityCount <= 1 || isBusy}
     />

invokeai/frontend/web/src/features/controlLayers/hooks/saveCanvasHooks.ts

@@ -51,7 +51,9 @@ const useSaveCanvas = ({ region, saveToGallery, toastOk, toastError, onSave, wit
 
   const saveCanvas = useCallback(async () => {
     const rect =
-      region === 'bbox' ? canvasManager.stateApi.getBbox().rect : canvasManager.stage.getVisibleRect('raster_layer');
+      region === 'bbox'
+        ? canvasManager.stateApi.getBbox().rect
+        : canvasManager.compositor.getVisibleRectOfType('raster_layer');
 
     if (rect.width === 0 || rect.height === 0) {
       toast({
@@ -68,12 +70,13 @@ const useSaveCanvas = ({ region, saveToGallery, toastOk, toastError, onSave, wit
       metadata = selectCanvasMetadata(store.getState());
     }
 
-    const result = await withResultAsync(() =>
-      canvasManager.compositor.rasterizeAndUploadCompositeRasterLayer(rect, {
+    const result = await withResultAsync(() => {
+      const rasterAdapters = canvasManager.compositor.getVisibleAdaptersOfType('raster_layer');
+      return canvasManager.compositor.getCompositeImageDTO(rasterAdapters, rect, {
         is_intermediate: !saveToGallery,
         metadata,
-      })
-    );
+      });
+    });
 
     if (result.isOk()) {
       if (onSave) {
@@ -86,7 +89,6 @@ const useSaveCanvas = ({ region, saveToGallery, toastOk, toastError, onSave, wit
     }
   }, [
     canvasManager.compositor,
-    canvasManager.stage,
     canvasManager.stateApi,
     onSave,
     region,

invokeai/frontend/web/src/features/controlLayers/hooks/useNextRenderableEntityIdentifier.ts

@@ -0,0 +1,25 @@
+import { createMemoizedSelector } from 'app/store/createMemoizedSelector';
+import { useAppSelector } from 'app/store/storeHooks';
+import { selectCanvasSlice, selectEntityIdentifierBelowThisOne } from 'features/controlLayers/store/selectors';
+import type { CanvasRenderableEntityIdentifier } from 'features/controlLayers/store/types';
+import { getEntityIdentifier } from 'features/controlLayers/store/types';
+import { useMemo } from 'react';
+
+export const useEntityIdentifierBelowThisOne = <T extends CanvasRenderableEntityIdentifier>(
+  entityIdentifier: T
+): T | null => {
+  const selector = useMemo(
+    () =>
+      createMemoizedSelector(selectCanvasSlice, (canvas) => {
+        const nextEntity = selectEntityIdentifierBelowThisOne(canvas, entityIdentifier);
+        if (!nextEntity) {
+          return null;
+        }
+        return getEntityIdentifier(nextEntity);
+      }),
+    [entityIdentifier]
+  );
+  const entityIdentifierBelowThisOne = useAppSelector(selector);
+
+  return entityIdentifierBelowThisOne as T | null;
+};

invokeai/frontend/web/src/features/controlLayers/hooks/useVisibleEntityCountByType.ts

@@ -0,0 +1,33 @@
+import { createSelector } from '@reduxjs/toolkit';
+import { useAppSelector } from 'app/store/storeHooks';
+import {
+  selectActiveControlLayerEntities,
+  selectActiveInpaintMaskEntities,
+  selectActiveRasterLayerEntities,
+  selectActiveReferenceImageEntities,
+  selectActiveRegionalGuidanceEntities,
+} from 'features/controlLayers/store/selectors';
+import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
+import { useMemo } from 'react';
+import { assert } from 'tsafe';
+
+export const useVisibleEntityCountByType = (type: CanvasEntityIdentifier['type']): number => {
+  const selectVisibleEntityCountByType = useMemo(() => {
+    switch (type) {
+      case 'control_layer':
+        return createSelector(selectActiveControlLayerEntities, (entities) => entities.length);
+      case 'raster_layer':
+        return createSelector(selectActiveRasterLayerEntities, (entities) => entities.length);
+      case 'inpaint_mask':
+        return createSelector(selectActiveInpaintMaskEntities, (entities) => entities.length);
+      case 'regional_guidance':
+        return createSelector(selectActiveRegionalGuidanceEntities, (entities) => entities.length);
+      case 'reference_image':
+        return createSelector(selectActiveReferenceImageEntities, (entities) => entities.length);
+      default:
+        assert(false, 'Invalid entity type');
+    }
+  }, [type]);
+  const visibleEntityCount = useAppSelector(selectVisibleEntityCountByType);
+  return visibleEntityCount;
+};

invokeai/frontend/web/src/features/controlLayers/konva/CanvasCacheModule.ts

@@ -1,15 +1,32 @@
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasModuleBase } from 'features/controlLayers/konva/CanvasModuleBase';
+import type { Transparency } from 'features/controlLayers/konva/util';
 import { getPrefixedId } from 'features/controlLayers/konva/util';
 import type { GenerationMode } from 'features/controlLayers/store/types';
 import { LRUCache } from 'lru-cache';
 import type { Logger } from 'roarr';
 
+type GetCacheEntryWithFallbackArg<T extends NonNullable<unknown>> = {
+  cache: LRUCache<string, T>;
+  key: string;
+  getValue: () => Promise<T>;
+  onHit?: (value: T) => void;
+  onMiss?: () => void;
+};
+
 type CanvasCacheModuleConfig = {
   /**
    * The maximum size of the image name cache.
    */
   imageNameCacheSize: number;
+  /**
+   * The maximum size of the image data cache.
+   */
+  imageDataCacheSize: number;
+  /**
+   * The maximum size of the transparency calculation cache.
+   */
+  transparencyCalculationCacheSize: number;
   /**
    * The maximum size of the canvas element cache.
    */
@@ -21,7 +38,9 @@ type CanvasCacheModuleConfig = {
 };
 
 const DEFAULT_CONFIG: CanvasCacheModuleConfig = {
-  imageNameCacheSize: 100,
+  imageNameCacheSize: 1000,
+  imageDataCacheSize: 32,
+  transparencyCalculationCacheSize: 1000,
   canvasElementCacheSize: 32,
   generationModeCacheSize: 100,
 };
@@ -41,26 +60,38 @@ export class CanvasCacheModule extends CanvasModuleBase {
   config: CanvasCacheModuleConfig = DEFAULT_CONFIG;
 
   /**
-   * A cache for storing image names. Used as a cache for results of layer/canvas/entity exports. For example, when we
-   * rasterize a layer and upload it to the server, we store the image name in this cache.
+   * A cache for storing image names.
    *
-   * The cache key is a hash of the exported entity's state and the export rect.
+   * For example, the key might be a hash of a composite of entities with the uploaded image name as the value.
    */
   imageNameCache = new LRUCache<string, string>({ max: this.config.imageNameCacheSize });
 
   /**
-   * A cache for storing canvas elements. Similar to the image name cache, but for canvas elements. The primary use is
-   * for caching composite layers. For example, the canvas compositor module uses this to store the canvas elements for
-   * individual raster layers when creating a composite of the layers.
+   * A cache for storing canvas elements.
    *
-   * The cache key is a hash of the exported entity's state and the export rect.
+   * For example, the key might be a hash of a composite of entities with the canvas element as the value.
    */
   canvasElementCache = new LRUCache<string, HTMLCanvasElement>({ max: this.config.canvasElementCacheSize });
+
   /**
-   * A cache for the generation mode calculation, which is fairly expensive.
+   * A cache for image data objects.
    *
-   * The cache key is a hash of all the objects that contribute to the generation mode calculation (e.g. the composite
-   * raster layer, the composite inpaint mask, and bounding box), and the value is the generation mode.
+   * For example, the key might be a hash of a composite of entities with the image data as the value.
+   */
+  imageDataCache = new LRUCache<string, ImageData>({ max: this.config.imageDataCacheSize });
+
+  /**
+   * A cache for transparency calculation results.
+   *
+   * For example, the key might be a hash of a composite of entities with the transparency as the value.
+   */
+  transparencyCalculationCache = new LRUCache<string, Transparency>({ max: this.config.imageDataCacheSize });
+
+  /**
+   * A cache for generation mode calculation results.
+   *
+   * For example, the key might be a hash of a composite of raster and inpaint mask entities with the generation mode
+   * as the value.
    */
   generationModeCache = new LRUCache<string, GenerationMode>({ max: this.config.generationModeCacheSize });
 
@@ -75,6 +106,33 @@ export class CanvasCacheModule extends CanvasModuleBase {
     this.log.debug('Creating cache module');
   }
 
+  /**
+   * A helper function for getting a cache entry with a fallback.
+   * @param param0.cache The LRUCache to get the entry from.
+   * @param param0.key The key to use to retrieve the entry.
+   * @param param0.getValue An async function to generate the value if the entry is not in the cache.
+   * @param param0.onHit An optional function to call when the entry is in the cache.
+   * @param param0.onMiss An optional function to call when the entry is not in the cache.
+   * @returns
+   */
+  static getWithFallback = async <T extends NonNullable<unknown>>({
+    cache,
+    getValue,
+    key,
+    onHit,
+    onMiss,
+  }: GetCacheEntryWithFallbackArg<T>): Promise<T> => {
+    let value = cache.get(key);
+    if (value === undefined) {
+      onMiss?.();
+      value = await getValue();
+      cache.set(key, value);
+    } else {
+      onHit?.(value);
+    }
+    return value;
+  };
+
   /**
    * Clears all caches.
    */

invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts

@@ -1,24 +1,55 @@
 import type { SerializableObject } from 'common/types';
 import { withResultAsync } from 'common/util/result';
+import { CanvasCacheModule } from 'features/controlLayers/konva/CanvasCacheModule';
+import type { CanvasEntityAdapter, CanvasEntityAdapterFromType } from 'features/controlLayers/konva/CanvasEntity/types';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasModuleBase } from 'features/controlLayers/konva/CanvasModuleBase';
+import type { Transparency } from 'features/controlLayers/konva/util';
 import {
   canvasToBlob,
   canvasToImageData,
   getImageDataTransparency,
   getPrefixedId,
+  getRectUnion,
+  mapId,
   previewBlob,
 } from 'features/controlLayers/konva/util';
-import type { GenerationMode, Rect } from 'features/controlLayers/store/types';
+import {
+  selectActiveControlLayerEntities,
+  selectActiveInpaintMaskEntities,
+  selectActiveRasterLayerEntities,
+  selectActiveRegionalGuidanceEntities,
+} from 'features/controlLayers/store/selectors';
+import type {
+  CanvasRenderableEntityIdentifier,
+  CanvasRenderableEntityState,
+  CanvasRenderableEntityType,
+  GenerationMode,
+  Rect,
+} from 'features/controlLayers/store/types';
+import { getEntityIdentifier } from 'features/controlLayers/store/types';
+import { imageDTOToImageObject } from 'features/controlLayers/store/util';
 import { selectAutoAddBoardId } from 'features/gallery/store/gallerySelectors';
+import { toast } from 'features/toast/toast';
+import { t } from 'i18next';
 import { atom, computed } from 'nanostores';
 import type { Logger } from 'roarr';
+import { serializeError } from 'serialize-error';
 import type { UploadOptions } from 'services/api/endpoints/images';
 import { getImageDTOSafe, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
 import stableHash from 'stable-hash';
+import type { Equals } from 'tsafe';
 import { assert } from 'tsafe';
 
+type CompositingOptions = {
+  /**
+   * The global composite operation to use when compositing each entity.
+   * See: https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation
+   */
+  globalCompositeOperation?: GlobalCompositeOperation;
+};
+
 /**
  * Handles compositing operations:
  * - Rasterizing and uploading the composite raster layer
@@ -54,41 +85,98 @@ export class CanvasCompositorModule extends CanvasModuleBase {
   }
 
   /**
-   * Gets the entity IDs of all raster layers that should be included in the composite raster layer.
-   * A raster layer is included if it is enabled and has objects. The ids are sorted by draw order.
-   * @returns An array of raster layer entity IDs
+   * Gets the rect union of all visible entities of the given entity type. This is used for "merge visible".
+   *
+   * If no entity type is provided, all visible entities are included in the rect.
+   *
+   * @param type The optional entity type
+   * @returns The rect
    */
-  getCompositeRasterLayerEntityIds = (): string[] => {
-    const validSortedIds = [];
-    const sortedIds = this.manager.stateApi.getRasterLayersState().entities.map(({ id }) => id);
-    for (const id of sortedIds) {
-      const adapter = this.manager.adapters.rasterLayers.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Raster layer adapter not found');
+  getVisibleRectOfType = (type?: CanvasRenderableEntityType): Rect => {
+    const rects = [];
+
+    for (const adapter of this.manager.getAllAdapters()) {
+      if (!adapter.state.isEnabled) {
         continue;
       }
-      if (adapter.state.isEnabled && adapter.state.objects.length > 0) {
-        validSortedIds.push(adapter.id);
+      if (type && adapter.state.type !== type) {
+        continue;
+      }
+      if (adapter.renderer.hasObjects()) {
+        rects.push(adapter.transformer.getRelativeRect());
       }
     }
-    return validSortedIds;
+
+    return getRectUnion(...rects);
   };
 
   /**
-   * Gets a hash of the composite raster layer, which includes the state of all raster layers that are included in the
-   * composite plus arbitrary extra data that should contribute to the hash (e.g. a rect).
-   * @param extra Any extra data to include in the hash
-   * @returns A hash for the composite raster layer
+   * Gets the rect union of the given entity adapters. This is used for "merge down" and "merge selected".
+   *
+   * Unlike `getVisibleRectOfType`, **disabled entities are included in the rect**, per the conventional behaviour of
+   * these merge methods.
+   *
+   * @param adapters The entity adapters to include in the rect
+   * @returns The rect
    */
-  getCompositeRasterLayerHash = (extra: SerializableObject): string => {
+  getRectOfAdapters = (adapters: CanvasEntityAdapter[]): Rect => {
+    const rects = [];
+
+    for (const adapter of adapters) {
+      if (adapter.renderer.hasObjects()) {
+        rects.push(adapter.transformer.getRelativeRect());
+      }
+    }
+
+    return getRectUnion(...rects);
+  };
+
+  /**
+   * Gets all visible adapters for the given entity type. Visible adapters are those that are not disabled and have
+   * objects to render. This is used for "merge visible" functionality and for calculating the generation mode.
+   *
+   * This includes all adapters that are not disabled and have objects to render.
+   *
+   * @param type The entity type
+   * @returns The adapters for the given entity type that are eligible to be included in a composite
+   */
+  getVisibleAdaptersOfType = <T extends CanvasRenderableEntityType>(type: T): CanvasEntityAdapterFromType<T>[] => {
+    let entities: CanvasRenderableEntityState[];
+
+    switch (type) {
+      case 'raster_layer':
+        entities = this.manager.stateApi.getRasterLayersState().entities;
+        break;
+      case 'inpaint_mask':
+        entities = this.manager.stateApi.getInpaintMasksState().entities;
+        break;
+      case 'control_layer':
+        entities = this.manager.stateApi.getControlLayersState().entities;
+        break;
+      case 'regional_guidance':
+        entities = this.manager.stateApi.getRegionsState().entities;
+        break;
+      default:
+        assert(false, `Unhandled entity type: ${type}`);
+    }
+
+    const adapters: CanvasEntityAdapter[] = entities
+      // Get the identifier for each entity
+      .map((entity) => getEntityIdentifier(entity))
+      // Get the adapter for each entity
+      .map(this.manager.getAdapter)
+      // Filter out null adapters
+      .filter((adapter) => !!adapter)
+      // Filter out adapters that are disabled or have no objects (and are thus not to be included in the composite)
+      .filter((adapter) => !adapter.$isDisabled.get() && adapter.renderer.hasObjects());
+
+    return adapters as CanvasEntityAdapterFromType<T>[];
+  };
+
+  getCompositeHash = (adapters: CanvasEntityAdapter[], extra: SerializableObject): string => {
     const adapterHashes: SerializableObject[] = [];
 
-    for (const id of this.getCompositeRasterLayerEntityIds()) {
-      const adapter = this.manager.adapters.rasterLayers.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Raster layer adapter not found');
-        continue;
-      }
+    for (const adapter of adapters) {
       adapterHashes.push(adapter.getHashableState());
     }
 
@@ -101,23 +189,33 @@ export class CanvasCompositorModule extends CanvasModuleBase {
   };
 
   /**
-   * Gets a canvas element for the composite raster layer. Only the region defined by the rect is included in the canvas.
+   * Composites the given canvas entities for the given rect and returns the resulting canvas.
    *
-   * If the hash of the composite raster layer is found in the cache, the cached canvas is returned.
+   * The canvas element is cached to avoid recomputing it when the canvas state has not changed.
    *
+   * The canvas entities are drawn in the order they are provided.
+   *
+   * @param adapters The adapters for the canvas entities to composite, in the order they should be drawn
    * @param rect The region to include in the canvas
-   * @returns A canvas element with the composite raster layer drawn on it
+   * @param compositingOptions Options for compositing the entities
+   * @returns The composite canvas
    */
-  getCompositeRasterLayerCanvas = (rect: Rect): HTMLCanvasElement => {
-    const hash = this.getCompositeRasterLayerHash({ rect });
+  getCompositeCanvas = (
+    adapters: CanvasEntityAdapter[],
+    rect: Rect,
+    compositingOptions?: CompositingOptions
+  ): HTMLCanvasElement => {
+    const entityIdentifiers = adapters.map((adapter) => adapter.entityIdentifier);
+
+    const hash = this.getCompositeHash(adapters, { rect });
     const cachedCanvas = this.manager.cache.canvasElementCache.get(hash);
 
     if (cachedCanvas) {
-      this.log.trace({ rect }, 'Using cached composite raster layer canvas');
+      this.log.debug({ entityIdentifiers, rect }, 'Using cached composite canvas');
       return cachedCanvas;
     }
 
-    this.log.trace({ rect }, 'Building composite raster layer canvas');
+    this.log.debug({ entityIdentifiers, rect }, 'Building composite canvas');
     this.$isCompositing.set(true);
 
     const canvas = document.createElement('canvas');
@@ -129,13 +227,12 @@ export class CanvasCompositorModule extends CanvasModuleBase {
 
     ctx.imageSmoothingEnabled = false;
 
-    for (const id of this.getCompositeRasterLayerEntityIds()) {
-      const adapter = this.manager.adapters.rasterLayers.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Raster layer adapter not found');
-        continue;
-      }
-      this.log.trace({ id }, 'Drawing raster layer to composite canvas');
+    if (compositingOptions?.globalCompositeOperation) {
+      ctx.globalCompositeOperation = compositingOptions.globalCompositeOperation;
+    }
+
+    for (const adapter of adapters) {
+      this.log.debug({ entityIdentifier: adapter.entityIdentifier }, 'Drawing entity to composite canvas');
       const adapterCanvas = adapter.getCanvas(rect);
       ctx.drawImage(adapterCanvas, 0, 0);
     }
@@ -145,23 +242,42 @@ export class CanvasCompositorModule extends CanvasModuleBase {
   };
 
   /**
-   * Rasterizes the composite raster layer and uploads it to the server.
+   * Composites the given canvas entities for the given rect and uploads the resulting image.
    *
-   * If the hash of the composite raster layer is found in the cache, the cached image DTO is returned.
+   * The uploaded image is cached to avoid recomputing it when the canvas state has not changed. The canvas elements
+   * created for each entity are also cached to avoid recomputing them when the canvas state has not changed.
    *
+   * The canvas entities are drawn in the order they are provided.
+   *
+   * @param adapters The adapters for the canvas entities to composite, in the order they should be drawn
    * @param rect The region to include in the rasterized image
-   * @param options Options for uploading the image
-   * @returns A promise that resolves to the uploaded image DTO
+   * @param uploadOptions Options for uploading the image
+   * @param compositingOptions Options for compositing the entities
+   * @returns A promise that resolves to the image DTO
    */
-  rasterizeAndUploadCompositeRasterLayer = async (
+  getCompositeImageDTO = async (
+    adapters: CanvasEntityAdapter[],
     rect: Rect,
-    options: Pick<UploadOptions, 'is_intermediate' | 'metadata'>
+    uploadOptions: Pick<UploadOptions, 'is_intermediate' | 'metadata'>,
+    compositingOptions?: CompositingOptions
   ): Promise<ImageDTO> => {
-    this.log.trace({ rect }, 'Rasterizing composite raster layer');
-
     assert(rect.width > 0 && rect.height > 0, 'Unable to rasterize empty rect');
 
-    const canvas = this.getCompositeRasterLayerCanvas(rect);
+    const hash = this.getCompositeHash(adapters, { rect });
+    const cachedImageName = this.manager.cache.imageNameCache.get(hash);
+
+    let imageDTO: ImageDTO | null = null;
+
+    if (cachedImageName) {
+      imageDTO = await getImageDTOSafe(cachedImageName);
+      if (imageDTO) {
+        this.log.debug({ rect, imageName: cachedImageName, imageDTO }, 'Using cached composite image');
+        return imageDTO;
+      }
+      this.log.warn({ rect, imageName: cachedImageName }, 'Cached image name not found, recompositing');
+    }
+
+    const canvas = this.getCompositeCanvas(adapters, rect, compositingOptions);
 
     this.$isProcessing.set(true);
     const blobResult = await withResultAsync(() => canvasToBlob(canvas));
@@ -173,217 +289,163 @@ export class CanvasCompositorModule extends CanvasModuleBase {
     const blob = blobResult.value;
 
     if (this.manager._isDebugging) {
-      previewBlob(blob, 'Composite raster layer canvas');
+      previewBlob(blob, 'Composite');
     }
 
     this.$isUploading.set(true);
     const uploadResult = await withResultAsync(() =>
       uploadImage({
         blob,
-        fileName: 'composite-raster-layer.png',
+        fileName: 'canvas-composite.png',
         image_category: 'general',
-        is_intermediate: options.is_intermediate,
-        board_id: options.is_intermediate ? undefined : selectAutoAddBoardId(this.manager.store.getState()),
-        metadata: options.metadata,
+        is_intermediate: uploadOptions.is_intermediate,
+        board_id: uploadOptions.is_intermediate ? undefined : selectAutoAddBoardId(this.manager.store.getState()),
+        metadata: uploadOptions.metadata,
       })
     );
     this.$isUploading.set(false);
     if (uploadResult.isErr()) {
       throw uploadResult.error;
     }
-    const imageDTO = uploadResult.value;
-    return imageDTO;
-  };
-
-  /**
-   * Gets the image DTO for the composite raster layer.
-   *
-   * If the image is found in the cache, the cached image DTO is returned.
-   *
-   * @param rect The region to include in the image
-   * @returns A promise that resolves to the image DTO
-   */
-  getCompositeRasterLayerImageDTO = async (rect: Rect): Promise<ImageDTO> => {
-    let imageDTO: ImageDTO | null = null;
-
-    const hash = this.getCompositeRasterLayerHash({ rect });
-    const cachedImageName = this.manager.cache.imageNameCache.get(hash);
-
-    if (cachedImageName) {
-      imageDTO = await getImageDTOSafe(cachedImageName);
-      if (imageDTO) {
-        this.log.trace({ rect, imageName: cachedImageName, imageDTO }, 'Using cached composite raster layer image');
-        return imageDTO;
-      }
-    }
-
-    imageDTO = await this.rasterizeAndUploadCompositeRasterLayer(rect, { is_intermediate: true });
+    imageDTO = uploadResult.value;
     this.manager.cache.imageNameCache.set(hash, imageDTO.image_name);
     return imageDTO;
   };
 
   /**
-   * Gets the entity IDs of all inpaint masks that should be included in the composite inpaint mask.
-   * An inpaint mask is included if it is enabled and has objects. The ids are sorted by draw order.
-   * @returns An array of inpaint mask entity IDs
+   * Creates a merged composite image from the given entities. The entities are drawn in the order they are provided.
+   *
+   * The merged image is uploaded to the server and a new entity is created with the uploaded image as the only object.
+   *
+   * All entities must have the same type.
+   *
+   * @param entityIdentifiers The entity identifiers to merge
+   * @param deleteMergedEntities Whether to delete the merged entities after creating the new merged entity
+   * @returns A promise that resolves to the image DTO, or null if the merge failed
    */
-  getCompositeInpaintMaskEntityIds = (): string[] => {
-    const validSortedIds = [];
-    const sortedIds = this.manager.stateApi.getInpaintMasksState().entities.map(({ id }) => id);
-    for (const id of sortedIds) {
-      const adapter = this.manager.adapters.inpaintMasks.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Inpaint mask adapter not found');
-        continue;
-      }
-      if (adapter.state.isEnabled && adapter.state.objects.length > 0) {
-        validSortedIds.push(adapter.id);
-      }
+  mergeByEntityIdentifiers = async <T extends CanvasRenderableEntityIdentifier>(
+    entityIdentifiers: T[],
+    deleteMergedEntities: boolean
+  ): Promise<ImageDTO | null> => {
+    if (entityIdentifiers.length <= 1) {
+      this.log.warn({ entityIdentifiers }, 'Cannot merge less than 2 entities');
+      return null;
     }
-    return validSortedIds;
-  };
+    const type = entityIdentifiers[0]?.type;
+    assert(type, 'Cannot merge entities with no type (this should never happen)');
 
-  /**
-   * Gets a hash of the composite inpaint mask, which includes the state of all inpaint masks that are included in the
-   * composite plus arbitrary extra data that should contribute to the hash (e.g. a rect).
-   * @param extra Any extra data to include in the hash
-   * @returns A hash for the composite inpaint mask
-   */
-  getCompositeInpaintMaskHash = (extra: SerializableObject): string => {
-    const adapterHashes: SerializableObject[] = [];
+    const adapters = this.manager.getAdapters(entityIdentifiers);
+    assert(adapters.length === entityIdentifiers.length, 'Failed to get all adapters for entity identifiers');
 
-    for (const id of this.getCompositeInpaintMaskEntityIds()) {
-      const adapter = this.manager.adapters.inpaintMasks.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Inpaint mask adapter not found');
-        continue;
-      }
-      adapterHashes.push(adapter.getHashableState());
-    }
+    const rect = this.getRectOfAdapters(adapters);
 
-    const data: SerializableObject = {
-      extra,
-      adapterHashes,
+    const compositingOptions: CompositingOptions = {
+      globalCompositeOperation: type === 'control_layer' ? 'lighter' : undefined,
     };
 
-    return stableHash(data);
-  };
-
-  /**
-   * Gets a canvas element for the composite inpaint mask. Only the region defined by the rect is included in the canvas.
-   *
-   * If the hash of the composite inpaint mask is found in the cache, the cached canvas is returned.
-   *
-   * @param rect The region to include in the canvas
-   * @returns A canvas element with the composite inpaint mask drawn on it
-   */
-  getCompositeInpaintMaskCanvas = (rect: Rect): HTMLCanvasElement => {
-    const hash = this.getCompositeInpaintMaskHash({ rect });
-    const cachedCanvas = this.manager.cache.canvasElementCache.get(hash);
-
-    if (cachedCanvas) {
-      this.log.trace({ rect }, 'Using cached composite inpaint mask canvas');
-      return cachedCanvas;
-    }
-
-    this.log.trace({ rect }, 'Building composite inpaint mask canvas');
-    this.$isCompositing.set(true);
-    const canvas = document.createElement('canvas');
-    canvas.width = rect.width;
-    canvas.height = rect.height;
-
-    const ctx = canvas.getContext('2d');
-    assert(ctx !== null);
-
-    ctx.imageSmoothingEnabled = false;
-
-    for (const id of this.getCompositeInpaintMaskEntityIds()) {
-      const adapter = this.manager.adapters.inpaintMasks.get(id);
-      if (!adapter) {
-        this.log.warn({ id }, 'Inpaint mask adapter not found');
-        continue;
-      }
-      this.log.trace({ id }, 'Drawing inpaint mask to composite canvas');
-      const adapterCanvas = adapter.getCanvas(rect);
-      ctx.drawImage(adapterCanvas, 0, 0);
-    }
-    this.manager.cache.canvasElementCache.set(hash, canvas);
-    this.$isCompositing.set(false);
-    return canvas;
-  };
-
-  /**
-   * Rasterizes the composite inpaint mask and uploads it to the server.
-   *
-   * If the hash of the composite inpaint mask is found in the cache, the cached image DTO is returned.
-   *
-   * @param rect The region to include in the rasterized image
-   * @param saveToGallery Whether to save the image to the gallery or just return the uploaded image DTO
-   * @returns A promise that resolves to the uploaded image DTO
-   */
-  rasterizeAndUploadCompositeInpaintMask = async (rect: Rect, saveToGallery: boolean) => {
-    this.log.trace({ rect }, 'Rasterizing composite inpaint mask');
-
-    assert(rect.width > 0 && rect.height > 0, 'Unable to rasterize empty rect');
-
-    const canvas = this.getCompositeInpaintMaskCanvas(rect);
-
-    this.$isProcessing.set(true);
-    const blobResult = await withResultAsync(() => canvasToBlob(canvas));
-    this.$isProcessing.set(false);
-
-    if (blobResult.isErr()) {
-      throw blobResult.error;
-    }
-    const blob = blobResult.value;
-
-    if (this.manager._isDebugging) {
-      previewBlob(blob, 'Composite inpaint mask canvas');
-    }
-
-    this.$isUploading.set(true);
-    const uploadResult = await withResultAsync(() =>
-      uploadImage({
-        blob,
-        fileName: 'composite-inpaint-mask.png',
-        image_category: 'general',
-        is_intermediate: !saveToGallery,
-        board_id: saveToGallery ? selectAutoAddBoardId(this.manager.store.getState()) : undefined,
-      })
+    const result = await withResultAsync(() =>
+      this.getCompositeImageDTO(adapters, rect, { is_intermediate: true }, compositingOptions)
     );
-    this.$isUploading.set(false);
-    if (uploadResult.isErr()) {
-      throw uploadResult.error;
+
+    if (result.isErr()) {
+      this.log.error({ error: serializeError(result.error) }, 'Failed to merge selected entities');
+      toast({ title: t('controlLayers.mergeVisibleError'), status: 'error' });
+      return null;
     }
-    const imageDTO = uploadResult.value;
-    return imageDTO;
+
+    // All layer types have the same arg - create a new entity with the image as the only object, positioned at the
+    // top left corner of the visible rect for the given entity type.
+    const addEntityArg = {
+      isSelected: true,
+      overrides: {
+        objects: [imageDTOToImageObject(result.value)],
+        position: { x: Math.floor(rect.x), y: Math.floor(rect.y) },
+      },
+      mergedEntitiesToDelete: deleteMergedEntities ? entityIdentifiers.map(mapId) : [],
+    };
+
+    switch (type) {
+      case 'raster_layer':
+        this.manager.stateApi.addRasterLayer(addEntityArg);
+        break;
+      case 'inpaint_mask':
+        this.manager.stateApi.addInpaintMask(addEntityArg);
+        break;
+      case 'regional_guidance':
+        this.manager.stateApi.addRegionalGuidance(addEntityArg);
+        break;
+      case 'control_layer':
+        this.manager.stateApi.addControlLayer(addEntityArg);
+        break;
+      default:
+        assert<Equals<typeof type, never>>(false, 'Unsupported type for merge');
+    }
+
+    toast({ title: t('controlLayers.mergeVisibleOk') });
+
+    return result.value;
   };
 
   /**
-   * Gets the image DTO for the composite inpaint mask.
+   * Merges all visible entities of the given type. This is used for "merge visible" functionality.
    *
-   * If the image is found in the cache, the cached image DTO is returned.
-   *
-   * @param rect The region to include in the image
-   * @returns A promise that resolves to the image DTO
+   * @param type The type of entity to merge
+   * @returns A promise that resolves to the image DTO, or null if the merge failed
    */
-  getCompositeInpaintMaskImageDTO = async (rect: Rect): Promise<ImageDTO> => {
-    let imageDTO: ImageDTO | null = null;
+  mergeVisibleOfType = (type: CanvasRenderableEntityType): Promise<ImageDTO | null> => {
+    let entities: CanvasRenderableEntityState[];
 
-    const hash = this.getCompositeInpaintMaskHash({ rect });
-    const cachedImageName = this.manager.cache.imageNameCache.get(hash);
-
-    if (cachedImageName) {
-      imageDTO = await getImageDTOSafe(cachedImageName);
-      if (imageDTO) {
-        this.log.trace({ rect, cachedImageName, imageDTO }, 'Using cached composite inpaint mask image');
-        return imageDTO;
-      }
+    switch (type) {
+      case 'raster_layer':
+        entities = this.manager.stateApi.runSelector(selectActiveRasterLayerEntities);
+        break;
+      case 'inpaint_mask':
+        entities = this.manager.stateApi.runSelector(selectActiveInpaintMaskEntities);
+        break;
+      case 'regional_guidance':
+        entities = this.manager.stateApi.runSelector(selectActiveRegionalGuidanceEntities);
+        break;
+      case 'control_layer':
+        entities = this.manager.stateApi.runSelector(selectActiveControlLayerEntities);
+        break;
+      default:
+        assert<Equals<typeof type, never>>(false, 'Unsupported type for merge');
     }
 
-    imageDTO = await this.rasterizeAndUploadCompositeInpaintMask(rect, false);
-    this.manager.cache.imageNameCache.set(hash, imageDTO.image_name);
-    return imageDTO;
+    const entityIdentifiers = entities.map(getEntityIdentifier);
+
+    return this.mergeByEntityIdentifiers(entityIdentifiers, false);
+  };
+
+  /**
+   * Calculates the transparency of the composite of the give adapters.
+   * @param adapters The adapters to composite
+   * @param rect The region to include in the composite
+   * @param hash The hash to use for caching the result
+   * @returns A promise that resolves to the transparency of the composite
+   */
+  getTransparency = (adapters: CanvasEntityAdapter[], rect: Rect, hash: string): Promise<Transparency> => {
+    const entityIdentifiers = adapters.map((adapter) => adapter.entityIdentifier);
+    const logCtx = { entityIdentifiers, rect };
+    return CanvasCacheModule.getWithFallback({
+      cache: this.manager.cache.transparencyCalculationCache,
+      key: hash,
+      getValue: async () => {
+        const compositeInpaintMaskCanvas = this.getCompositeCanvas(adapters, rect);
+
+        const compositeInpaintMaskImageData = await CanvasCacheModule.getWithFallback({
+          cache: this.manager.cache.imageDataCache,
+          key: hash,
+          getValue: () => Promise.resolve(canvasToImageData(compositeInpaintMaskCanvas)),
+          onHit: () => this.log.trace(logCtx, 'Using cached image data'),
+          onMiss: () => this.log.trace(logCtx, 'Calculating image data'),
+        });
+
+        return getImageDataTransparency(compositeInpaintMaskImageData);
+      },
+      onHit: () => this.log.trace(logCtx, 'Using cached transparency'),
+      onMiss: () => this.log.trace(logCtx, 'Calculating transparency'),
+    });
   };
 
   /**
@@ -404,29 +466,37 @@ export class CanvasCompositorModule extends CanvasModuleBase {
    *
    * @returns The generation mode
    */
-  getGenerationMode(): GenerationMode {
+  getGenerationMode = async (): Promise<GenerationMode> => {
     const { rect } = this.manager.stateApi.getBbox();
 
-    const compositeInpaintMaskHash = this.getCompositeInpaintMaskHash({ rect });
-    const compositeRasterLayerHash = this.getCompositeRasterLayerHash({ rect });
+    const rasterLayerAdapters = this.manager.compositor.getVisibleAdaptersOfType('raster_layer');
+    const compositeRasterLayerHash = this.getCompositeHash(rasterLayerAdapters, { rect });
+
+    const inpaintMaskAdapters = this.manager.compositor.getVisibleAdaptersOfType('inpaint_mask');
+    const compositeInpaintMaskHash = this.getCompositeHash(inpaintMaskAdapters, { rect });
+
     const hash = stableHash({ rect, compositeInpaintMaskHash, compositeRasterLayerHash });
     const cachedGenerationMode = this.manager.cache.generationModeCache.get(hash);
 
     if (cachedGenerationMode) {
-      this.log.trace({ rect, cachedGenerationMode }, 'Using cached generation mode');
+      this.log.debug({ rect, cachedGenerationMode }, 'Using cached generation mode');
       return cachedGenerationMode;
     }
 
-    const compositeInpaintMaskCanvas = this.getCompositeInpaintMaskCanvas(rect);
-    this.$isProcessing.set(true);
-    const compositeInpaintMaskImageData = canvasToImageData(compositeInpaintMaskCanvas);
-    const compositeInpaintMaskTransparency = getImageDataTransparency(compositeInpaintMaskImageData);
-    this.$isProcessing.set(false);
+    this.log.debug({ rect }, 'Calculating generation mode');
 
-    const compositeRasterLayerCanvas = this.getCompositeRasterLayerCanvas(rect);
     this.$isProcessing.set(true);
-    const compositeRasterLayerImageData = canvasToImageData(compositeRasterLayerCanvas);
-    const compositeRasterLayerTransparency = getImageDataTransparency(compositeRasterLayerImageData);
+    const compositeRasterLayerTransparency = await this.getTransparency(
+      rasterLayerAdapters,
+      rect,
+      compositeRasterLayerHash
+    );
+
+    const compositeInpaintMaskTransparency = await this.getTransparency(
+      inpaintMaskAdapters,
+      rect,
+      compositeInpaintMaskHash
+    );
     this.$isProcessing.set(false);
 
     let generationMode: GenerationMode;
@@ -447,7 +517,7 @@ export class CanvasCompositorModule extends CanvasModuleBase {
 
     this.manager.cache.generationModeCache.set(hash, generationMode);
     return generationMode;
-  }
+  };
 
   repr = () => {
     return {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterBase.ts

@@ -97,7 +97,10 @@ export abstract class CanvasEntityAdapterBase<
   abstract getCanvas: (rect?: Rect) => HTMLCanvasElement;
 
   /**
-   * Gets a hashable representation of the entity's state.
+   * Gets a hashable representation of the entity's _renderable_ state. This should exclude any properties that are not
+   * relevant to rendering the entity.
+   *
+   * This is used for caching.
    */
   abstract getHashableState: () => SerializableObject;
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterControlLayer.ts

@@ -78,7 +78,12 @@ export class CanvasEntityAdapterControlLayer extends CanvasEntityAdapterBase<
   };
 
   getHashableState = (): SerializableObject => {
-    const keysToOmit: (keyof CanvasControlLayerState)[] = ['name', 'controlAdapter', 'withTransparencyEffect'];
+    const keysToOmit: (keyof CanvasControlLayerState)[] = [
+      'name',
+      'controlAdapter',
+      'withTransparencyEffect',
+      'isLocked',
+    ];
     return omit(this.state, keysToOmit);
   };
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterInpaintMask.ts

@@ -70,7 +70,7 @@ export class CanvasEntityAdapterInpaintMask extends CanvasEntityAdapterBase<
   };
 
   getHashableState = (): SerializableObject => {
-    const keysToOmit: (keyof CanvasInpaintMaskState)[] = ['fill', 'name', 'opacity'];
+    const keysToOmit: (keyof CanvasInpaintMaskState)[] = ['fill', 'name', 'opacity', 'isLocked'];
     return omit(this.state, keysToOmit);
   };
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer.ts

@@ -71,7 +71,7 @@ export class CanvasEntityAdapterRasterLayer extends CanvasEntityAdapterBase<
   };
 
   getHashableState = (): SerializableObject => {
-    const keysToOmit: (keyof CanvasRasterLayerState)[] = ['name'];
+    const keysToOmit: (keyof CanvasRasterLayerState)[] = ['name', 'isLocked'];
     return omit(this.state, keysToOmit);
   };
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRegionalGuidance.ts

@@ -70,7 +70,16 @@ export class CanvasEntityAdapterRegionalGuidance extends CanvasEntityAdapterBase
   };
 
   getHashableState = (): SerializableObject => {
-    const keysToOmit: (keyof CanvasRegionalGuidanceState)[] = ['fill', 'name', 'opacity'];
+    const keysToOmit: (keyof CanvasRegionalGuidanceState)[] = [
+      'fill',
+      'name',
+      'opacity',
+      'isLocked',
+      'autoNegative',
+      'positivePrompt',
+      'negativePrompt',
+      'referenceImages',
+    ];
     return omit(this.state, keysToOmit);
   };
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityFilterer.ts

@@ -9,7 +9,7 @@ import { addCoords, getKonvaNodeDebugAttrs, getPrefixedId } from 'features/contr
 import { selectAutoProcess } from 'features/controlLayers/store/canvasSettingsSlice';
 import type { FilterConfig } from 'features/controlLayers/store/filters';
 import { getFilterForModel, IMAGE_FILTERS } from 'features/controlLayers/store/filters';
-import type { CanvasEntityType, CanvasImageState } from 'features/controlLayers/store/types';
+import type { CanvasImageState, CanvasRenderableEntityType } from 'features/controlLayers/store/types';
 import { imageDTOToImageObject } from 'features/controlLayers/store/util';
 import Konva from 'konva';
 import { debounce } from 'lodash-es';
@@ -350,7 +350,7 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
    * Saves the filtered image as a new entity of the given type.
    * @param type The type of entity to save the filtered image as.
    */
-  saveAs = (type: Exclude<CanvasEntityType, 'reference_image'>) => {
+  saveAs = (type: CanvasRenderableEntityType) => {
     const imageState = this.$imageState.get();
     if (!imageState) {
       this.log.warn('No image state to apply filter to');

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/types.ts

@@ -2,9 +2,15 @@ import type { CanvasEntityAdapterControlLayer } from 'features/controlLayers/kon
 import type { CanvasEntityAdapterInpaintMask } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterInpaintMask';
 import type { CanvasEntityAdapterRasterLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer';
 import type { CanvasEntityAdapterRegionalGuidance } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRegionalGuidance';
+import type { CanvasRenderableEntityType } from 'features/controlLayers/store/types';
 
 export type CanvasEntityAdapter =
   | CanvasEntityAdapterRasterLayer
   | CanvasEntityAdapterControlLayer
   | CanvasEntityAdapterInpaintMask
   | CanvasEntityAdapterRegionalGuidance;
+
+export type CanvasEntityAdapterFromType<T extends CanvasRenderableEntityType> = Extract<
+  CanvasEntityAdapter,
+  { state: { type: T } }
+>;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts

@@ -8,7 +8,7 @@ import { CanvasEntityAdapterControlLayer } from 'features/controlLayers/konva/Ca
 import { CanvasEntityAdapterInpaintMask } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterInpaintMask';
 import { CanvasEntityAdapterRasterLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer';
 import { CanvasEntityAdapterRegionalGuidance } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRegionalGuidance';
-import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types';
+import type { CanvasEntityAdapter, CanvasEntityAdapterFromType } from 'features/controlLayers/konva/CanvasEntity/types';
 import { CanvasEntityRendererModule } from 'features/controlLayers/konva/CanvasEntityRendererModule';
 import { CanvasModuleBase } from 'features/controlLayers/konva/CanvasModuleBase';
 import { CanvasProgressImageModule } from 'features/controlLayers/konva/CanvasProgressImageModule';
@@ -18,7 +18,11 @@ import { CanvasToolModule } from 'features/controlLayers/konva/CanvasTool/Canvas
 import { CanvasWorkerModule } from 'features/controlLayers/konva/CanvasWorkerModule.js';
 import { getPrefixedId } from 'features/controlLayers/konva/util';
 import { $canvasManager } from 'features/controlLayers/store/ephemeral';
-import type { CanvasEntityIdentifier, CanvasEntityType } from 'features/controlLayers/store/types';
+import type {
+  CanvasEntityIdentifier,
+  CanvasRenderableEntityIdentifier,
+  CanvasRenderableEntityType,
+} from 'features/controlLayers/store/types';
 import {
   isControlLayerEntityIdentifier,
   isInpaintMaskEntityIdentifier,
@@ -135,44 +139,35 @@ export class CanvasManager extends CanvasModuleBase {
     this.konva.previewLayer.add(this.tool.konva.group);
   }
 
-  getAdapter = <T extends CanvasEntityType = CanvasEntityType>(
+  getAdapter = <T extends CanvasRenderableEntityType = CanvasRenderableEntityType>(
     entityIdentifier: CanvasEntityIdentifier<T>
-  ): Extract<CanvasEntityAdapter, { state: { type: T } }> | null => {
+  ): CanvasEntityAdapterFromType<T> | null => {
+    let adapter: CanvasEntityAdapter | undefined;
+
     switch (entityIdentifier.type) {
       case 'raster_layer':
-        return (
-          (this.adapters.rasterLayers.get(entityIdentifier.id) as Extract<
-            CanvasEntityAdapter,
-            { state: { type: T } }
-          >) ?? null
-        );
+        adapter = this.adapters.rasterLayers.get(entityIdentifier.id);
+        break;
       case 'control_layer':
-        return (
-          (this.adapters.controlLayers.get(entityIdentifier.id) as Extract<
-            CanvasEntityAdapter,
-            { state: { type: T } }
-          >) ?? null
-        );
+        adapter = this.adapters.controlLayers.get(entityIdentifier.id);
+        break;
       case 'regional_guidance':
-        return (
-          (this.adapters.regionMasks.get(entityIdentifier.id) as Extract<
-            CanvasEntityAdapter,
-            { state: { type: T } }
-          >) ?? null
-        );
+        adapter = this.adapters.regionMasks.get(entityIdentifier.id);
+        break;
       case 'inpaint_mask':
-        return (
-          (this.adapters.inpaintMasks.get(entityIdentifier.id) as Extract<
-            CanvasEntityAdapter,
-            { state: { type: T } }
-          >) ?? null
-        );
+        adapter = this.adapters.inpaintMasks.get(entityIdentifier.id);
+        break;
       default:
         return null;
     }
+    if (!adapter) {
+      return null;
+    }
+
+    return adapter as CanvasEntityAdapterFromType<T>;
   };
 
-  deleteAdapter = (entityIdentifier: CanvasEntityIdentifier): boolean => {
+  deleteAdapter = (entityIdentifier: CanvasRenderableEntityIdentifier): boolean => {
     switch (entityIdentifier.type) {
       case 'raster_layer':
         return this.adapters.rasterLayers.delete(entityIdentifier.id);
@@ -187,6 +182,18 @@ export class CanvasManager extends CanvasModuleBase {
     }
   };
 
+  getAdapters = (entityIdentifiers: CanvasRenderableEntityIdentifier[]): CanvasEntityAdapter[] => {
+    const adapters: CanvasEntityAdapter[] = [];
+    for (const entityIdentifier of entityIdentifiers) {
+      const adapter = this.getAdapter(entityIdentifier);
+      if (!adapter) {
+        continue;
+      }
+      adapters.push(adapter);
+    }
+    return adapters;
+  };
+
   getAllAdapters = (): CanvasEntityAdapter[] => {
     return [
       ...this.adapters.rasterLayers.values(),
@@ -196,7 +203,7 @@ export class CanvasManager extends CanvasModuleBase {
     ];
   };
 
-  createAdapter = (entityIdentifier: CanvasEntityIdentifier): CanvasEntityAdapter => {
+  createAdapter = (entityIdentifier: CanvasRenderableEntityIdentifier): CanvasEntityAdapter => {
     if (isRasterLayerEntityIdentifier(entityIdentifier)) {
       const adapter = new CanvasEntityAdapterRasterLayer(entityIdentifier, this);
       this.adapters.rasterLayers.set(adapter.id, adapter);

invokeai/frontend/web/src/features/controlLayers/konva/CanvasSegmentAnythingModule.ts

@@ -15,8 +15,8 @@ import {
 } from 'features/controlLayers/konva/util';
 import { selectAutoProcess } from 'features/controlLayers/store/canvasSettingsSlice';
 import type {
-  CanvasEntityType,
   CanvasImageState,
+  CanvasRenderableEntityType,
   Coordinate,
   RgbaColor,
   SAMPointLabel,
@@ -697,7 +697,7 @@ export class CanvasSegmentAnythingModule extends CanvasModuleBase {
    * Saves the segmented image as a new entity of the given type.
    * @param type The type of entity to save the segmented image as.
    */
-  saveAs = (type: Exclude<CanvasEntityType, 'reference_image'>) => {
+  saveAs = (type: CanvasRenderableEntityType) => {
     const imageState = this.$imageState.get();
     if (!imageState) {
       this.log.error('No image state to save as');

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStageModule.ts

@@ -1,14 +1,8 @@
 import type { Property } from 'csstype';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasModuleBase } from 'features/controlLayers/konva/CanvasModuleBase';
-import { getKonvaNodeDebugAttrs, getPrefixedId, getRectUnion } from 'features/controlLayers/konva/util';
-import type {
-  CanvasEntityIdentifier,
-  Coordinate,
-  Dimensions,
-  Rect,
-  StageAttrs,
-} from 'features/controlLayers/store/types';
+import { getKonvaNodeDebugAttrs, getPrefixedId } from 'features/controlLayers/konva/util';
+import type { Coordinate, Dimensions, Rect, StageAttrs } from 'features/controlLayers/store/types';
 import Konva from 'konva';
 import type { KonvaEventObject } from 'konva/lib/Node';
 import { clamp } from 'lodash-es';
@@ -146,24 +140,6 @@ export class CanvasStageModule extends CanvasModuleBase {
     }
   };
 
-  getVisibleRect = (type?: Exclude<CanvasEntityIdentifier['type'], 'ip_adapter'>): Rect => {
-    const rects = [];
-
-    for (const adapter of this.manager.getAllAdapters()) {
-      if (!adapter.state.isEnabled) {
-        continue;
-      }
-      if (type && adapter.state.type !== type) {
-        continue;
-      }
-      if (adapter.renderer.hasObjects()) {
-        rects.push(adapter.transformer.getRelativeRect());
-      }
-    }
-
-    return getRectUnion(...rects);
-  };
-
   /**
    * Fits the bbox to the stage. This will center the bbox and scale it to fit the stage with some padding.
    */
@@ -177,7 +153,7 @@ export class CanvasStageModule extends CanvasModuleBase {
    * Fits the visible canvas to the stage. This will center the canvas and scale it to fit the stage with some padding.
    */
   fitLayersToStage = (): void => {
-    const rect = this.getVisibleRect();
+    const rect = this.manager.compositor.getVisibleRectOfType();
     if (rect.width === 0 || rect.height === 0) {
       this.fitBboxToStage();
     } else {

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApiModule.ts

@@ -47,7 +47,7 @@ import type {
   Rect,
   RgbaColor,
 } from 'features/controlLayers/store/types';
-import { RGBA_BLACK } from 'features/controlLayers/store/types';
+import { isRenderableEntityIdentifier, RGBA_BLACK } from 'features/controlLayers/store/types';
 import type { Graph } from 'features/nodes/util/graph/generation/Graph';
 import { atom, computed } from 'nanostores';
 import type { Logger } from 'roarr';
@@ -583,10 +583,13 @@ export class CanvasStateApiModule extends CanvasModuleBase {
    */
   getSelectedEntityAdapter = (): CanvasEntityAdapter | null => {
     const state = this.getCanvasState();
-    if (state.selectedEntityIdentifier) {
-      return this.manager.getAdapter(state.selectedEntityIdentifier);
+    if (!state.selectedEntityIdentifier) {
+      return null;
     }
-    return null;
+    if (!isRenderableEntityIdentifier(state.selectedEntityIdentifier)) {
+      return null;
+    }
+    return this.manager.getAdapter(state.selectedEntityIdentifier);
   };
 
   /**

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasBboxToolModule.ts

@@ -390,7 +390,7 @@ export class CanvasBboxToolModule extends CanvasModuleBase {
   };
 
   fitToLayers = (): void => {
-    const visibleRect = this.manager.stage.getVisibleRect();
+    const visibleRect = this.manager.compositor.getVisibleRectOfType();
 
     // Can't fit the bbox to nothing
     if (visibleRect.height === 0 || visibleRect.width === 0) {

invokeai/frontend/web/src/features/controlLayers/store/canvasSlice.ts

@@ -123,27 +123,28 @@ export const canvasSlice = createSlice({
           id: string;
           overrides?: Partial<CanvasRasterLayerState>;
           isSelected?: boolean;
-          isMergingVisible?: boolean;
+          mergedEntitiesToDelete?: string[];
         }>
       ) => {
-        const { id, overrides, isSelected, isMergingVisible } = action.payload;
+        const { id, overrides, isSelected, mergedEntitiesToDelete = [] } = action.payload;
         const entityState = getRasterLayerState(id, overrides);
 
-        if (isMergingVisible) {
-          // When merging visible, we delete all disabled layers
-          state.rasterLayers.entities = state.rasterLayers.entities.filter((layer) => !layer.isEnabled);
-        }
-
         state.rasterLayers.entities.push(entityState);
 
-        if (isSelected) {
+        if (mergedEntitiesToDelete.length > 0) {
+          state.rasterLayers.entities = state.rasterLayers.entities.filter(
+            (entity) => !mergedEntitiesToDelete.includes(entity.id)
+          );
+        }
+
+        if (isSelected || mergedEntitiesToDelete.length > 0) {
           state.selectedEntityIdentifier = getEntityIdentifier(entityState);
         }
       },
       prepare: (payload: {
         overrides?: Partial<CanvasRasterLayerState>;
         isSelected?: boolean;
-        isMergingVisible?: boolean;
+        mergedEntitiesToDelete?: string[];
       }) => ({
         payload: { ...payload, id: getPrefixedId('raster_layer') },
       }),
@@ -271,19 +272,34 @@ export const canvasSlice = createSlice({
     controlLayerAdded: {
       reducer: (
         state,
-        action: PayloadAction<{ id: string; overrides?: Partial<CanvasControlLayerState>; isSelected?: boolean }>
+        action: PayloadAction<{
+          id: string;
+          overrides?: Partial<CanvasControlLayerState>;
+          isSelected?: boolean;
+          mergedEntitiesToDelete?: string[];
+        }>
       ) => {
-        const { id, overrides, isSelected } = action.payload;
+        const { id, overrides, isSelected, mergedEntitiesToDelete = [] } = action.payload;
 
         const entityState = getControlLayerState(id, overrides);
 
         state.controlLayers.entities.push(entityState);
 
-        if (isSelected) {
+        if (mergedEntitiesToDelete.length > 0) {
+          state.controlLayers.entities = state.controlLayers.entities.filter(
+            (entity) => !mergedEntitiesToDelete.includes(entity.id)
+          );
+        }
+
+        if (isSelected || mergedEntitiesToDelete.length > 0) {
           state.selectedEntityIdentifier = getEntityIdentifier(entityState);
         }
       },
-      prepare: (payload: { overrides?: Partial<CanvasControlLayerState>; isSelected?: boolean }) => ({
+      prepare: (payload: {
+        overrides?: Partial<CanvasControlLayerState>;
+        isSelected?: boolean;
+        mergedEntitiesToDelete?: string[];
+      }) => ({
         payload: { ...payload, id: getPrefixedId('control_layer') },
       }),
     },
@@ -595,19 +611,34 @@ export const canvasSlice = createSlice({
     rgAdded: {
       reducer: (
         state,
-        action: PayloadAction<{ id: string; overrides?: Partial<CanvasRegionalGuidanceState>; isSelected?: boolean }>
+        action: PayloadAction<{
+          id: string;
+          overrides?: Partial<CanvasRegionalGuidanceState>;
+          isSelected?: boolean;
+          mergedEntitiesToDelete?: string[];
+        }>
       ) => {
-        const { id, overrides, isSelected } = action.payload;
+        const { id, overrides, isSelected, mergedEntitiesToDelete = [] } = action.payload;
 
         const entityState = getRegionalGuidanceState(id, overrides);
 
         state.regionalGuidance.entities.push(entityState);
 
-        if (isSelected) {
+        if (mergedEntitiesToDelete.length > 0) {
+          state.regionalGuidance.entities = state.regionalGuidance.entities.filter(
+            (entity) => !mergedEntitiesToDelete.includes(entity.id)
+          );
+        }
+
+        if (isSelected || mergedEntitiesToDelete.length > 0) {
           state.selectedEntityIdentifier = getEntityIdentifier(entityState);
         }
       },
-      prepare: (payload?: { overrides?: Partial<CanvasRegionalGuidanceState>; isSelected?: boolean }) => ({
+      prepare: (payload?: {
+        overrides?: Partial<CanvasRegionalGuidanceState>;
+        isSelected?: boolean;
+        mergedEntitiesToDelete?: string[];
+      }) => ({
         payload: { ...payload, id: getPrefixedId('regional_guidance') },
       }),
     },
@@ -822,28 +853,29 @@ export const canvasSlice = createSlice({
           id: string;
           overrides?: Partial<CanvasInpaintMaskState>;
           isSelected?: boolean;
-          isMergingVisible?: boolean;
+          mergedEntitiesToDelete?: string[];
         }>
       ) => {
-        const { id, overrides, isSelected, isMergingVisible } = action.payload;
+        const { id, overrides, isSelected, mergedEntitiesToDelete = [] } = action.payload;
 
         const entityState = getInpaintMaskState(id, overrides);
 
-        if (isMergingVisible) {
-          // When merging visible, we delete all disabled layers
-          state.inpaintMasks.entities = state.inpaintMasks.entities.filter((layer) => !layer.isEnabled);
-        }
-
         state.inpaintMasks.entities.push(entityState);
 
-        if (isSelected) {
+        if (mergedEntitiesToDelete.length > 0) {
+          state.inpaintMasks.entities = state.inpaintMasks.entities.filter(
+            (entity) => !mergedEntitiesToDelete.includes(entity.id)
+          );
+        }
+
+        if (isSelected || mergedEntitiesToDelete.length > 0) {
           state.selectedEntityIdentifier = getEntityIdentifier(entityState);
         }
       },
       prepare: (payload?: {
         overrides?: Partial<CanvasInpaintMaskState>;
         isSelected?: boolean;
-        isMergingVisible?: boolean;
+        mergedEntitiesToDelete?: string[];
       }) => ({
         payload: { ...payload, id: getPrefixedId('inpaint_mask') },
       }),

invokeai/frontend/web/src/features/controlLayers/store/selectors.ts

@@ -12,6 +12,8 @@ import type {
   CanvasMetadata,
   CanvasRasterLayerState,
   CanvasRegionalGuidanceState,
+  CanvasRenderableEntityIdentifier,
+  CanvasRenderableEntityState,
   CanvasState,
 } from 'features/controlLayers/store/types';
 import { isRasterLayerEntityIdentifier } from 'features/controlLayers/store/types';
@@ -43,23 +45,25 @@ const selectEntityCountAll = createSelector(selectCanvasSlice, (canvas) => {
   );
 });
 
-const selectActiveRasterLayerEntities = createSelector(selectCanvasSlice, (canvas) =>
-  canvas.rasterLayers.entities.filter((e) => e.isEnabled && e.objects.length > 0)
+const isVisibleEntity = (entity: CanvasRenderableEntityState) => entity.isEnabled && entity.objects.length > 0;
+
+export const selectActiveRasterLayerEntities = createSelector(selectCanvasSlice, (canvas) =>
+  canvas.rasterLayers.entities.filter(isVisibleEntity)
 );
 
-const selectActiveControlLayerEntities = createSelector(selectCanvasSlice, (canvas) =>
-  canvas.controlLayers.entities.filter((e) => e.isEnabled && e.objects.length > 0)
+export const selectActiveControlLayerEntities = createSelector(selectCanvasSlice, (canvas) =>
+  canvas.controlLayers.entities.filter(isVisibleEntity)
 );
 
-const selectActiveInpaintMaskEntities = createSelector(selectCanvasSlice, (canvas) =>
-  canvas.inpaintMasks.entities.filter((e) => e.isEnabled && e.objects.length > 0)
+export const selectActiveInpaintMaskEntities = createSelector(selectCanvasSlice, (canvas) =>
+  canvas.inpaintMasks.entities.filter(isVisibleEntity)
 );
 
-const selectActiveRegionalGuidanceEntities = createSelector(selectCanvasSlice, (canvas) =>
-  canvas.regionalGuidance.entities.filter((e) => e.isEnabled && e.objects.length > 0)
+export const selectActiveRegionalGuidanceEntities = createSelector(selectCanvasSlice, (canvas) =>
+  canvas.regionalGuidance.entities.filter(isVisibleEntity)
 );
 
-const selectActiveIPAdapterEntities = createSelector(selectCanvasSlice, (canvas) =>
+export const selectActiveReferenceImageEntities = createSelector(selectCanvasSlice, (canvas) =>
   canvas.referenceImages.entities.filter((e) => e.isEnabled)
 );
 
@@ -78,7 +82,7 @@ export const selectEntityCountActive = createSelector(
   selectActiveControlLayerEntities,
   selectActiveInpaintMaskEntities,
   selectActiveRegionalGuidanceEntities,
-  selectActiveIPAdapterEntities,
+  selectActiveReferenceImageEntities,
   (
     activeRasterLayerEntities,
     activeControlLayerEntities,
@@ -148,7 +152,46 @@ export function selectEntity<T extends CanvasEntityIdentifier>(
   }
 
   // This cast is safe, but TS seems to be unable to infer the type
-  return entity as Extract<CanvasEntityState, T>;
+  return entity as Extract<CanvasEntityState, T> | undefined;
+}
+
+/**
+ * Selects the entity identifier for the entity that is below the given entity in terms of draw order.
+ */
+export function selectEntityIdentifierBelowThisOne<T extends CanvasRenderableEntityIdentifier>(
+  state: CanvasState,
+  entityIdentifier: T
+): Extract<CanvasEntityState, T> | undefined {
+  const { id, type } = entityIdentifier;
+
+  let entities: CanvasRenderableEntityState[];
+
+  switch (type) {
+    case 'raster_layer': {
+      entities = state.rasterLayers.entities;
+      break;
+    }
+    case 'control_layer': {
+      entities = state.controlLayers.entities;
+      break;
+    }
+    case 'inpaint_mask': {
+      entities = state.inpaintMasks.entities;
+      break;
+    }
+    case 'regional_guidance': {
+      entities = state.regionalGuidance.entities;
+      break;
+    }
+  }
+
+  // Must reverse to get the draw order
+  const reversedEntities = entities.toReversed();
+  const idx = reversedEntities.findIndex((entity) => entity.id === id);
+  const entity = reversedEntities.at(idx + 1);
+
+  // This cast is safe, but TS seems to be unable to infer the type
+  return entity as Extract<CanvasEntityState, T> | undefined;
 }
 
 export const selectRasterLayerEntities = createSelector(selectCanvasSlice, (canvas) => canvas.rasterLayers.entities);

invokeai/frontend/web/src/features/controlLayers/store/types.ts

@@ -332,6 +332,7 @@ const zCanvasRenderableEntityState = z.discriminatedUnion('type', [
   zCanvasInpaintMaskState,
 ]);
 export type CanvasRenderableEntityState = z.infer<typeof zCanvasRenderableEntityState>;
+export type CanvasRenderableEntityType = CanvasRenderableEntityState['type'];
 
 const zCanvasEntityType = z.union([
   zCanvasRasterLayerState.shape.type,
@@ -347,7 +348,7 @@ export const zCanvasEntityIdentifer = z.object({
   type: zCanvasEntityType,
 });
 export type CanvasEntityIdentifier<T extends CanvasEntityType = CanvasEntityType> = { id: string; type: T };
-
+export type CanvasRenderableEntityIdentifier = CanvasEntityIdentifier<CanvasRenderableEntityType>;
 export type LoRA = {
   id: string;
   isEnabled: boolean;
@@ -465,7 +466,7 @@ export type EntityRasterizedPayload = EntityIdentifierPayload<{
 
 export type GenerationMode = 'txt2img' | 'img2img' | 'inpaint' | 'outpaint';
 
-function isRenderableEntityType(
+export function isRenderableEntityType(
   entityType: CanvasEntityState['type']
 ): entityType is CanvasRenderableEntityState['type'] {
   return (
@@ -537,6 +538,12 @@ export function isRenderableEntity(entity: CanvasEntityState): entity is CanvasR
   return isRenderableEntityType(entity.type);
 }
 
+export function isRenderableEntityIdentifier(
+  entityIdentifier: CanvasEntityIdentifier
+): entityIdentifier is CanvasRenderableEntityIdentifier {
+  return isRenderableEntityType(entityIdentifier.type);
+}
+
 export const getEntityIdentifier = <T extends CanvasEntityType>(
   entity: Extract<CanvasEntityState, { type: T }>
 ): CanvasEntityIdentifier<T> => {

invokeai/frontend/web/src/features/nodes/util/graph/generation/addImageToImage.ts

@@ -32,8 +32,8 @@ export const addImageToImage = async ({
   fp32,
 }: AddImageToImageArg): Promise<Invocation<'img_resize' | 'l2i' | 'flux_vae_decode'>> => {
   denoise.denoising_start = denoising_start;
-
-  const { image_name } = await manager.compositor.getCompositeRasterLayerImageDTO(bbox.rect);
+  const adapters = manager.compositor.getVisibleAdaptersOfType('raster_layer');
+  const { image_name } = await manager.compositor.getCompositeImageDTO(adapters, bbox.rect, { is_intermediate: true });
 
   if (!isEqual(scaledSize, originalSize)) {
     // Resize the initial image to the scaled size, denoise, then resize back to the original size

invokeai/frontend/web/src/features/nodes/util/graph/generation/addInpaint.ts

@@ -45,8 +45,15 @@ export const addInpaint = async ({
 
   const { bbox } = canvas;
 
-  const initialImage = await manager.compositor.getCompositeRasterLayerImageDTO(bbox.rect);
-  const maskImage = await manager.compositor.getCompositeInpaintMaskImageDTO(bbox.rect);
+  const rasterAdapters = manager.compositor.getVisibleAdaptersOfType('raster_layer');
+  const initialImage = await manager.compositor.getCompositeImageDTO(rasterAdapters, bbox.rect, {
+    is_intermediate: true,
+  });
+
+  const inpaintMaskAdapters = manager.compositor.getVisibleAdaptersOfType('inpaint_mask');
+  const maskImage = await manager.compositor.getCompositeImageDTO(inpaintMaskAdapters, bbox.rect, {
+    is_intermediate: true,
+  });
 
   if (!isEqual(scaledSize, originalSize)) {
     // Scale before processing requires some resizing

invokeai/frontend/web/src/features/nodes/util/graph/generation/addOutpaint.ts

@@ -45,8 +45,16 @@ export const addOutpaint = async ({
 
   const { bbox } = canvas;
 
-  const initialImage = await manager.compositor.getCompositeRasterLayerImageDTO(bbox.rect);
-  const maskImage = await manager.compositor.getCompositeInpaintMaskImageDTO(bbox.rect);
+  const rasterAdapters = manager.compositor.getVisibleAdaptersOfType('raster_layer');
+  const initialImage = await manager.compositor.getCompositeImageDTO(rasterAdapters, bbox.rect, {
+    is_intermediate: true,
+  });
+
+  const inpaintMaskAdapters = manager.compositor.getVisibleAdaptersOfType('inpaint_mask');
+  const maskImage = await manager.compositor.getCompositeImageDTO(inpaintMaskAdapters, bbox.rect, {
+    is_intermediate: true,
+  });
+
   const infill = getInfill(g, params);
 
   if (!isEqual(scaledSize, originalSize)) {

invokeai/frontend/web/src/features/nodes/util/graph/generation/buildFLUXGraph.ts

@@ -34,7 +34,7 @@ export const buildFLUXGraph = async (
   state: RootState,
   manager: CanvasManager
 ): Promise<{ g: Graph; noise: Invocation<'noise' | 'flux_denoise'>; posCond: Invocation<'flux_text_encoder'> }> => {
-  const generationMode = manager.compositor.getGenerationMode();
+  const generationMode = await manager.compositor.getGenerationMode();
   log.debug({ generationMode }, 'Building FLUX graph');
 
   const params = selectParamsSlice(state);

invokeai/frontend/web/src/features/nodes/util/graph/generation/buildSD1Graph.ts

@@ -37,7 +37,7 @@ export const buildSD1Graph = async (
   state: RootState,
   manager: CanvasManager
 ): Promise<{ g: Graph; noise: Invocation<'noise'>; posCond: Invocation<'compel'> }> => {
-  const generationMode = manager.compositor.getGenerationMode();
+  const generationMode = await manager.compositor.getGenerationMode();
   log.debug({ generationMode }, 'Building SD1/SD2 graph');
 
   const params = selectParamsSlice(state);

invokeai/frontend/web/src/features/nodes/util/graph/generation/buildSDXLGraph.ts

@@ -37,7 +37,7 @@ export const buildSDXLGraph = async (
   state: RootState,
   manager: CanvasManager
 ): Promise<{ g: Graph; noise: Invocation<'noise'>; posCond: Invocation<'sdxl_compel_prompt'> }> => {
-  const generationMode = manager.compositor.getGenerationMode();
+  const generationMode = await manager.compositor.getGenerationMode();
   log.debug({ generationMode }, 'Building SDXL graph');
 
   const params = selectParamsSlice(state);

invokeai/version/invokeai_version.py

@@ -1 +1 @@
-__version__ = "5.3.0"
+__version__ = "5.3.1rc1"