chore: bump version to v5.10.0dev3

This allows it to pull in sentencepiece on its own. In 0.10.0, it didn't have this package listed as a dependency, but in recent releases it does. So we are able to remove sentencepiece as an explicit dep.

.dockerignore

@@ -1,9 +1,11 @@
 *
 !invokeai
 !pyproject.toml
+!uv.lock
 !docker/docker-entrypoint.sh
 !LICENSE
 
+**/dist
 **/node_modules
 **/__pycache__
-**/*.egg-info
+**/*.egg-info

.github/workflows/build-container.yml

@@ -97,6 +97,8 @@ jobs:
           context: .
           file: docker/Dockerfile
           platforms: ${{ env.PLATFORMS }}
+          build-args: |
+            GPU_DRIVER=${{ matrix.gpu-driver }}
           push: ${{ github.ref == 'refs/heads/main' || github.ref_type == 'tag' || github.event.inputs.push-to-registry }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/build-installer.yml

@@ -17,7 +17,7 @@ jobs:
       - name: setup python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: pyproject.toml
 

.nvmrc

@@ -1 +1 @@
-v22.12.0
+v22.14.0

docker/Dockerfile

@@ -1,77 +1,6 @@
 # syntax=docker/dockerfile:1.4
 
-## Builder stage
-
-FROM library/ubuntu:24.04 AS builder
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt update && apt-get install -y \
-        build-essential \
-        git
-
-# Install `uv` for package management
-COPY --from=ghcr.io/astral-sh/uv:0.6.0 /uv /uvx /bin/
-
-ENV VIRTUAL_ENV=/opt/venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ENV INVOKEAI_SRC=/opt/invokeai
-ENV PYTHON_VERSION=3.11
-ENV UV_PYTHON=3.11
-ENV UV_COMPILE_BYTECODE=1
-ENV UV_LINK_MODE=copy
-ENV UV_PROJECT_ENVIRONMENT="$VIRTUAL_ENV"
-ENV UV_INDEX="https://download.pytorch.org/whl/cu124"
-
-ARG GPU_DRIVER=cuda
-# unused but available
-ARG BUILDPLATFORM
-
-# Switch to the `ubuntu` user to work around dependency issues with uv-installed python
-RUN mkdir -p ${VIRTUAL_ENV} && \
-    mkdir -p ${INVOKEAI_SRC} && \
-    chmod -R a+w /opt && \
-    mkdir ~ubuntu/.cache && chown ubuntu: ~ubuntu/.cache
-USER ubuntu
-
-# Install python
-RUN --mount=type=cache,target=/home/ubuntu/.cache/uv,uid=1000,gid=1000 \
-  uv python install ${PYTHON_VERSION}
-
-WORKDIR ${INVOKEAI_SRC}
-
-# Install project's dependencies as a separate layer so they aren't rebuilt every commit.
-# bind-mount instead of copy to defer adding sources to the image until next layer.
-#
-# NOTE: there are no pytorch builds for arm64 + cuda, only cpu
-# x86_64/CUDA is the default
-RUN --mount=type=cache,target=/home/ubuntu/.cache/uv,uid=1000,gid=1000 \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    --mount=type=bind,source=invokeai/version,target=invokeai/version \
-    if [ "$TARGETPLATFORM" = "linux/arm64" ] || [ "$GPU_DRIVER" = "cpu" ]; then \
-      UV_INDEX="https://download.pytorch.org/whl/cpu"; \
-    elif [ "$GPU_DRIVER" = "rocm" ]; then \
-      UV_INDEX="https://download.pytorch.org/whl/rocm6.1"; \
-    fi && \
-    uv sync --no-install-project
-
-# Now that the bulk of the dependencies have been installed, copy in the project files that change more frequently.
-COPY invokeai invokeai
-COPY pyproject.toml .
-
-RUN --mount=type=cache,target=/home/ubuntu/.cache/uv,uid=1000,gid=1000 \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    if [ "$TARGETPLATFORM" = "linux/arm64" ] || [ "$GPU_DRIVER" = "cpu" ]; then \
-      UV_INDEX="https://download.pytorch.org/whl/cpu"; \
-    elif [ "$GPU_DRIVER" = "rocm" ]; then \
-      UV_INDEX="https://download.pytorch.org/whl/rocm6.1"; \
-    fi && \
-    uv sync
-
-
-#### Build the Web UI ------------------------------------
+#### Web UI ------------------------------------
 
 FROM docker.io/node:22-slim AS web-builder
 ENV PNPM_HOME="/pnpm"
@@ -85,69 +14,89 @@ RUN --mount=type=cache,target=/pnpm/store \
     pnpm install --frozen-lockfile
 RUN npx vite build
 
-#### Runtime stage ---------------------------------------
+## Backend ---------------------------------------
 
-FROM library/ubuntu:24.04 AS runtime
+FROM library/ubuntu:24.04
 
 ARG DEBIAN_FRONTEND=noninteractive
-ENV PYTHONUNBUFFERED=1
-ENV PYTHONDONTWRITEBYTECODE=1
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt update && apt install -y --no-install-recommends \
+    ca-certificates \
+    git \
+    gosu \
+    libglib2.0-0 \
+    libgl1 \
+    libglx-mesa0 \
+    build-essential \
+    libopencv-dev \
+    libstdc++-10-dev
 
-RUN apt update && apt install -y --no-install-recommends \
-        git \
-        curl \
-        vim \
-        tmux \
-        ncdu \
-        iotop \
-        bzip2 \
-        gosu \
-        magic-wormhole \
-        libglib2.0-0 \
-        libgl1 \
-        libglx-mesa0 \
-        build-essential \
-        libopencv-dev \
-        libstdc++-10-dev &&\
-    apt-get clean && apt-get autoclean
+ENV \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    VIRTUAL_ENV=/opt/venv \
+    INVOKEAI_SRC=/opt/invokeai \
+    PYTHON_VERSION=3.12 \
+    UV_PYTHON=3.12 \
+    UV_COMPILE_BYTECODE=1 \
+    UV_MANAGED_PYTHON=1 \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_INDEX="https://download.pytorch.org/whl/cu124" \
+    INVOKEAI_ROOT=/invokeai \
+    INVOKEAI_HOST=0.0.0.0 \
+    INVOKEAI_PORT=9090 \
+    PATH="/opt/venv/bin:$PATH" \
+    CONTAINER_UID=${CONTAINER_UID:-1000} \
+    CONTAINER_GID=${CONTAINER_GID:-1000}
 
-ENV INVOKEAI_SRC=/opt/invokeai
-ENV VIRTUAL_ENV=/opt/venv
-ENV UV_PROJECT_ENVIRONMENT="$VIRTUAL_ENV"
-ENV PYTHON_VERSION=3.11
-ENV INVOKEAI_ROOT=/invokeai
-ENV INVOKEAI_HOST=0.0.0.0
-ENV INVOKEAI_PORT=9090
-ENV PATH="$VIRTUAL_ENV/bin:$INVOKEAI_SRC:$PATH"
-ENV CONTAINER_UID=${CONTAINER_UID:-1000}
-ENV CONTAINER_GID=${CONTAINER_GID:-1000}
+ARG GPU_DRIVER=cuda
 
 # Install `uv` for package management
-# and install python for the ubuntu user (expected to exist on ubuntu >=24.x)
-# this is too tiny to optimize with multi-stage builds, but maybe we'll come back to it
-COPY --from=ghcr.io/astral-sh/uv:0.6.0 /uv /uvx /bin/
-USER ubuntu
-RUN uv python install ${PYTHON_VERSION}
-USER root
+COPY --from=ghcr.io/astral-sh/uv:0.6.9 /uv /uvx /bin/
 
-# --link requires buldkit w/ dockerfile syntax 1.4
-COPY --link --from=builder ${INVOKEAI_SRC} ${INVOKEAI_SRC}
-COPY --link --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-COPY --link --from=web-builder /build/dist ${INVOKEAI_SRC}/invokeai/frontend/web/dist
-
-# Link amdgpu.ids for ROCm builds
-# contributed by https://github.com/Rubonnek
-RUN mkdir -p "/opt/amdgpu/share/libdrm" &&\
-  ln -s "/usr/share/libdrm/amdgpu.ids" "/opt/amdgpu/share/libdrm/amdgpu.ids"
+# Install python & allow non-root user to use it by traversing the /root dir without read permissions
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv python install ${PYTHON_VERSION} && \
+    # chmod --recursive a+rX /root/.local/share/uv/python
+    chmod 711 /root
 
 WORKDIR ${INVOKEAI_SRC}
 
+# Install project's dependencies as a separate layer so they aren't rebuilt every commit.
+# bind-mount instead of copy to defer adding sources to the image until next layer.
+#
+# NOTE: there are no pytorch builds for arm64 + cuda, only cpu
+# x86_64/CUDA is the default
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    # this is just to get the package manager to recognize that the project exists, without making changes to the docker layer
+    --mount=type=bind,source=invokeai/version,target=invokeai/version \
+    if [ "$TARGETPLATFORM" = "linux/arm64" ] || [ "$GPU_DRIVER" = "cpu" ]; then UV_INDEX="https://download.pytorch.org/whl/cpu"; \
+    elif [ "$GPU_DRIVER" = "rocm" ]; then UV_INDEX="https://download.pytorch.org/whl/rocm6.2"; \
+    fi && \
+    uv sync --frozen
+
 # build patchmatch
 RUN cd /usr/lib/$(uname -p)-linux-gnu/pkgconfig/ && ln -sf opencv4.pc opencv.pc
 RUN python -c "from patchmatch import patch_match"
 
+# Link amdgpu.ids for ROCm builds
+# contributed by https://github.com/Rubonnek
+RUN mkdir -p "/opt/amdgpu/share/libdrm" &&\
+    ln -s "/usr/share/libdrm/amdgpu.ids" "/opt/amdgpu/share/libdrm/amdgpu.ids"
+
 RUN mkdir -p ${INVOKEAI_ROOT} && chown -R ${CONTAINER_UID}:${CONTAINER_GID} ${INVOKEAI_ROOT}
 
 COPY docker/docker-entrypoint.sh ./
 ENTRYPOINT ["/opt/invokeai/docker-entrypoint.sh"]
 CMD ["invokeai-web"]
+
+# --link requires buldkit w/ dockerfile syntax 1.4, does not work with podman
+COPY --link --from=web-builder /build/dist ${INVOKEAI_SRC}/invokeai/frontend/web/dist
+
+# add sources last to minimize image changes on code changes
+COPY invokeai ${INVOKEAI_SRC}/invokeai

docs/contributing/dev-environment.md

@@ -41,7 +41,7 @@ If you just want to use Invoke, you should use the [launcher][launcher link].
      With the modifications made, the install command should look something like this:
 
       ```sh
-      uv pip install -e ".[dev,test,docs,xformers]" --python 3.11 --python-preference only-managed --index=https://download.pytorch.org/whl/cu124 --reinstall
+      uv pip install -e ".[dev,test,docs,xformers]" --python 3.12 --python-preference only-managed --index=https://download.pytorch.org/whl/cu124 --reinstall
       ```
 
 6. At this point, you should have Invoke installed, a venv set up and activated, and the server running. But you will see a warning in the terminal that no UI was found. If you go to the URL for the server, you won't get a UI.

docs/installation/manual.md

@@ -43,10 +43,10 @@ The following commands vary depending on the version of Invoke being installed a
 3. Create a virtual environment in that directory:
 
     ```sh
-    uv venv --relocatable --prompt invoke --python 3.11 --python-preference only-managed .venv
+    uv venv --relocatable --prompt invoke --python 3.12 --python-preference only-managed .venv
     ```
 
-    This command creates a portable virtual environment at `.venv` complete with a portable python 3.11. It doesn't matter if your system has no python installed, or has a different version - `uv` will handle everything.
+    This command creates a portable virtual environment at `.venv` complete with a portable python 3.12. It doesn't matter if your system has no python installed, or has a different version - `uv` will handle everything.
 
 4. Activate the virtual environment:
 
@@ -88,13 +88,13 @@ The following commands vary depending on the version of Invoke being installed a
 8. Install the `invokeai` package. Substitute the package specifier and version.
 
     ```sh
-    uv pip install <PACKAGE_SPECIFIER>==<VERSION> --python 3.11 --python-preference only-managed --force-reinstall
+    uv pip install <PACKAGE_SPECIFIER>==<VERSION> --python 3.12 --python-preference only-managed --force-reinstall
     ```
 
     If you determined you needed to use a `PyPI` index URL in the previous step, you'll need to add `--index=<INDEX_URL>` like this:
 
     ```sh
-    uv pip install <PACKAGE_SPECIFIER>==<VERSION> --python 3.11 --python-preference only-managed --index=<INDEX_URL> --force-reinstall
+    uv pip install <PACKAGE_SPECIFIER>==<VERSION> --python 3.12 --python-preference only-managed --index=<INDEX_URL> --force-reinstall
     ```
 
 9. Deactivate and reactivate your venv so that the invokeai-specific commands become available in the environment:

docs/installation/requirements.md

@@ -41,7 +41,7 @@ The requirements below are rough guidelines for best performance. GPUs with less
 
     You don't need to do this if you are installing with the [Invoke Launcher](./quick_start.md).
 
-Invoke requires python 3.10 or 3.11. If you don't already have one of these versions installed, we suggest installing 3.11, as it will be supported for longer.
+Invoke requires python 3.10 through 3.12. If you don't already have one of these versions installed, we suggest installing 3.12, as it will be supported for longer.
 
 Check that your system has an up-to-date Python installed by running `python3 --version` in the terminal (Linux, macOS) or cmd/powershell (Windows).
 
@@ -49,19 +49,19 @@ Check that your system has an up-to-date Python installed by running `python3 --
 
     === "Windows"
 
-        - Install python 3.11 with [an official installer].
+        - Install python with [an official installer].
         - The installer includes an option to add python to your PATH. Be sure to enable this. If you missed it, re-run the installer, choose to modify an existing installation, and tick that checkbox.
         - You may need to install [Microsoft Visual C++ Redistributable].
 
     === "macOS"
 
-        - Install python 3.11 with [an official installer].
+        - Install python with [an official installer].
         - If model installs fail with a certificate error, you may need to run this command (changing the python version to match what you have installed): `/Applications/Python\ 3.10/Install\ Certificates.command`
         - If you haven't already, you will need to install the XCode CLI Tools by running `xcode-select --install` in a terminal.
 
     === "Linux"
 
-        - Installing python varies depending on your system. On Ubuntu, you can use the [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa).
+        - Installing python varies depending on your system. We recommend [using `uv` to manage your python installation](https://docs.astral.sh/uv/concepts/python-versions/#installing-a-python-version).
         - You'll need to install `libglib2.0-0` and `libgl1-mesa-glx` for OpenCV to work. For example, on a Debian system: `sudo apt update && sudo apt install -y libglib2.0-0 libgl1-mesa-glx`
 
 ## Drivers

invokeai/app/api/dependencies.py

@@ -37,7 +37,13 @@ from invokeai.app.services.style_preset_records.style_preset_records_sqlite impo
 from invokeai.app.services.urls.urls_default import LocalUrlService
 from invokeai.app.services.workflow_records.workflow_records_sqlite import SqliteWorkflowRecordsStorage
 from invokeai.app.services.workflow_thumbnails.workflow_thumbnails_disk import WorkflowThumbnailFileStorageDisk
-from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningFieldData
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
+    BasicConditioningInfo,
+    ConditioningFieldData,
+    FLUXConditioningInfo,
+    SD3ConditioningInfo,
+    SDXLConditioningInfo,
+)
 from invokeai.backend.util.logging import InvokeAILogger
 from invokeai.version.invokeai_version import __version__
 
@@ -101,10 +107,25 @@ class ApiDependencies:
         images = ImageService()
         invocation_cache = MemoryInvocationCache(max_cache_size=config.node_cache_size)
         tensors = ObjectSerializerForwardCache(
-            ObjectSerializerDisk[torch.Tensor](output_folder / "tensors", ephemeral=True)
+            ObjectSerializerDisk[torch.Tensor](
+                output_folder / "tensors",
+                safe_globals=[torch.Tensor],
+                ephemeral=True,
+            ),
+            max_cache_size=0,
         )
         conditioning = ObjectSerializerForwardCache(
-            ObjectSerializerDisk[ConditioningFieldData](output_folder / "conditioning", ephemeral=True)
+            ObjectSerializerDisk[ConditioningFieldData](
+                output_folder / "conditioning",
+                safe_globals=[
+                    ConditioningFieldData,
+                    BasicConditioningInfo,
+                    SDXLConditioningInfo,
+                    FLUXConditioningInfo,
+                    SD3ConditioningInfo,
+                ],
+                ephemeral=True,
+            ),
         )
         download_queue_service = DownloadQueueService(app_config=configuration, event_bus=events)
         model_images_service = ModelImageFileStorageDisk(model_images_folder / "model_images")

invokeai/app/invocations/controlnet.py

@@ -0,0 +1,128 @@
+# Invocations for ControlNet image preprocessors
+# initial implementation by Gregg Helt, 2023
+from typing import List, Union
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from invokeai.app.invocations.baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    Classification,
+    invocation,
+    invocation_output,
+)
+from invokeai.app.invocations.fields import (
+    FieldDescriptions,
+    ImageField,
+    InputField,
+    OutputField,
+    UIType,
+)
+from invokeai.app.invocations.model import ModelIdentifierField
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.invocations.util import validate_begin_end_step, validate_weights
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.app.util.controlnet_utils import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES, heuristic_resize
+from invokeai.backend.image_util.util import np_to_pil, pil_to_np
+
+
+class ControlField(BaseModel):
+    image: ImageField = Field(description="The control image")
+    control_model: ModelIdentifierField = Field(description="The ControlNet model to use")
+    control_weight: Union[float, List[float]] = Field(default=1, description="The weight given to the ControlNet")
+    begin_step_percent: float = Field(
+        default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
+    )
+    end_step_percent: float = Field(
+        default=1, ge=0, le=1, description="When the ControlNet is last applied (% of total steps)"
+    )
+    control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode to use")
+    resize_mode: CONTROLNET_RESIZE_VALUES = Field(default="just_resize", description="The resize mode to use")
+
+    @field_validator("control_weight")
+    @classmethod
+    def validate_control_weight(cls, v):
+        validate_weights(v)
+        return v
+
+    @model_validator(mode="after")
+    def validate_begin_end_step_percent(self):
+        validate_begin_end_step(self.begin_step_percent, self.end_step_percent)
+        return self
+
+
+@invocation_output("control_output")
+class ControlOutput(BaseInvocationOutput):
+    """node output for ControlNet info"""
+
+    # Outputs
+    control: ControlField = OutputField(description=FieldDescriptions.control)
+
+
+@invocation("controlnet", title="ControlNet - SD1.5, SDXL", tags=["controlnet"], category="controlnet", version="1.1.3")
+class ControlNetInvocation(BaseInvocation):
+    """Collects ControlNet info to pass to other nodes"""
+
+    image: ImageField = InputField(description="The control image")
+    control_model: ModelIdentifierField = InputField(
+        description=FieldDescriptions.controlnet_model, ui_type=UIType.ControlNetModel
+    )
+    control_weight: Union[float, List[float]] = InputField(
+        default=1.0, ge=-1, le=2, description="The weight given to the ControlNet"
+    )
+    begin_step_percent: float = InputField(
+        default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
+    )
+    end_step_percent: float = InputField(
+        default=1, ge=0, le=1, description="When the ControlNet is last applied (% of total steps)"
+    )
+    control_mode: CONTROLNET_MODE_VALUES = InputField(default="balanced", description="The control mode used")
+    resize_mode: CONTROLNET_RESIZE_VALUES = InputField(default="just_resize", description="The resize mode used")
+
+    @field_validator("control_weight")
+    @classmethod
+    def validate_control_weight(cls, v):
+        validate_weights(v)
+        return v
+
+    @model_validator(mode="after")
+    def validate_begin_end_step_percent(self) -> "ControlNetInvocation":
+        validate_begin_end_step(self.begin_step_percent, self.end_step_percent)
+        return self
+
+    def invoke(self, context: InvocationContext) -> ControlOutput:
+        return ControlOutput(
+            control=ControlField(
+                image=self.image,
+                control_model=self.control_model,
+                control_weight=self.control_weight,
+                begin_step_percent=self.begin_step_percent,
+                end_step_percent=self.end_step_percent,
+                control_mode=self.control_mode,
+                resize_mode=self.resize_mode,
+            ),
+        )
+
+
+@invocation(
+    "heuristic_resize",
+    title="Heuristic Resize",
+    tags=["image, controlnet"],
+    category="image",
+    version="1.0.1",
+    classification=Classification.Prototype,
+)
+class HeuristicResizeInvocation(BaseInvocation):
+    """Resize an image using a heuristic method. Preserves edge maps."""
+
+    image: ImageField = InputField(description="The image to resize")
+    width: int = InputField(default=512, ge=1, description="The width to resize to (px)")
+    height: int = InputField(default=512, ge=1, description="The height to resize to (px)")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+        np_img = pil_to_np(image)
+        np_resized = heuristic_resize(np_img, (self.width, self.height))
+        resized = np_to_pil(np_resized)
+        image_dto = context.images.save(image=resized)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/controlnet_image_processors.py

@@ -1,716 +0,0 @@
-# Invocations for ControlNet image preprocessors
-# initial implementation by Gregg Helt, 2023
-# heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux
-from builtins import bool, float
-from pathlib import Path
-from typing import Dict, List, Literal, Union
-
-import cv2
-import numpy as np
-from controlnet_aux import (
-    ContentShuffleDetector,
-    LeresDetector,
-    MediapipeFaceDetector,
-    MidasDetector,
-    MLSDdetector,
-    NormalBaeDetector,
-    PidiNetDetector,
-    SamDetector,
-    ZoeDetector,
-)
-from controlnet_aux.util import HWC3, ade_palette
-from PIL import Image
-from pydantic import BaseModel, Field, field_validator, model_validator
-from transformers import pipeline
-from transformers.pipelines import DepthEstimationPipeline
-
-from invokeai.app.invocations.baseinvocation import (
-    BaseInvocation,
-    BaseInvocationOutput,
-    Classification,
-    invocation,
-    invocation_output,
-)
-from invokeai.app.invocations.fields import (
-    FieldDescriptions,
-    ImageField,
-    InputField,
-    OutputField,
-    UIType,
-    WithBoard,
-    WithMetadata,
-)
-from invokeai.app.invocations.model import ModelIdentifierField
-from invokeai.app.invocations.primitives import ImageOutput
-from invokeai.app.invocations.util import validate_begin_end_step, validate_weights
-from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.app.util.controlnet_utils import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES, heuristic_resize
-from invokeai.backend.image_util.canny import get_canny_edges
-from invokeai.backend.image_util.depth_anything.depth_anything_pipeline import DepthAnythingPipeline
-from invokeai.backend.image_util.dw_openpose import DWPOSE_MODELS, DWOpenposeDetector
-from invokeai.backend.image_util.hed import HEDProcessor
-from invokeai.backend.image_util.lineart import LineartProcessor
-from invokeai.backend.image_util.lineart_anime import LineartAnimeProcessor
-from invokeai.backend.image_util.util import np_to_pil, pil_to_np
-
-
-class ControlField(BaseModel):
-    image: ImageField = Field(description="The control image")
-    control_model: ModelIdentifierField = Field(description="The ControlNet model to use")
-    control_weight: Union[float, List[float]] = Field(default=1, description="The weight given to the ControlNet")
-    begin_step_percent: float = Field(
-        default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
-    )
-    end_step_percent: float = Field(
-        default=1, ge=0, le=1, description="When the ControlNet is last applied (% of total steps)"
-    )
-    control_mode: CONTROLNET_MODE_VALUES = Field(default="balanced", description="The control mode to use")
-    resize_mode: CONTROLNET_RESIZE_VALUES = Field(default="just_resize", description="The resize mode to use")
-
-    @field_validator("control_weight")
-    @classmethod
-    def validate_control_weight(cls, v):
-        validate_weights(v)
-        return v
-
-    @model_validator(mode="after")
-    def validate_begin_end_step_percent(self):
-        validate_begin_end_step(self.begin_step_percent, self.end_step_percent)
-        return self
-
-
-@invocation_output("control_output")
-class ControlOutput(BaseInvocationOutput):
-    """node output for ControlNet info"""
-
-    # Outputs
-    control: ControlField = OutputField(description=FieldDescriptions.control)
-
-
-@invocation("controlnet", title="ControlNet - SD1.5, SDXL", tags=["controlnet"], category="controlnet", version="1.1.3")
-class ControlNetInvocation(BaseInvocation):
-    """Collects ControlNet info to pass to other nodes"""
-
-    image: ImageField = InputField(description="The control image")
-    control_model: ModelIdentifierField = InputField(
-        description=FieldDescriptions.controlnet_model, ui_type=UIType.ControlNetModel
-    )
-    control_weight: Union[float, List[float]] = InputField(
-        default=1.0, ge=-1, le=2, description="The weight given to the ControlNet"
-    )
-    begin_step_percent: float = InputField(
-        default=0, ge=0, le=1, description="When the ControlNet is first applied (% of total steps)"
-    )
-    end_step_percent: float = InputField(
-        default=1, ge=0, le=1, description="When the ControlNet is last applied (% of total steps)"
-    )
-    control_mode: CONTROLNET_MODE_VALUES = InputField(default="balanced", description="The control mode used")
-    resize_mode: CONTROLNET_RESIZE_VALUES = InputField(default="just_resize", description="The resize mode used")
-
-    @field_validator("control_weight")
-    @classmethod
-    def validate_control_weight(cls, v):
-        validate_weights(v)
-        return v
-
-    @model_validator(mode="after")
-    def validate_begin_end_step_percent(self) -> "ControlNetInvocation":
-        validate_begin_end_step(self.begin_step_percent, self.end_step_percent)
-        return self
-
-    def invoke(self, context: InvocationContext) -> ControlOutput:
-        return ControlOutput(
-            control=ControlField(
-                image=self.image,
-                control_model=self.control_model,
-                control_weight=self.control_weight,
-                begin_step_percent=self.begin_step_percent,
-                end_step_percent=self.end_step_percent,
-                control_mode=self.control_mode,
-                resize_mode=self.resize_mode,
-            ),
-        )
-
-
-# This invocation exists for other invocations to subclass it - do not register with @invocation!
-class ImageProcessorInvocation(BaseInvocation, WithMetadata, WithBoard):
-    """Base class for invocations that preprocess images for ControlNet"""
-
-    image: ImageField = InputField(description="The image to process")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        # superclass just passes through image without processing
-        return image
-
-    def load_image(self, context: InvocationContext) -> Image.Image:
-        # allows override for any special formatting specific to the preprocessor
-        return context.images.get_pil(self.image.image_name, "RGB")
-
-    def invoke(self, context: InvocationContext) -> ImageOutput:
-        self._context = context
-        raw_image = self.load_image(context)
-        # image type should be PIL.PngImagePlugin.PngImageFile ?
-        processed_image = self.run_processor(raw_image)
-
-        # currently can't see processed image in node UI without a showImage node,
-        #    so for now setting image_type to RESULT instead of INTERMEDIATE so will get saved in gallery
-        image_dto = context.images.save(image=processed_image)
-
-        """Builds an ImageOutput and its ImageField"""
-        processed_image_field = ImageField(image_name=image_dto.image_name)
-        return ImageOutput(
-            image=processed_image_field,
-            # width=processed_image.width,
-            width=image_dto.width,
-            # height=processed_image.height,
-            height=image_dto.height,
-            # mode=processed_image.mode,
-        )
-
-
-@invocation(
-    "canny_image_processor",
-    title="Canny Processor",
-    tags=["controlnet", "canny"],
-    category="controlnet",
-    version="1.3.3",
-    classification=Classification.Deprecated,
-)
-class CannyImageProcessorInvocation(ImageProcessorInvocation):
-    """Canny edge detection for ControlNet"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    low_threshold: int = InputField(
-        default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)"
-    )
-    high_threshold: int = InputField(
-        default=200, ge=0, le=255, description="The high threshold of the Canny pixel gradient (0-255)"
-    )
-
-    def load_image(self, context: InvocationContext) -> Image.Image:
-        # Keep alpha channel for Canny processing to detect edges of transparent areas
-        return context.images.get_pil(self.image.image_name, "RGBA")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        processed_image = get_canny_edges(
-            image,
-            self.low_threshold,
-            self.high_threshold,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-        )
-        return processed_image
-
-
-@invocation(
-    "hed_image_processor",
-    title="HED (softedge) Processor",
-    tags=["controlnet", "hed", "softedge"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class HedImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies HED edge detection to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    # safe not supported in controlnet_aux v0.0.3
-    # safe: bool = InputField(default=False, description=FieldDescriptions.safe_mode)
-    scribble: bool = InputField(default=False, description=FieldDescriptions.scribble_mode)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        hed_processor = HEDProcessor()
-        processed_image = hed_processor.run(
-            image,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-            # safe not supported in controlnet_aux v0.0.3
-            # safe=self.safe,
-            scribble=self.scribble,
-        )
-        return processed_image
-
-
-@invocation(
-    "lineart_image_processor",
-    title="Lineart Processor",
-    tags=["controlnet", "lineart"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class LineartImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies line art processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    coarse: bool = InputField(default=False, description="Whether to use coarse mode")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        lineart_processor = LineartProcessor()
-        processed_image = lineart_processor.run(
-            image, detect_resolution=self.detect_resolution, image_resolution=self.image_resolution, coarse=self.coarse
-        )
-        return processed_image
-
-
-@invocation(
-    "lineart_anime_image_processor",
-    title="Lineart Anime Processor",
-    tags=["controlnet", "lineart", "anime"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies line art anime processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        processor = LineartAnimeProcessor()
-        processed_image = processor.run(
-            image,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-        )
-        return processed_image
-
-
-@invocation(
-    "midas_depth_image_processor",
-    title="Midas Depth Processor",
-    tags=["controlnet", "midas"],
-    category="controlnet",
-    version="1.2.4",
-    classification=Classification.Deprecated,
-)
-class MidasDepthImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies Midas depth processing to image"""
-
-    a_mult: float = InputField(default=2.0, ge=0, description="Midas parameter `a_mult` (a = a_mult * PI)")
-    bg_th: float = InputField(default=0.1, ge=0, description="Midas parameter `bg_th`")
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    # depth_and_normal not supported in controlnet_aux v0.0.3
-    # depth_and_normal: bool = InputField(default=False, description="whether to use depth and normal mode")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        # TODO: replace from_pretrained() calls with context.models.download_and_cache() (or similar)
-        midas_processor = MidasDetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = midas_processor(
-            image,
-            a=np.pi * self.a_mult,
-            bg_th=self.bg_th,
-            image_resolution=self.image_resolution,
-            detect_resolution=self.detect_resolution,
-            # dept_and_normal not supported in controlnet_aux v0.0.3
-            # depth_and_normal=self.depth_and_normal,
-        )
-        return processed_image
-
-
-@invocation(
-    "normalbae_image_processor",
-    title="Normal BAE Processor",
-    tags=["controlnet"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class NormalbaeImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies NormalBae processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        normalbae_processor = NormalBaeDetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = normalbae_processor(
-            image, detect_resolution=self.detect_resolution, image_resolution=self.image_resolution
-        )
-        return processed_image
-
-
-@invocation(
-    "mlsd_image_processor",
-    title="MLSD Processor",
-    tags=["controlnet", "mlsd"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class MlsdImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies MLSD processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    thr_v: float = InputField(default=0.1, ge=0, description="MLSD parameter `thr_v`")
-    thr_d: float = InputField(default=0.1, ge=0, description="MLSD parameter `thr_d`")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        mlsd_processor = MLSDdetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = mlsd_processor(
-            image,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-            thr_v=self.thr_v,
-            thr_d=self.thr_d,
-        )
-        return processed_image
-
-
-@invocation(
-    "pidi_image_processor",
-    title="PIDI Processor",
-    tags=["controlnet", "pidi"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class PidiImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies PIDI processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    safe: bool = InputField(default=False, description=FieldDescriptions.safe_mode)
-    scribble: bool = InputField(default=False, description=FieldDescriptions.scribble_mode)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        pidi_processor = PidiNetDetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = pidi_processor(
-            image,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-            safe=self.safe,
-            scribble=self.scribble,
-        )
-        return processed_image
-
-
-@invocation(
-    "content_shuffle_image_processor",
-    title="Content Shuffle Processor",
-    tags=["controlnet", "contentshuffle"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies content shuffle processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-    h: int = InputField(default=512, ge=0, description="Content shuffle `h` parameter")
-    w: int = InputField(default=512, ge=0, description="Content shuffle `w` parameter")
-    f: int = InputField(default=256, ge=0, description="Content shuffle `f` parameter")
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        content_shuffle_processor = ContentShuffleDetector()
-        processed_image = content_shuffle_processor(
-            image,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-            h=self.h,
-            w=self.w,
-            f=self.f,
-        )
-        return processed_image
-
-
-# should work with controlnet_aux >= 0.0.4 and timm <= 0.6.13
-@invocation(
-    "zoe_depth_image_processor",
-    title="Zoe (Depth) Processor",
-    tags=["controlnet", "zoe", "depth"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class ZoeDepthImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies Zoe depth processing to image"""
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        zoe_depth_processor = ZoeDetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = zoe_depth_processor(image)
-        return processed_image
-
-
-@invocation(
-    "mediapipe_face_processor",
-    title="Mediapipe Face Processor",
-    tags=["controlnet", "mediapipe", "face"],
-    category="controlnet",
-    version="1.2.4",
-    classification=Classification.Deprecated,
-)
-class MediapipeFaceProcessorInvocation(ImageProcessorInvocation):
-    """Applies mediapipe face processing to image"""
-
-    max_faces: int = InputField(default=1, ge=1, description="Maximum number of faces to detect")
-    min_confidence: float = InputField(default=0.5, ge=0, le=1, description="Minimum confidence for face detection")
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        mediapipe_face_processor = MediapipeFaceDetector()
-        processed_image = mediapipe_face_processor(
-            image,
-            max_faces=self.max_faces,
-            min_confidence=self.min_confidence,
-            image_resolution=self.image_resolution,
-            detect_resolution=self.detect_resolution,
-        )
-        return processed_image
-
-
-@invocation(
-    "leres_image_processor",
-    title="Leres (Depth) Processor",
-    tags=["controlnet", "leres", "depth"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class LeresImageProcessorInvocation(ImageProcessorInvocation):
-    """Applies leres processing to image"""
-
-    thr_a: float = InputField(default=0, description="Leres parameter `thr_a`")
-    thr_b: float = InputField(default=0, description="Leres parameter `thr_b`")
-    boost: bool = InputField(default=False, description="Whether to use boost mode")
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        leres_processor = LeresDetector.from_pretrained("lllyasviel/Annotators")
-        processed_image = leres_processor(
-            image,
-            thr_a=self.thr_a,
-            thr_b=self.thr_b,
-            boost=self.boost,
-            detect_resolution=self.detect_resolution,
-            image_resolution=self.image_resolution,
-        )
-        return processed_image
-
-
-@invocation(
-    "tile_image_processor",
-    title="Tile Resample Processor",
-    tags=["controlnet", "tile"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class TileResamplerProcessorInvocation(ImageProcessorInvocation):
-    """Tile resampler processor"""
-
-    # res: int = InputField(default=512, ge=0, le=1024, description="The pixel resolution for each tile")
-    down_sampling_rate: float = InputField(default=1.0, ge=1.0, le=8.0, description="Down sampling rate")
-
-    # tile_resample copied from sd-webui-controlnet/scripts/processor.py
-    def tile_resample(
-        self,
-        np_img: np.ndarray,
-        res=512,  # never used?
-        down_sampling_rate=1.0,
-    ):
-        np_img = HWC3(np_img)
-        if down_sampling_rate < 1.1:
-            return np_img
-        H, W, C = np_img.shape
-        H = int(float(H) / float(down_sampling_rate))
-        W = int(float(W) / float(down_sampling_rate))
-        np_img = cv2.resize(np_img, (W, H), interpolation=cv2.INTER_AREA)
-        return np_img
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        np_img = np.array(image, dtype=np.uint8)
-        processed_np_image = self.tile_resample(
-            np_img,
-            # res=self.tile_size,
-            down_sampling_rate=self.down_sampling_rate,
-        )
-        processed_image = Image.fromarray(processed_np_image)
-        return processed_image
-
-
-@invocation(
-    "segment_anything_processor",
-    title="Segment Anything Processor",
-    tags=["controlnet", "segmentanything"],
-    category="controlnet",
-    version="1.2.4",
-    classification=Classification.Deprecated,
-)
-class SegmentAnythingProcessorInvocation(ImageProcessorInvocation):
-    """Applies segment anything processing to image"""
-
-    detect_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.detect_res)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        # segment_anything_processor = SamDetector.from_pretrained("ybelkada/segment-anything", subfolder="checkpoints")
-        segment_anything_processor = SamDetectorReproducibleColors.from_pretrained(
-            "ybelkada/segment-anything", subfolder="checkpoints"
-        )
-        np_img = np.array(image, dtype=np.uint8)
-        processed_image = segment_anything_processor(
-            np_img, image_resolution=self.image_resolution, detect_resolution=self.detect_resolution
-        )
-        return processed_image
-
-
-class SamDetectorReproducibleColors(SamDetector):
-    # overriding SamDetector.show_anns() method to use reproducible colors for segmentation image
-    #     base class show_anns() method randomizes colors,
-    #     which seems to also lead to non-reproducible image generation
-    # so using ADE20k color palette instead
-    def show_anns(self, anns: List[Dict]):
-        if len(anns) == 0:
-            return
-        sorted_anns = sorted(anns, key=(lambda x: x["area"]), reverse=True)
-        h, w = anns[0]["segmentation"].shape
-        final_img = Image.fromarray(np.zeros((h, w, 3), dtype=np.uint8), mode="RGB")
-        palette = ade_palette()
-        for i, ann in enumerate(sorted_anns):
-            m = ann["segmentation"]
-            img = np.empty((m.shape[0], m.shape[1], 3), dtype=np.uint8)
-            # doing modulo just in case number of annotated regions exceeds number of colors in palette
-            ann_color = palette[i % len(palette)]
-            img[:, :] = ann_color
-            final_img.paste(Image.fromarray(img, mode="RGB"), (0, 0), Image.fromarray(np.uint8(m * 255)))
-        return np.array(final_img, dtype=np.uint8)
-
-
-@invocation(
-    "color_map_image_processor",
-    title="Color Map Processor",
-    tags=["controlnet"],
-    category="controlnet",
-    version="1.2.3",
-    classification=Classification.Deprecated,
-)
-class ColorMapImageProcessorInvocation(ImageProcessorInvocation):
-    """Generates a color map from the provided image"""
-
-    color_map_tile_size: int = InputField(default=64, ge=1, description=FieldDescriptions.tile_size)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        np_image = np.array(image, dtype=np.uint8)
-        height, width = np_image.shape[:2]
-
-        width_tile_size = min(self.color_map_tile_size, width)
-        height_tile_size = min(self.color_map_tile_size, height)
-
-        color_map = cv2.resize(
-            np_image,
-            (width // width_tile_size, height // height_tile_size),
-            interpolation=cv2.INTER_CUBIC,
-        )
-        color_map = cv2.resize(color_map, (width, height), interpolation=cv2.INTER_NEAREST)
-        color_map = Image.fromarray(color_map)
-        return color_map
-
-
-DEPTH_ANYTHING_MODEL_SIZES = Literal["large", "base", "small", "small_v2"]
-# DepthAnything V2 Small model is licensed under Apache 2.0 but not the base and large models.
-DEPTH_ANYTHING_MODELS = {
-    "large": "LiheYoung/depth-anything-large-hf",
-    "base": "LiheYoung/depth-anything-base-hf",
-    "small": "LiheYoung/depth-anything-small-hf",
-    "small_v2": "depth-anything/Depth-Anything-V2-Small-hf",
-}
-
-
-@invocation(
-    "depth_anything_image_processor",
-    title="Depth Anything Processor",
-    tags=["controlnet", "depth", "depth anything"],
-    category="controlnet",
-    version="1.1.3",
-    classification=Classification.Deprecated,
-)
-class DepthAnythingImageProcessorInvocation(ImageProcessorInvocation):
-    """Generates a depth map based on the Depth Anything algorithm"""
-
-    model_size: DEPTH_ANYTHING_MODEL_SIZES = InputField(
-        default="small_v2", description="The size of the depth model to use"
-    )
-    resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        def load_depth_anything(model_path: Path):
-            depth_anything_pipeline = pipeline(model=str(model_path), task="depth-estimation", local_files_only=True)
-            assert isinstance(depth_anything_pipeline, DepthEstimationPipeline)
-            return DepthAnythingPipeline(depth_anything_pipeline)
-
-        with self._context.models.load_remote_model(
-            source=DEPTH_ANYTHING_MODELS[self.model_size], loader=load_depth_anything
-        ) as depth_anything_detector:
-            assert isinstance(depth_anything_detector, DepthAnythingPipeline)
-            depth_map = depth_anything_detector.generate_depth(image)
-
-            # Resizing to user target specified size
-            new_height = int(image.size[1] * (self.resolution / image.size[0]))
-            depth_map = depth_map.resize((self.resolution, new_height))
-
-            return depth_map
-
-
-@invocation(
-    "dw_openpose_image_processor",
-    title="DW Openpose Image Processor",
-    tags=["controlnet", "dwpose", "openpose"],
-    category="controlnet",
-    version="1.1.1",
-    classification=Classification.Deprecated,
-)
-class DWOpenposeImageProcessorInvocation(ImageProcessorInvocation):
-    """Generates an openpose pose from an image using DWPose"""
-
-    draw_body: bool = InputField(default=True)
-    draw_face: bool = InputField(default=False)
-    draw_hands: bool = InputField(default=False)
-    image_resolution: int = InputField(default=512, ge=1, description=FieldDescriptions.image_res)
-
-    def run_processor(self, image: Image.Image) -> Image.Image:
-        onnx_det = self._context.models.download_and_cache_model(DWPOSE_MODELS["yolox_l.onnx"])
-        onnx_pose = self._context.models.download_and_cache_model(DWPOSE_MODELS["dw-ll_ucoco_384.onnx"])
-
-        dw_openpose = DWOpenposeDetector(onnx_det=onnx_det, onnx_pose=onnx_pose)
-        processed_image = dw_openpose(
-            image,
-            draw_face=self.draw_face,
-            draw_hands=self.draw_hands,
-            draw_body=self.draw_body,
-            resolution=self.image_resolution,
-        )
-        return processed_image
-
-
-@invocation(
-    "heuristic_resize",
-    title="Heuristic Resize",
-    tags=["image, controlnet"],
-    category="image",
-    version="1.0.1",
-    classification=Classification.Prototype,
-)
-class HeuristicResizeInvocation(BaseInvocation):
-    """Resize an image using a heuristic method. Preserves edge maps."""
-
-    image: ImageField = InputField(description="The image to resize")
-    width: int = InputField(default=512, ge=1, description="The width to resize to (px)")
-    height: int = InputField(default=512, ge=1, description="The height to resize to (px)")
-
-    def invoke(self, context: InvocationContext) -> ImageOutput:
-        image = context.images.get_pil(self.image.image_name, "RGB")
-        np_img = pil_to_np(image)
-        np_resized = heuristic_resize(np_img, (self.width, self.height))
-        resized = np_to_pil(np_resized)
-        image_dto = context.images.save(image=resized)
-        return ImageOutput.build(image_dto)

invokeai/app/invocations/denoise_latents.py

@@ -22,7 +22,7 @@ from transformers import CLIPVisionModelWithProjection
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
 from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
-from invokeai.app.invocations.controlnet_image_processors import ControlField
+from invokeai.app.invocations.controlnet import ControlField
 from invokeai.app.invocations.fields import (
     ConditioningField,
     DenoiseMaskField,

invokeai/app/invocations/dw_openpose.py

@@ -4,7 +4,7 @@ from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
 from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
 from invokeai.app.invocations.primitives import ImageOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.backend.image_util.dw_openpose import DWOpenposeDetector2
+from invokeai.backend.image_util.dw_openpose import DWOpenposeDetector
 
 
 @invocation(
@@ -25,20 +25,20 @@ class DWOpenposeDetectionInvocation(BaseInvocation, WithMetadata, WithBoard):
     def invoke(self, context: InvocationContext) -> ImageOutput:
         image = context.images.get_pil(self.image.image_name, "RGB")
 
-        onnx_det_path = context.models.download_and_cache_model(DWOpenposeDetector2.get_model_url_det())
-        onnx_pose_path = context.models.download_and_cache_model(DWOpenposeDetector2.get_model_url_pose())
+        onnx_det_path = context.models.download_and_cache_model(DWOpenposeDetector.get_model_url_det())
+        onnx_pose_path = context.models.download_and_cache_model(DWOpenposeDetector.get_model_url_pose())
 
         loaded_session_det = context.models.load_local_model(
-            onnx_det_path, DWOpenposeDetector2.create_onnx_inference_session
+            onnx_det_path, DWOpenposeDetector.create_onnx_inference_session
         )
         loaded_session_pose = context.models.load_local_model(
-            onnx_pose_path, DWOpenposeDetector2.create_onnx_inference_session
+            onnx_pose_path, DWOpenposeDetector.create_onnx_inference_session
         )
 
         with loaded_session_det as session_det, loaded_session_pose as session_pose:
             assert isinstance(session_det, ort.InferenceSession)
             assert isinstance(session_pose, ort.InferenceSession)
-            detector = DWOpenposeDetector2(session_det=session_det, session_pose=session_pose)
+            detector = DWOpenposeDetector(session_det=session_det, session_pose=session_pose)
             detected_image = detector.run(
                 image,
                 draw_face=self.draw_face,

invokeai/app/invocations/metadata_linked.py

@@ -14,7 +14,7 @@ from invokeai.app.invocations.baseinvocation import (
     invocation,
     invocation_output,
 )
-from invokeai.app.invocations.controlnet_image_processors import ControlField, ControlNetInvocation
+from invokeai.app.invocations.controlnet import ControlField, ControlNetInvocation
 from invokeai.app.invocations.denoise_latents import DenoiseLatentsInvocation
 from invokeai.app.invocations.fields import (
     FieldDescriptions,

invokeai/app/invocations/tiled_multi_diffusion_denoise_latents.py

@@ -9,7 +9,7 @@ from pydantic import field_validator
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
 from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
-from invokeai.app.invocations.controlnet_image_processors import ControlField
+from invokeai.app.invocations.controlnet import ControlField
 from invokeai.app.invocations.denoise_latents import DenoiseLatentsInvocation, get_scheduler
 from invokeai.app.invocations.fields import (
     ConditioningField,

invokeai/app/services/object_serializer/object_serializer_disk.py

@@ -21,10 +21,16 @@ class ObjectSerializerDisk(ObjectSerializerBase[T]):
     """Disk-backed storage for arbitrary python objects. Serialization is handled by `torch.save` and `torch.load`.
 
     :param output_dir: The folder where the serialized objects will be stored
+    :param safe_globals: A list of types to be added to the safe globals for torch serialization
     :param ephemeral: If True, objects will be stored in a temporary directory inside the given output_dir and cleaned up on exit
     """
 
-    def __init__(self, output_dir: Path, ephemeral: bool = False):
+    def __init__(
+        self,
+        output_dir: Path,
+        safe_globals: list[type],
+        ephemeral: bool = False,
+    ) -> None:
         super().__init__()
         self._ephemeral = ephemeral
         self._base_output_dir = output_dir
@@ -42,6 +48,8 @@ class ObjectSerializerDisk(ObjectSerializerBase[T]):
         self._output_dir = Path(self._tempdir.name) if self._tempdir else self._base_output_dir
         self.__obj_class_name: Optional[str] = None
 
+        torch.serialization.add_safe_globals(safe_globals) if safe_globals else None
+
     def load(self, name: str) -> T:
         file_path = self._get_path(name)
         try:

invokeai/app/util/startup_utils.py

@@ -65,9 +65,6 @@ def apply_monkeypatches() -> None:
 
     import invokeai.backend.util.hotfixes  # noqa: F401 (monkeypatching on import)
 
-    if torch.backends.mps.is_available():
-        import invokeai.backend.util.mps_fixes  # noqa: F401 (monkeypatching on import)
-
 
 def register_mime_types() -> None:
     """Register additional mime types for windows."""

invokeai/backend/image_util/dw_openpose/__init__.py

@@ -5,62 +5,14 @@ import huggingface_hub
 import numpy as np
 import onnxruntime as ort
 import torch
-from controlnet_aux.util import resize_image
 from PIL import Image
 
 from invokeai.backend.image_util.dw_openpose.onnxdet import inference_detector
 from invokeai.backend.image_util.dw_openpose.onnxpose import inference_pose
 from invokeai.backend.image_util.dw_openpose.utils import NDArrayInt, draw_bodypose, draw_facepose, draw_handpose
-from invokeai.backend.image_util.dw_openpose.wholebody import Wholebody
 from invokeai.backend.image_util.util import np_to_pil
 from invokeai.backend.util.devices import TorchDevice
 
-DWPOSE_MODELS = {
-    "yolox_l.onnx": "https://huggingface.co/yzd-v/DWPose/resolve/main/yolox_l.onnx?download=true",
-    "dw-ll_ucoco_384.onnx": "https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.onnx?download=true",
-}
-
-
-def draw_pose(
-    pose: Dict[str, NDArrayInt | Dict[str, NDArrayInt]],
-    H: int,
-    W: int,
-    draw_face: bool = True,
-    draw_body: bool = True,
-    draw_hands: bool = True,
-    resolution: int = 512,
-) -> Image.Image:
-    bodies = pose["bodies"]
-    faces = pose["faces"]
-    hands = pose["hands"]
-
-    assert isinstance(bodies, dict)
-    candidate = bodies["candidate"]
-
-    assert isinstance(bodies, dict)
-    subset = bodies["subset"]
-
-    canvas = np.zeros(shape=(H, W, 3), dtype=np.uint8)
-
-    if draw_body:
-        canvas = draw_bodypose(canvas, candidate, subset)
-
-    if draw_hands:
-        assert isinstance(hands, np.ndarray)
-        canvas = draw_handpose(canvas, hands)
-
-    if draw_face:
-        assert isinstance(hands, np.ndarray)
-        canvas = draw_facepose(canvas, faces)  # type: ignore
-
-    dwpose_image: Image.Image = resize_image(
-        canvas,
-        resolution,
-    )
-    dwpose_image = Image.fromarray(dwpose_image)
-
-    return dwpose_image
-
 
 class DWOpenposeDetector:
     """
@@ -68,62 +20,6 @@ class DWOpenposeDetector:
     Credits: https://github.com/IDEA-Research/DWPose
     """
 
-    def __init__(self, onnx_det: Path, onnx_pose: Path) -> None:
-        self.pose_estimation = Wholebody(onnx_det=onnx_det, onnx_pose=onnx_pose)
-
-    def __call__(
-        self,
-        image: Image.Image,
-        draw_face: bool = False,
-        draw_body: bool = True,
-        draw_hands: bool = False,
-        resolution: int = 512,
-    ) -> Image.Image:
-        np_image = np.array(image)
-        H, W, C = np_image.shape
-
-        with torch.no_grad():
-            candidate, subset = self.pose_estimation(np_image)
-            nums, keys, locs = candidate.shape
-            candidate[..., 0] /= float(W)
-            candidate[..., 1] /= float(H)
-            body = candidate[:, :18].copy()
-            body = body.reshape(nums * 18, locs)
-            score = subset[:, :18]
-            for i in range(len(score)):
-                for j in range(len(score[i])):
-                    if score[i][j] > 0.3:
-                        score[i][j] = int(18 * i + j)
-                    else:
-                        score[i][j] = -1
-
-            un_visible = subset < 0.3
-            candidate[un_visible] = -1
-
-            # foot = candidate[:, 18:24]
-
-            faces = candidate[:, 24:92]
-
-            hands = candidate[:, 92:113]
-            hands = np.vstack([hands, candidate[:, 113:]])
-
-            bodies = {"candidate": body, "subset": score}
-            pose = {"bodies": bodies, "hands": hands, "faces": faces}
-
-            return draw_pose(
-                pose, H, W, draw_face=draw_face, draw_hands=draw_hands, draw_body=draw_body, resolution=resolution
-            )
-
-
-class DWOpenposeDetector2:
-    """
-    Code from the original implementation of the DW Openpose Detector.
-    Credits: https://github.com/IDEA-Research/DWPose
-
-    This implementation is similar to DWOpenposeDetector, with some alterations to allow the onnx models to be loaded
-    and managed by the model manager.
-    """
-
     hf_repo_id = "yzd-v/DWPose"
     hf_filename_onnx_det = "yolox_l.onnx"
     hf_filename_onnx_pose = "dw-ll_ucoco_384.onnx"
@@ -213,7 +109,7 @@ class DWOpenposeDetector2:
             bodies = {"candidate": body, "subset": score}
             pose = {"bodies": bodies, "hands": hands, "faces": faces}
 
-            return DWOpenposeDetector2.draw_pose(
+            return DWOpenposeDetector.draw_pose(
                 pose, H, W, draw_face=draw_face, draw_hands=draw_hands, draw_body=draw_body
             )
 

invokeai/backend/image_util/dw_openpose/utils.py

@@ -3,7 +3,6 @@
 import math
 
 import cv2
-import matplotlib
 import numpy as np
 import numpy.typing as npt
 
@@ -127,11 +126,13 @@ def draw_handpose(canvas: NDArrayInt, all_hand_peaks: NDArrayInt) -> NDArrayInt:
             x2 = int(x2 * W)
             y2 = int(y2 * H)
             if x1 > eps and y1 > eps and x2 > eps and y2 > eps:
+                hsv_color = np.array([[[ie / float(len(edges)) * 180, 255, 255]]], dtype=np.uint8)
+                rgb_color = cv2.cvtColor(hsv_color, cv2.COLOR_HSV2RGB)[0, 0]
                 cv2.line(
                     canvas,
                     (x1, y1),
                     (x2, y2),
-                    matplotlib.colors.hsv_to_rgb([ie / float(len(edges)), 1.0, 1.0]) * 255,
+                    rgb_color.tolist(),
                     thickness=2,
                 )
 

invokeai/backend/image_util/dw_openpose/wholebody.py

@@ -1,44 +0,0 @@
-# Code from the original DWPose Implementation: https://github.com/IDEA-Research/DWPose
-# Modified pathing to suit Invoke
-
-
-from pathlib import Path
-
-import numpy as np
-import onnxruntime as ort
-
-from invokeai.app.services.config.config_default import get_config
-from invokeai.backend.image_util.dw_openpose.onnxdet import inference_detector
-from invokeai.backend.image_util.dw_openpose.onnxpose import inference_pose
-from invokeai.backend.util.devices import TorchDevice
-
-config = get_config()
-
-
-class Wholebody:
-    def __init__(self, onnx_det: Path, onnx_pose: Path):
-        device = TorchDevice.choose_torch_device()
-
-        providers = ["CUDAExecutionProvider"] if device.type == "cuda" else ["CPUExecutionProvider"]
-
-        self.session_det = ort.InferenceSession(path_or_bytes=onnx_det, providers=providers)
-        self.session_pose = ort.InferenceSession(path_or_bytes=onnx_pose, providers=providers)
-
-    def __call__(self, oriImg):
-        det_result = inference_detector(self.session_det, oriImg)
-        keypoints, scores = inference_pose(self.session_pose, det_result, oriImg)
-
-        keypoints_info = np.concatenate((keypoints, scores[..., None]), axis=-1)
-        # compute neck joint
-        neck = np.mean(keypoints_info[:, [5, 6]], axis=1)
-        # neck score when visualizing pred
-        neck[:, 2:4] = np.logical_and(keypoints_info[:, 5, 2:4] > 0.3, keypoints_info[:, 6, 2:4] > 0.3).astype(int)
-        new_keypoints_info = np.insert(keypoints_info, 17, neck, axis=1)
-        mmpose_idx = [17, 6, 8, 10, 7, 9, 12, 14, 16, 13, 15, 2, 1, 4, 3]
-        openpose_idx = [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17]
-        new_keypoints_info[:, openpose_idx] = new_keypoints_info[:, mmpose_idx]
-        keypoints_info = new_keypoints_info
-
-        keypoints, scores = keypoints_info[..., :2], keypoints_info[..., 2]
-
-        return keypoints, scores

invokeai/backend/stable_diffusion/diffusion/conditioning_data.py

@@ -69,6 +69,9 @@ class SD3ConditioningInfo:
 
 @dataclass
 class ConditioningFieldData:
+    # If you change this class, adding more types, you _must_ update the instantiation of ObjectSerializerDisk in
+    # invokeai/app/api/dependencies.py, adding the types to the list of safe globals. If you do not, torch will be
+    # unable to deserialize the object and will raise an error.
     conditionings: (
         List[BasicConditioningInfo]
         | List[SDXLConditioningInfo]

invokeai/backend/util/mps_fixes.py

@@ -1,245 +0,0 @@
-import math
-
-import diffusers
-import torch
-
-if torch.backends.mps.is_available():
-    torch.empty = torch.zeros
-
-
-_torch_layer_norm = torch.nn.functional.layer_norm
-
-
-def new_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-05):
-    if input.device.type == "mps" and input.dtype == torch.float16:
-        input = input.float()
-        if weight is not None:
-            weight = weight.float()
-        if bias is not None:
-            bias = bias.float()
-        return _torch_layer_norm(input, normalized_shape, weight, bias, eps).half()
-    else:
-        return _torch_layer_norm(input, normalized_shape, weight, bias, eps)
-
-
-torch.nn.functional.layer_norm = new_layer_norm
-
-
-_torch_tensor_permute = torch.Tensor.permute
-
-
-def new_torch_tensor_permute(input, *dims):
-    result = _torch_tensor_permute(input, *dims)
-    if input.device == "mps" and input.dtype == torch.float16:
-        result = result.contiguous()
-    return result
-
-
-torch.Tensor.permute = new_torch_tensor_permute
-
-
-_torch_lerp = torch.lerp
-
-
-def new_torch_lerp(input, end, weight, *, out=None):
-    if input.device.type == "mps" and input.dtype == torch.float16:
-        input = input.float()
-        end = end.float()
-        if isinstance(weight, torch.Tensor):
-            weight = weight.float()
-        if out is not None:
-            out_fp32 = torch.zeros_like(out, dtype=torch.float32)
-        else:
-            out_fp32 = None
-        result = _torch_lerp(input, end, weight, out=out_fp32)
-        if out is not None:
-            out.copy_(out_fp32.half())
-            del out_fp32
-        return result.half()
-
-    else:
-        return _torch_lerp(input, end, weight, out=out)
-
-
-torch.lerp = new_torch_lerp
-
-
-_torch_interpolate = torch.nn.functional.interpolate
-
-
-def new_torch_interpolate(
-    input,
-    size=None,
-    scale_factor=None,
-    mode="nearest",
-    align_corners=None,
-    recompute_scale_factor=None,
-    antialias=False,
-):
-    if input.device.type == "mps" and input.dtype == torch.float16:
-        return _torch_interpolate(
-            input.float(), size, scale_factor, mode, align_corners, recompute_scale_factor, antialias
-        ).half()
-    else:
-        return _torch_interpolate(input, size, scale_factor, mode, align_corners, recompute_scale_factor, antialias)
-
-
-torch.nn.functional.interpolate = new_torch_interpolate
-
-# TODO: refactor it
-_SlicedAttnProcessor = diffusers.models.attention_processor.SlicedAttnProcessor
-
-
-class ChunkedSlicedAttnProcessor:
-    r"""
-    Processor for implementing sliced attention.
-
-    Args:
-        slice_size (`int`, *optional*):
-            The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and
-            `attention_head_dim` must be a multiple of the `slice_size`.
-    """
-
-    def __init__(self, slice_size):
-        assert isinstance(slice_size, int)
-        slice_size = 1  # TODO: maybe implement chunking in batches too when enough memory
-        self.slice_size = slice_size
-        self._sliced_attn_processor = _SlicedAttnProcessor(slice_size)
-
-    def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None):
-        if self.slice_size != 1 or attn.upcast_attention:
-            return self._sliced_attn_processor(attn, hidden_states, encoder_hidden_states, attention_mask)
-
-        residual = hidden_states
-
-        input_ndim = hidden_states.ndim
-
-        if input_ndim == 4:
-            batch_size, channel, height, width = hidden_states.shape
-            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
-
-        batch_size, sequence_length, _ = (
-            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
-        )
-        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
-
-        if attn.group_norm is not None:
-            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
-
-        query = attn.to_q(hidden_states)
-        dim = query.shape[-1]
-        query = attn.head_to_batch_dim(query)
-
-        if encoder_hidden_states is None:
-            encoder_hidden_states = hidden_states
-        elif attn.norm_cross:
-            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
-
-        key = attn.to_k(encoder_hidden_states)
-        value = attn.to_v(encoder_hidden_states)
-        key = attn.head_to_batch_dim(key)
-        value = attn.head_to_batch_dim(value)
-
-        batch_size_attention, query_tokens, _ = query.shape
-        hidden_states = torch.zeros(
-            (batch_size_attention, query_tokens, dim // attn.heads), device=query.device, dtype=query.dtype
-        )
-
-        chunk_tmp_tensor = torch.empty(
-            self.slice_size, query.shape[1], key.shape[1], dtype=query.dtype, device=query.device
-        )
-
-        for i in range(batch_size_attention // self.slice_size):
-            start_idx = i * self.slice_size
-            end_idx = (i + 1) * self.slice_size
-
-            query_slice = query[start_idx:end_idx]
-            key_slice = key[start_idx:end_idx]
-            attn_mask_slice = attention_mask[start_idx:end_idx] if attention_mask is not None else None
-
-            self.get_attention_scores_chunked(
-                attn,
-                query_slice,
-                key_slice,
-                attn_mask_slice,
-                hidden_states[start_idx:end_idx],
-                value[start_idx:end_idx],
-                chunk_tmp_tensor,
-            )
-
-        hidden_states = attn.batch_to_head_dim(hidden_states)
-
-        # linear proj
-        hidden_states = attn.to_out[0](hidden_states)
-        # dropout
-        hidden_states = attn.to_out[1](hidden_states)
-
-        if input_ndim == 4:
-            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
-
-        if attn.residual_connection:
-            hidden_states = hidden_states + residual
-
-        hidden_states = hidden_states / attn.rescale_output_factor
-
-        return hidden_states
-
-    def get_attention_scores_chunked(self, attn, query, key, attention_mask, hidden_states, value, chunk):
-        # batch size = 1
-        assert query.shape[0] == 1
-        assert key.shape[0] == 1
-        assert value.shape[0] == 1
-        assert hidden_states.shape[0] == 1
-
-        # dtype = query.dtype
-        if attn.upcast_attention:
-            query = query.float()
-            key = key.float()
-
-        # out_item_size = query.dtype.itemsize
-        # if attn.upcast_attention:
-        #    out_item_size = torch.float32.itemsize
-        out_item_size = query.element_size()
-        if attn.upcast_attention:
-            out_item_size = 4
-
-        chunk_size = 2**29
-
-        out_size = query.shape[1] * key.shape[1] * out_item_size
-        chunks_count = min(query.shape[1], math.ceil((out_size - 1) / chunk_size))
-        chunk_step = max(1, int(query.shape[1] / chunks_count))
-
-        key = key.transpose(-1, -2)
-
-        def _get_chunk_view(tensor, start, length):
-            if start + length > tensor.shape[1]:
-                length = tensor.shape[1] - start
-            # print(f"view: [{tensor.shape[0]},{tensor.shape[1]},{tensor.shape[2]}] - start: {start}, length: {length}")
-            return tensor[:, start : start + length]
-
-        for chunk_pos in range(0, query.shape[1], chunk_step):
-            if attention_mask is not None:
-                torch.baddbmm(
-                    _get_chunk_view(attention_mask, chunk_pos, chunk_step),
-                    _get_chunk_view(query, chunk_pos, chunk_step),
-                    key,
-                    beta=1,
-                    alpha=attn.scale,
-                    out=chunk,
-                )
-            else:
-                torch.baddbmm(
-                    torch.zeros((1, 1, 1), device=query.device, dtype=query.dtype),
-                    _get_chunk_view(query, chunk_pos, chunk_step),
-                    key,
-                    beta=0,
-                    alpha=attn.scale,
-                    out=chunk,
-                )
-            chunk = chunk.softmax(dim=-1)
-            torch.bmm(chunk, value, out=_get_chunk_view(hidden_states, chunk_pos, chunk_step))
-
-        # del chunk
-
-
-diffusers.models.attention_processor.SlicedAttnProcessor = ChunkedSlicedAttnProcessor

invokeai/frontend/web/package.json

@@ -162,5 +162,6 @@
   },
   "engines": {
     "pnpm": "8"
-  }
+  },
+  "packageManager": "pnpm@8.15.9+sha512.499434c9d8fdd1a2794ebf4552b3b25c0a633abcee5bb15e7b5de90f32f47b513aca98cd5cfd001c31f0db454bc3804edccd578501e4ca293a6816166bbd9f81"
 }

invokeai/frontend/web/src/features/nodes/types/openapi.ts

@@ -79,4 +79,4 @@ export const isInvocationOutputSchemaObject = (
 
 export const isInvocationFieldSchema = (
   obj: OpenAPIV3_1.ReferenceObject | OpenAPIV3_1.SchemaObject
-): obj is InvocationFieldSchema => !('$ref' in obj);
+): obj is InvocationFieldSchema => 'field_kind' in obj;

invokeai/version/invokeai_version.py

@@ -1 +1 @@
-__version__ = "5.9.1"
+__version__ = "5.10.0dev3"

pins.json

@@ -0,0 +1,14 @@
+{
+  "python": "3.12",
+  "torchIndexUrl": {
+    "win32": {
+      "cuda": "https://download.pytorch.org/whl/cu126"
+    },
+    "linux": {
+      "cpu": "https://download.pytorch.org/whl/cpu",
+      "rocm": "https://download.pytorch.org/whl/rocm6.2.4",
+      "cuda": "https://download.pytorch.org/whl/cu126"
+    },
+    "darwin": {}
+  }
+}

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "InvokeAI"
 description = "An implementation of Stable Diffusion which provides various new features and options to aid the image generation process"
-requires-python = ">=3.10, <3.12"
+requires-python = ">=3.10, <3.13"
 readme = { content-type = "text/markdown", file = "README.md" }
 keywords = ["stable-diffusion", "AI"]
 dynamic = ["version"]
@@ -33,69 +33,46 @@ classifiers = [
 ]
 dependencies = [
   # Core generation dependencies, pinned for reproducible builds.
-  "accelerate==1.0.1",
-  "bitsandbytes==0.45.0; sys_platform!='darwin'",
-  "clip_anytorch==2.6.0",       # replacing "clip @ https://github.com/openai/CLIP/archive/eaa22acb90a5876642d0507623e859909230a52d.zip",
+  "accelerate",
+  "bitsandbytes; sys_platform!='darwin'",
   "compel==2.0.2",
-  "controlnet-aux",
-  "diffusers[torch]==0.31.0",
-  "gguf==0.10.0",
-  "invisible-watermark==0.2.0", # needed to install SDXL base and refiner using their repo_ids
-  "mediapipe==0.10.14",         # needed for "mediapipeface" controlnet model
+  "diffusers[torch]",
+  "gguf",
+  "invisible-watermark==0.2.0",           # needed to install SDXL base and refiner using their repo_ids
+  "mediapipe==0.10.14",                   # needed for "mediapipeface" controlnet model
   "numpy<2.0.0",
   "onnx==1.16.1",
   "onnxruntime==1.19.2",
   "opencv-python==4.9.0.80",
-  "pytorch-lightning==2.1.3",
-  "safetensors==0.4.3",
-  # sentencepiece is required to load T5TokenizerFast (used by FLUX).
-  "sentencepiece==0.2.0",
-  "spandrel==0.3.4",
-  "timm~=1.0.0",
-  "torch<2.5.0",                # torch and related dependencies are loosely pinned, will respect requirement of `diffusers[torch]`
-  "torchmetrics",
-  "torchsde",
+  "safetensors",
+  "spandrel",
+  "torch~=2.6.0",                         # torch and related dependencies are loosely pinned, will respect requirement of `diffusers[torch]`
+  "torchsde",                             # diffusers needs this for SDE solvers, but it is not an explicit dep of diffusers
   "torchvision",
-  "transformers==4.46.3",
+  "transformers",
 
   # Core application dependencies, pinned for reproducible builds.
-  "fastapi-events==0.11.1",
-  "fastapi==0.111.0",
-  "huggingface-hub==0.26.1",
-  "pydantic-settings==2.2.1",
-  "pydantic==2.7.2",
-  "python-socketio==5.11.1",
-  "uvicorn[standard]==0.28.0",
+  "fastapi-events",
+  "fastapi",
+  "huggingface-hub",
+  "pydantic-settings",
+  "pydantic",
+  "python-socketio",
+  "uvicorn[standard]",
 
   # Auxiliary dependencies, pinned only if necessary.
-  "albumentations",
   "blake3",
-  "click",
-  "datasets",
   "Deprecated",
   "dnspython",
   "dynamicprompts",
   "einops",
-  "facexlib",
-  # Exclude 3.9.1 which has a problem on windows, see https://github.com/matplotlib/matplotlib/issues/28551
-  "matplotlib!=3.9.1",
-  "npyscreen",
-  "omegaconf",
   "picklescan",
   "pillow",
   "prompt-toolkit",
-  "pympler",
   "pypatchmatch",
-  "pyperclip",
-  "pyreadline3",
   "python-multipart",
   "requests",
-  "rich~=13.3",
-  "scikit-image",
   "semver~=3.0.1",
-  "test-tube",
-  "windows-curses; sys_platform=='win32'",
-  "humanize==4.12.1",
 ]
 
 [project.optional-dependencies]
@@ -127,7 +104,8 @@ dependencies = [
   "pytest-datadir",
   "requests_testadapter",
   "httpx",
-  "polyfactory==2.19.0"
+  "polyfactory==2.19.0",
+  "humanize==4.12.1",
 ]
 
 [project.scripts]
@@ -207,9 +185,9 @@ exclude = [
   ".venv*",
   "*.ipynb",
   "invokeai/backend/image_util/mediapipe_face/", # External code
-  "invokeai/backend/image_util/mlsd/", # External code
-  "invokeai/backend/image_util/normal_bae/", # External code
-  "invokeai/backend/image_util/pidi/", # External code
+  "invokeai/backend/image_util/mlsd/",           # External code
+  "invokeai/backend/image_util/normal_bae/",     # External code
+  "invokeai/backend/image_util/pidi/",           # External code
 ]
 
 [tool.ruff.lint]

tests/test_object_serializer_disk.py

@@ -21,16 +21,18 @@ def count_files(path: Path):
 
 @pytest.fixture
 def obj_serializer(tmp_path: Path):
-    return ObjectSerializerDisk[MockDataclass](tmp_path)
+    return ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass])
 
 
 @pytest.fixture
 def fwd_cache(tmp_path: Path):
-    return ObjectSerializerForwardCache(ObjectSerializerDisk[MockDataclass](tmp_path), max_cache_size=2)
+    return ObjectSerializerForwardCache(
+        ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass]), max_cache_size=2
+    )
 
 
 def test_obj_serializer_disk_initializes(tmp_path: Path):
-    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path)
+    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass])
     assert obj_serializer._output_dir == tmp_path
 
 
@@ -70,7 +72,7 @@ def test_obj_serializer_disk_deletes(obj_serializer: ObjectSerializerDisk[MockDa
 
 
 def test_obj_serializer_ephemeral_creates_tempdir(tmp_path: Path):
-    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=True)
+    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=True)
     assert isinstance(obj_serializer._tempdir, tempfile.TemporaryDirectory)
     assert obj_serializer._base_output_dir == tmp_path
     assert obj_serializer._output_dir != tmp_path
@@ -78,21 +80,21 @@ def test_obj_serializer_ephemeral_creates_tempdir(tmp_path: Path):
 
 
 def test_obj_serializer_ephemeral_deletes_tempdir(tmp_path: Path):
-    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=True)
+    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=True)
     tempdir_path = obj_serializer._output_dir
     del obj_serializer
     assert not tempdir_path.exists()
 
 
 def test_obj_serializer_ephemeral_deletes_tempdir_on_stop(tmp_path: Path):
-    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=True)
+    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=True)
     tempdir_path = obj_serializer._output_dir
     obj_serializer.stop(None)  # pyright: ignore [reportArgumentType]
     assert not tempdir_path.exists()
 
 
 def test_obj_serializer_ephemeral_writes_to_tempdir(tmp_path: Path):
-    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=True)
+    obj_serializer = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=True)
     obj_1 = MockDataclass(foo="bar")
     obj_1_name = obj_serializer.save(obj_1)
     assert Path(obj_serializer._output_dir, obj_1_name).exists()
@@ -102,19 +104,19 @@ def test_obj_serializer_ephemeral_writes_to_tempdir(tmp_path: Path):
 def test_obj_serializer_ephemeral_deletes_dangling_tempdirs_on_init(tmp_path: Path):
     tempdir = tmp_path / "tmpdir"
     tempdir.mkdir()
-    ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=True)
+    ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=True)
     assert not tempdir.exists()
 
 
 def test_obj_serializer_does_not_delete_tempdirs_on_init(tmp_path: Path):
     tempdir = tmp_path / "tmpdir"
     tempdir.mkdir()
-    ObjectSerializerDisk[MockDataclass](tmp_path, ephemeral=False)
+    ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass], ephemeral=False)
     assert tempdir.exists()
 
 
 def test_obj_serializer_disk_different_types(tmp_path: Path):
-    obj_serializer_1 = ObjectSerializerDisk[MockDataclass](tmp_path)
+    obj_serializer_1 = ObjectSerializerDisk[MockDataclass](tmp_path, safe_globals=[MockDataclass])
     obj_1 = MockDataclass(foo="bar")
     obj_1_name = obj_serializer_1.save(obj_1)
     obj_1_loaded = obj_serializer_1.load(obj_1_name)
@@ -123,19 +125,19 @@ def test_obj_serializer_disk_different_types(tmp_path: Path):
     assert obj_1_loaded.foo == "bar"
     assert obj_1_name.startswith("MockDataclass_")
 
-    obj_serializer_2 = ObjectSerializerDisk[int](tmp_path)
+    obj_serializer_2 = ObjectSerializerDisk[int](tmp_path, safe_globals=[int])
     obj_2_name = obj_serializer_2.save(9001)
     assert obj_serializer_2._obj_class_name == "int"
     assert obj_serializer_2.load(obj_2_name) == 9001
     assert obj_2_name.startswith("int_")
 
-    obj_serializer_3 = ObjectSerializerDisk[str](tmp_path)
+    obj_serializer_3 = ObjectSerializerDisk[str](tmp_path, safe_globals=[str])
     obj_3_name = obj_serializer_3.save("foo")
     assert obj_serializer_3._obj_class_name == "str"
     assert obj_serializer_3.load(obj_3_name) == "foo"
     assert obj_3_name.startswith("str_")
 
-    obj_serializer_4 = ObjectSerializerDisk[torch.Tensor](tmp_path)
+    obj_serializer_4 = ObjectSerializerDisk[torch.Tensor](tmp_path, safe_globals=[torch.Tensor])
     obj_4_name = obj_serializer_4.save(torch.tensor([1, 2, 3]))
     obj_4_loaded = obj_serializer_4.load(obj_4_name)
     assert obj_serializer_4._obj_class_name == "Tensor"