mirror of
https://github.com/invoke-ai/InvokeAI.git
synced 2026-01-14 17:48:00 -05:00
fd4c3bd27a
- Move the estimation logic to utility functions - Estimate memory _within_ the encode and decode methods, ensuring we _always_ estimate working memory when running a VAE
73 lines
3.0 KiB
Python
73 lines
3.0 KiB
Python
import einops
|
|
import torch
|
|
|
|
from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
|
|
from invokeai.app.invocations.fields import (
|
|
FieldDescriptions,
|
|
ImageField,
|
|
Input,
|
|
InputField,
|
|
)
|
|
from invokeai.app.invocations.model import VAEField
|
|
from invokeai.app.invocations.primitives import LatentsOutput
|
|
from invokeai.app.services.shared.invocation_context import InvocationContext
|
|
from invokeai.backend.flux.modules.autoencoder import AutoEncoder
|
|
from invokeai.backend.model_manager import LoadedModel
|
|
from invokeai.backend.stable_diffusion.diffusers_pipeline import image_resized_to_grid_as_tensor
|
|
from invokeai.backend.util.devices import TorchDevice
|
|
from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_flux
|
|
|
|
|
|
@invocation(
|
|
"flux_vae_encode",
|
|
title="Image to Latents - FLUX",
|
|
tags=["latents", "image", "vae", "i2l", "flux"],
|
|
category="latents",
|
|
version="1.0.1",
|
|
)
|
|
class FluxVaeEncodeInvocation(BaseInvocation):
|
|
"""Encodes an image into latents."""
|
|
|
|
image: ImageField = InputField(
|
|
description="The image to encode.",
|
|
)
|
|
vae: VAEField = InputField(
|
|
description=FieldDescriptions.vae,
|
|
input=Input.Connection,
|
|
)
|
|
|
|
@staticmethod
|
|
def vae_encode(vae_info: LoadedModel, image_tensor: torch.Tensor) -> torch.Tensor:
|
|
# TODO(ryand): Expose seed parameter at the invocation level.
|
|
# TODO(ryand): Write a util function for generating random tensors that is consistent across devices / dtypes.
|
|
# There's a starting point in get_noise(...), but it needs to be extracted and generalized. This function
|
|
# should be used for VAE encode sampling.
|
|
assert isinstance(vae_info.model, AutoEncoder)
|
|
estimated_working_memory = estimate_vae_working_memory_flux(
|
|
operation="encode", image_tensor=image_tensor, vae=vae_info.model
|
|
)
|
|
generator = torch.Generator(device=TorchDevice.choose_torch_device()).manual_seed(0)
|
|
with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae):
|
|
assert isinstance(vae, AutoEncoder)
|
|
vae_dtype = next(iter(vae.parameters())).dtype
|
|
image_tensor = image_tensor.to(device=TorchDevice.choose_torch_device(), dtype=vae_dtype)
|
|
latents = vae.encode(image_tensor, sample=True, generator=generator)
|
|
return latents
|
|
|
|
@torch.no_grad()
|
|
def invoke(self, context: InvocationContext) -> LatentsOutput:
|
|
image = context.images.get_pil(self.image.image_name)
|
|
|
|
vae_info = context.models.load(self.vae.vae)
|
|
|
|
image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB"))
|
|
if image_tensor.dim() == 3:
|
|
image_tensor = einops.rearrange(image_tensor, "c h w -> 1 c h w")
|
|
|
|
context.util.signal_progress("Running VAE")
|
|
latents = self.vae_encode(vae_info=vae_info, image_tensor=image_tensor)
|
|
|
|
latents = latents.to("cpu")
|
|
name = context.tensors.save(tensor=latents)
|
|
return LatentsOutput.build(latents_name=name, latents=latents, seed=None)
|