merge with main

Opening this PR to make incremental doc updates and improvements ahead
of 3.0

.gitignore

@@ -34,7 +34,7 @@ __pycache__/
 .Python
 build/
 develop-eggs/
-dist/
+# dist/
 downloads/
 eggs/
 .eggs/
@@ -79,6 +79,7 @@ cov.xml
 .pytest.ini
 cover/
 junit/
+notes/
 
 # Translations
 *.mo
@@ -201,7 +202,8 @@ checkpoints
 # If it's a Mac
 .DS_Store
 
-invokeai/frontend/web/dist/*
+invokeai/frontend/yarn.lock
+invokeai/frontend/node_modules
 
 # Let the frontend manage its own gitignore
 !invokeai/frontend/web/*

LICENSE

@@ -1,21 +1,176 @@
-MIT License
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-Copyright (c) 2022 InvokeAI Team
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+   1. Definitions.
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   

README.md

@@ -3,8 +3,8 @@
 ![project hero](https://github.com/invoke-ai/InvokeAI/assets/31807370/1a917d94-e099-4fa1-a70f-7dd8d0691018)
 
 # Invoke AI - Generative AI for Professional Creatives
-## Image Generation for Stable Diffusion, Custom-Trained Models, and more. 
-  Learn more about us and get started instantly at [invoke.ai](https://invoke.ai)
+## Professional Creative Tools for Stable Diffusion, Custom-Trained Models, and more. 
+  To learn more about Invoke AI, get started instantly, or implement our Business solutions, visit [invoke.ai](https://invoke.ai)
 
 
 [![discord badge]][discord link]
@@ -329,24 +329,24 @@ InvokeAI offers a locally hosted Web Server & React Frontend, with an industry l
 
 The Unified Canvas is a fully integrated canvas implementation with support for all core generation capabilities, in/outpainting, brush tools, and more. This creative tool unlocks the capability for artists to create with AI as a creative collaborator, and can be used to augment AI-generated imagery, sketches, photography, renders, and more.
 
-### *Advanced Prompt Syntax*
+### *Node Architecture & Editor (Beta)*
 
-Invoke AI's advanced prompt syntax allows for token weighting, cross-attention control, and prompt blending, allowing for fine-tuned tweaking of your invocations and exploration of the latent space. 
+Invoke AI's backend is built on a graph-based execution architecture. This allows for customizable generation pipelines to be developed by professional users looking to create specific workflows to support their production use-cases, and will be extended in the future with additional capabilities.
 
-### *Command Line Interface*
+### *Board & Gallery Management*
 
-For users utilizing a terminal-based environment, or who want to take advantage of CLI features, InvokeAI offers an extensive and actively supported command-line interface that provides the full suite of generation functionality available in the tool.
+Invoke AI provides an organized gallery system for easily storing, accessing, and remixing your content in the Invoke workspace. Images can be dragged/dropped onto any Image-base UI element in the application, and rich metadata within the Image allows for easy recall of key prompts or settings used in your workflow. 
 
 ### Other features
 
 - *Support for both ckpt and diffusers models*
 - *SD 2.0, 2.1 support*
-- *Upscaling & Face Restoration Tools*
+- *Upscaling Tools*
 - *Embedding Manager & Support*
 - *Model Manager & Support*
 - *Node-Based Architecture*
 - *Node-Based Plug-&-Play UI (Beta)*
-- *Boards & Gallery Management
+- *SDXL Support* (Coming soon)
 
 ### Latest Changes
 
@@ -359,7 +359,7 @@ Notes](https://github.com/invoke-ai/InvokeAI/releases) and the
 Please check out our **[Q&A](https://invoke-ai.github.io/InvokeAI/help/TROUBLESHOOT/#faq)** to get solutions for common installation
 problems and other issues.
 
-## 🤝 Contributing
+## Contributing
 
 Anyone who wishes to contribute to this project, whether documentation, features, bug fixes, code
 cleanup, testing, or code reviews, is very much encouraged to do so.
@@ -378,7 +378,7 @@ to become part of our community.
 
 Welcome to InvokeAI!
 
-### 👥 Contributors
+### Contributors
 
 This fork is a combined effort of various people from across the world.
 [Check out the list of all these amazing people](https://invoke-ai.github.io/InvokeAI/other/CONTRIBUTORS/). We thank them for

docs/contributing/INVOCATIONS.md

@@ -1,8 +1,521 @@
 # Invocations
 
-Invocations represent a single operation, its inputs, and its outputs. These
-operations and their outputs can be chained together to generate and modify
-images.
+Features in InvokeAI are added in the form of modular node-like systems called
+**Invocations**.
+
+An Invocation is simply a single operation that takes in some inputs and gives
+out some outputs. We can then chain multiple Invocations together to create more
+complex functionality.
+
+## Invocations Directory
+
+InvokeAI Invocations can be found in the `invokeai/app/invocations` directory.
+
+You can add your new functionality to one of the existing Invocations in this
+directory or create a new file in this directory as per your needs.
+
+**Note:** _All Invocations must be inside this directory for InvokeAI to
+recognize them as valid Invocations._
+
+## Creating A New Invocation
+
+In order to understand the process of creating a new Invocation, let us actually
+create one.
+
+In our example, let us create an Invocation that will take in an image, resize
+it and output the resized image.
+
+The first set of things we need to do when creating a new Invocation are -
+
+- Create a new class that derives from a predefined parent class called
+  `BaseInvocation`.
+- The name of every Invocation must end with the word `Invocation` in order for
+  it to be recognized as an Invocation.
+- Every Invocation must have a `docstring` that describes what this Invocation
+  does.
+- Every Invocation must have a unique `type` field defined which becomes its
+  indentifier.
+- Invocations are strictly typed. We make use of the native
+  [typing](https://docs.python.org/3/library/typing.html) library and the
+  installed [pydantic](https://pydantic-docs.helpmanual.io/) library for
+  validation.
+
+So let us do that.
+
+```python
+from typing import Literal
+from .baseinvocation import BaseInvocation
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+```
+
+That's great.
+
+Now we have setup the base of our new Invocation. Let us think about what inputs
+our Invocation takes.
+
+- We need an `image` that we are going to resize.
+- We will need new `width` and `height` values to which we need to resize the
+  image to.
+
+### **Inputs**
+
+Every Invocation input is a pydantic `Field` and like everything else should be
+strictly typed and defined.
+
+So let us create these inputs for our Invocation. First up, the `image` input we
+need. Generally, we can use standard variable types in Python but InvokeAI
+already has a custom `ImageField` type that handles all the stuff that is needed
+for image inputs.
+
+But what is this `ImageField` ..? It is a special class type specifically
+written to handle how images are dealt with in InvokeAI. We will cover how to
+create your own custom field types later in this guide. For now, let's go ahead
+and use it.
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation
+from ..models.image import ImageField
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+```
+
+Let us break down our input code.
+
+```python
+image: Union[ImageField, None] = Field(description="The input image", default=None)
+```
+
+| Part      | Value                                                | Description                                                                                        |
+| --------- | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------- |
+| Name      | `image`                                              | The variable that will hold our image                                                              |
+| Type Hint | `Union[ImageField, None]`                            | The types for our field. Indicates that the image can either be an `ImageField` type or `None`     |
+| Field     | `Field(description="The input image", default=None)` | The image variable is a field which needs a description and a default value that we set to `None`. |
+
+Great. Now let us create our other inputs for `width` and `height`
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation
+from ..models.image import ImageField
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    width: int = Field(default=512, ge=64, le=2048, description="Width of the new image")
+    height: int = Field(default=512, ge=64, le=2048, description="Height of the new image")
+```
+
+As you might have noticed, we added two new parameters to the field type for
+`width` and `height` called `gt` and `le`. These basically stand for _greater
+than or equal to_ and _less than or equal to_. There are various other param
+types for field that you can find on the **pydantic** documentation.
+
+**Note:** _Any time it is possible to define constraints for our field, we
+should do it so the frontend has more information on how to parse this field._
+
+Perfect. We now have our inputs. Let us do something with these.
+
+### **Invoke Function**
+
+The `invoke` function is where all the magic happens. This function provides you
+the `context` parameter that is of the type `InvocationContext` which will give
+you access to the current context of the generation and all the other services
+that are provided by it by InvokeAI.
+
+Let us create this function first.
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation, InvocationContext
+from ..models.image import ImageField
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    width: int = Field(default=512, ge=64, le=2048, description="Width of the new image")
+    height: int = Field(default=512, ge=64, le=2048, description="Height of the new image")
+
+    def invoke(self, context: InvocationContext):
+        pass
+```
+
+### **Outputs**
+
+The output of our Invocation will be whatever is returned by this `invoke`
+function. Like with our inputs, we need to strongly type and define our outputs
+too.
+
+What is our output going to be? Another image. Normally you'd have to create a
+type for this but InvokeAI already offers you an `ImageOutput` type that handles
+all the necessary info related to image outputs. So let us use that.
+
+We will cover how to create your own output types later in this guide.
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation, InvocationContext
+from ..models.image import ImageField
+from .image import ImageOutput
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    width: int = Field(default=512, ge=64, le=2048, description="Width of the new image")
+    height: int = Field(default=512, ge=64, le=2048, description="Height of the new image")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        pass
+```
+
+Perfect. Now that we have our Invocation setup, let us do what we want to do.
+
+- We will first load the image. Generally we do this using the `PIL` library but
+  we can use one of the services provided by InvokeAI to load the image.
+- We will resize the image using `PIL` to our input data.
+- We will output this image in the format we set above.
+
+So let's do that.
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation, InvocationContext
+from ..models.image import ImageField, ResourceOrigin, ImageCategory
+from .image import ImageOutput
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    width: int = Field(default=512, ge=64, le=2048, description="Width of the new image")
+    height: int = Field(default=512, ge=64, le=2048, description="Height of the new image")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        # Load the image using InvokeAI's predefined Image Service.
+        image = context.services.images.get_pil_image(self.image.image_origin, self.image.image_name)
+
+        # Resizing the image
+        # Because we used the above service, we already have a PIL image. So we can simply resize.
+        resized_image = image.resize((self.width, self.height))
+
+        # Preparing the image for output using InvokeAI's predefined Image Service.
+        output_image = context.services.images.create(
+            image=resized_image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+        )
+
+        # Returning the Image
+        return ImageOutput(
+            image=ImageField(
+                image_name=output_image.image_name,
+                image_origin=output_image.image_origin,
+            ),
+            width=output_image.width,
+            height=output_image.height,
+        )
+```
+
+**Note:** Do not be overwhelmed by the `ImageOutput` process. InvokeAI has a
+certain way that the images need to be dispatched in order to be stored and read
+correctly. In 99% of the cases when dealing with an image output, you can simply
+copy-paste the template above.
+
+That's it. You made your own **Resize Invocation**.
+
+## Result
+
+Once you make your Invocation correctly, the rest of the process is fully
+automated for you.
+
+When you launch InvokeAI, you can go to `http://localhost:9090/docs` and see
+your new Invocation show up there with all the relevant info.
+
+![resize invocation](../assets/contributing/resize_invocation.png)
+
+When you launch the frontend UI, you can go to the Node Editor tab and find your
+new Invocation ready to be used.
+
+![resize node editor](../assets/contributing/resize_node_editor.png)
+
+# Advanced
+
+## Custom Input Fields
+
+Now that you know how to create your own Invocations, let us dive into slightly
+more advanced topics.
+
+While creating your own Invocations, you might run into a scenario where the
+existing input types in InvokeAI do not meet your requirements. In such cases,
+you can create your own input types.
+
+Let us create one as an example. Let us say we want to create a color input
+field that represents a color code. But before we start on that here are some
+general good practices to keep in mind.
+
+**Good Practices**
+
+- There is no naming convention for input fields but we highly recommend that
+  you name it something appropriate like `ColorField`.
+- It is not mandatory but it is heavily recommended to add a relevant
+  `docstring` to describe your input field.
+- Keep your field in the same file as the Invocation that it is made for or in
+  another file where it is relevant.
+
+All input types a class that derive from the `BaseModel` type from `pydantic`.
+So let's create one.
+
+```python
+from pydantic import BaseModel
+
+class ColorField(BaseModel):
+    '''A field that holds the rgba values of a color'''
+    pass
+```
+
+Perfect. Now let us create our custom inputs for our field. This is exactly
+similar how you created input fields for your Invocation. All the same rules
+apply. Let us create four fields representing the _red(r)_, _blue(b)_,
+_green(g)_ and _alpha(a)_ channel of the color.
+
+```python
+class ColorField(BaseModel):
+    '''A field that holds the rgba values of a color'''
+    r: int = Field(ge=0, le=255, description="The red channel")
+    g: int = Field(ge=0, le=255, description="The green channel")
+    b: int = Field(ge=0, le=255, description="The blue channel")
+    a: int = Field(ge=0, le=255, description="The alpha channel")
+```
+
+That's it. We now have a new input field type that we can use in our Invocations
+like this.
+
+```python
+color: ColorField = Field(default=ColorField(r=0, g=0, b=0, a=0), description='Background color of an image')
+```
+
+**Extra Config**
+
+All input fields also take an additional `Config` class that you can use to do
+various advanced things like setting required parameters and etc.
+
+Let us do that for our _ColorField_ and enforce all the values because we did
+not define any defaults for our fields.
+
+```python
+class ColorField(BaseModel):
+    '''A field that holds the rgba values of a color'''
+    r: int = Field(ge=0, le=255, description="The red channel")
+    g: int = Field(ge=0, le=255, description="The green channel")
+    b: int = Field(ge=0, le=255, description="The blue channel")
+    a: int = Field(ge=0, le=255, description="The alpha channel")
+
+    class Config:
+        schema_extra = {"required": ["r", "g", "b", "a"]}
+```
+
+Now it becomes mandatory for the user to supply all the values required by our
+input field.
+
+We will discuss the `Config` class in extra detail later in this guide and how
+you can use it to make your Invocations more robust.
+
+## Custom Output Types
+
+Like with custom inputs, sometimes you might find yourself needing custom
+outputs that InvokeAI does not provide. We can easily set one up.
+
+Now that you are familiar with Invocations and Inputs, let us use that knowledge
+to put together a custom output type for an Invocation that returns _width_,
+_height_ and _background_color_ that we need to create a blank image.
+
+- A custom output type is a class that derives from the parent class of
+  `BaseInvocationOutput`.
+- It is not mandatory but we recommend using names ending with `Output` for
+  output types. So we'll call our class `BlankImageOutput`
+- It is not mandatory but we highly recommend adding a `docstring` to describe
+  what your output type is for.
+- Like Invocations, each output type should have a `type` variable that is
+  **unique**
+
+Now that we know the basic rules for creating a new output type, let us go ahead
+and make it.
+
+```python
+from typing import Literal
+from pydantic import Field
+
+from .baseinvocation import BaseInvocationOutput
+
+class BlankImageOutput(BaseInvocationOutput):
+    '''Base output type for creating a blank image'''
+    type: Literal['blank_image_output'] = 'blank_image_output'
+
+    # Inputs
+    width: int = Field(description='Width of blank image')
+    height: int = Field(description='Height of blank image')
+    bg_color: ColorField = Field(description='Background color of blank image')
+
+    class Config:
+        schema_extra = {"required": ["type", "width", "height", "bg_color"]}
+```
+
+All set. We now have an output type that requires what we need to create a
+blank_image. And if you noticed it, we even used the `Config` class to ensure
+the fields are required.
+
+## Custom Configuration
+
+As you might have noticed when making inputs and outputs, we used a class called
+`Config` from _pydantic_ to further customize them. Because our inputs and
+outputs essentially inherit from _pydantic_'s `BaseModel` class, all
+[configuration options](https://docs.pydantic.dev/latest/usage/schema/#schema-customization)
+that are valid for _pydantic_ classes are also valid for our inputs and outputs.
+You can do the same for your Invocations too but InvokeAI makes our life a
+little bit easier on that end.
+
+InvokeAI provides a custom configuration class called `InvocationConfig`
+particularly for configuring Invocations. This is exactly the same as the raw
+`Config` class from _pydantic_ with some extra stuff on top to help faciliate
+parsing of the scheme in the frontend UI.
+
+At the current moment, tihs `InvocationConfig` class is further improved with
+the following features related the `ui`.
+
+| Config Option | Field Type                                                                                                    | Example                                                                                                               |
+| ------------- | ------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
+| type_hints    | `Dict[str, Literal["integer", "float", "boolean", "string", "enum", "image", "latents", "model", "control"]]` | `type_hint: "model"` provides type hints related to the model like displaying a list of available models              |
+| tags          | `List[str]`                                                                                                   | `tags: ['resize', 'image']` will classify your invocation under the tags of resize and image.                         |
+| title         | `str`                                                                                                         | `title: 'Resize Image` will rename your to this custom title rather than infer from the name of the Invocation class. |
+
+So let us update your `ResizeInvocation` with some extra configuration and see
+how that works.
+
+```python
+from typing import Literal, Union
+from pydantic import Field
+
+from .baseinvocation import BaseInvocation, InvocationContext, InvocationConfig
+from ..models.image import ImageField, ResourceOrigin, ImageCategory
+from .image import ImageOutput
+
+class ResizeInvocation(BaseInvocation):
+    '''Resizes an image'''
+    type: Literal['resize'] = 'resize'
+
+    # Inputs
+    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    width: int = Field(default=512, ge=64, le=2048, description="Width of the new image")
+    height: int = Field(default=512, ge=64, le=2048, description="Height of the new image")
+
+    class Config(InvocationConfig):
+        schema_extra: {
+            ui: {
+                tags: ['resize', 'image'],
+                title: ['My Custom Resize']
+            }
+        }
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        # Load the image using InvokeAI's predefined Image Service.
+        image = context.services.images.get_pil_image(self.image.image_origin, self.image.image_name)
+
+        # Resizing the image
+        # Because we used the above service, we already have a PIL image. So we can simply resize.
+        resized_image = image.resize((self.width, self.height))
+
+        # Preparing the image for output using InvokeAI's predefined Image Service.
+        output_image = context.services.images.create(
+            image=resized_image,
+            image_origin=ResourceOrigin.INTERNAL,
+            image_category=ImageCategory.GENERAL,
+            node_id=self.id,
+            session_id=context.graph_execution_state_id,
+            is_intermediate=self.is_intermediate,
+        )
+
+        # Returning the Image
+        return ImageOutput(
+            image=ImageField(
+                image_name=output_image.image_name,
+                image_origin=output_image.image_origin,
+            ),
+            width=output_image.width,
+            height=output_image.height,
+        )
+```
+
+We now customized our code to let the frontend know that our Invocation falls
+under `resize` and `image` categories. So when the user searches for these
+particular words, our Invocation will show up too.
+
+We also set a custom title for our Invocation. So instead of being called
+`Resize`, it will be called `My Custom Resize`.
+
+As simple as that.
+
+As time goes by, InvokeAI will further improve and add more customizability for
+Invocation configuration. We will have more documentation regarding this at a
+later time.
+
+# **[TODO]**
+
+## Custom Components For Frontend
+
+Every backend input type should have a corresponding frontend component so the
+UI knows what to render when you use a particular field type.
+
+If you are using existing field types, we already have components for those. So
+you don't have to worry about creating anything new. But this might not always
+be the case. Sometimes you might want to create new field types and have the
+frontend UI deal with it in a different way.
+
+This is where we venture into the world of React and Javascript and create our
+own new components for our Invocations. Do not fear the world of JS. It's
+actually pretty straightforward.
+
+Let us create a new component for our custom color field we created above. When
+we use a color field, let us say we want the UI to display a color picker for
+the user to pick from rather than entering values. That is what we will build
+now.
+
+---
+
+# OLD -- TO BE DELETED OR MOVED LATER
+
+---
 
 ## Creating a new invocation
 

docs/features/CONCEPTS.md

@@ -1,9 +1,12 @@
 ---
-title: Concepts Library
+title: Concepts 
 ---
 
 # :material-library-shelves: The Hugging Face Concepts Library and Importing Textual Inversion files
 
+With the advances in research, many new capabilities are available to customize the knowledge and understanding of novel concepts not originally contained in the base model. 
+
+
 ## Using Textual Inversion Files
 
 Textual inversion (TI) files are small models that customize the output of
@@ -12,18 +15,16 @@ and artistic styles. They are also known as "embeds" in the machine learning
 world.
 
 Each TI file introduces one or more vocabulary terms to the SD model. These are
-known in InvokeAI as "triggers." Triggers are often, but not always, denoted
-using angle brackets as in "<trigger-phrase>". The two most common type of
+known in InvokeAI as "triggers." Triggers are denoted using angle brackets 
+as in "<trigger-phrase>". The two most common type of
 TI files that you'll encounter are `.pt` and `.bin` files, which are produced by
 different TI training packages. InvokeAI supports both formats, but its
-[built-in TI training system](TEXTUAL_INVERSION.md) produces `.pt`.
+[built-in TI training system](TRAINING.md) produces `.pt`.
 
 The [Hugging Face company](https://huggingface.co/sd-concepts-library) has
 amassed a large ligrary of >800 community-contributed TI files covering a
-broad range of subjects and styles. InvokeAI has built-in support for this
-library which downloads and merges TI files automatically upon request. You can
-also install your own or others' TI files by placing them in a designated
-directory.
+broad range of subjects and styles. You can also install your own or others' TI files 
+by placing them in the designated directory for the compatible model type
 
 ### An Example
 
@@ -41,66 +42,43 @@ You can also combine styles and concepts:
   | :--------------------------------------------------------: |
   | ![](../assets/concepts/image5.png)                         |
 </figure>
-## Using a Hugging Face Concept
 
-!!! warning "Authenticating to HuggingFace"
-
-    Some concepts require valid authentication to HuggingFace. Without it, they will not be downloaded
-    and will be silently ignored.
-
-    If you used an installer to install InvokeAI, you may have already set a HuggingFace token.
-    If you skipped this step, you can:
-
-    - run the InvokeAI configuration script again (if you used a manual installer): `invokeai-configure`
-    - set one of the `HUGGINGFACE_TOKEN` or `HUGGING_FACE_HUB_TOKEN` environment variables to contain your token
-
-    Finally, if you already used any HuggingFace library on your computer, you might already have a token
-    in your local cache. Check for a hidden `.huggingface` directory in your home folder. If it
-    contains a `token` file, then you are all set.
-
-
-Hugging Face TI concepts are downloaded and installed automatically as you
-require them. This requires your machine to be connected to the Internet. To
-find out what each concept is for, you can browse the
-[Hugging Face concepts library](https://huggingface.co/sd-concepts-library) and
-look at examples of what each concept produces.
-
-To load concepts, you will need to open the Web UI's configuration
-dialogue and activate "Show Textual Inversions from HF Concepts
-Library". This will then add a list of HF Concepts to the dropdown
-"Add Textual Inversion" menu. Select the concept(s) of your choice and
-they will be incorporated into the positive prompt. A few concepts are
-designed for the negative prompt, in which case you can add them to
-the negative prompt box by select the down arrow icon next to the
-textual inversion menu.
-
-There are nearly 1000 HF concepts, more than will fit into a menu. For
-this reason we only show the most popular concepts (those which have
-received 5 or more likes). If you wish to use a concept that is not on
-the list, you may simply type its name surrounded by brackets. For
-example, to load the concept named "xidiversity", add `<xidiversity>`
-to the positive or negative prompt text.
 
 ## Installing your Own TI Files
 
 You may install any number of `.pt` and `.bin` files simply by copying them into
-the `embeddings` directory of the InvokeAI runtime directory (usually `invokeai`
-in your home directory). You may create subdirectories in order to organize the
-files in any way you wish. Be careful not to overwrite one file with another.
+the `embedding` directory of the corresponding InvokeAI models directory (usually `invokeai`
+in your home directory). For example, you can simply move a Stable Diffusion 1.5 embedding file to
+the `sd-1/embedding` folder. Be careful not to overwrite one file with another.
 For example, TI files generated by the Hugging Face toolkit share the named
-`learned_embedding.bin`. You can use subdirectories to keep them distinct.
+`learned_embedding.bin`. You can rename these, or use subdirectories to keep them distinct.
 
-At startup time, InvokeAI will scan the `embeddings` directory and load any TI
-files it finds there. At startup you will see a message similar to this one:
+At startup time, InvokeAI will scan the various `embedding` directories and load any TI
+files it finds there for compatible models. At startup you will see a message similar to this one:
 
 ```bash
 >> Current embedding manager terms: <HOI4-Leader>, <princess-knight>
 ```
+To use these when generating, simply type the `<` key in your prompt to open the Textual Inversion WebUI and 
+select the embedding you'd like to use. This UI has type-ahead support, so you can easily find supported embeddings.
 
-The terms you can use will appear in the "Add Textual Inversion"
-dropdown menu above the HF Concepts.
+## Using LoRAs
 
-## Further Reading
+LoRA files are models that customize the output of Stable Diffusion image generation.
+Larger than embeddings, but much smaller than full models, they augment SD with improved
+understanding of subjects and artistic styles.
+
+Unlike TI files, LoRAs do not introduce novel vocabulary into the model's known tokens. Instead,
+LoRAs augment the model's weights that are applied to generate imagery. LoRAs may be supplied
+with a "trigger" word that they have been explicitly trained on, or may simply apply their 
+effect without being triggered.
+
+LoRAs are typically stored in .safetensors files, which are the most secure way to store and transmit
+these types of weights. You may install any number of `.safetensors` LoRA files simply by copying them into
+the `lora` directory of the corresponding InvokeAI models directory (usually `invokeai`
+in your home directory). For example, you can simply move a Stable Diffusion 1.5 LoRA file to
+the `sd-1/lora` folder.
+
+To use these when generating, open the LoRA menu item in the options panel, select the LoRAs you want to apply
+and ensure that they have the appropriate weight recommended by the model provider. Typically, most LoRAs perform best at a weight of .75-1. 
 
-Please see [the repository](https://github.com/rinongal/textual_inversion) and
-associated paper for details and limitations.

docs/features/PROMPTS.md

@@ -301,5 +301,48 @@ summoning up the concept of some sort of scifi creature? Let's find out.
 Indeed, removing the word "hybrid" produces an image that is more like what we'd
 expect.
 
-In conclusion, prompt blending is great for exploring creative space,
-but takes some trial and error to achieve the desired effect.
+## Dynamic Prompts
+
+Dynamic Prompts are a powerful feature designed to produce a variety of prompts based on user-defined options. Using a special syntax, you can construct a prompt with multiple possibilities, and the system will automatically generate a series of permutations based on your settings. This is extremely beneficial for ideation, exploring various scenarios, or testing different concepts swiftly and efficiently.
+
+### Structure of a Dynamic Prompt
+
+A Dynamic Prompt comprises of regular text, supplemented with alternatives enclosed within curly braces {} and separated by a vertical bar |. For example: {option1|option2|option3}. The system will then select one of the options to include in the final prompt. This flexible system allows for options to be placed throughout the text as needed.
+
+Furthermore, Dynamic Prompts can designate multiple selections from a single group of options. This feature is triggered by prefixing the options with a numerical value followed by $$. For example, in {2$$option1|option2|option3}, the system will select two distinct options from the set.
+### Creating Dynamic Prompts
+
+To create a Dynamic Prompt, follow these steps:
+
+    Draft your sentence or phrase, identifying words or phrases with multiple possible options.
+    Encapsulate the different options within curly braces {}.
+    Within the braces, separate each option using a vertical bar |.
+    If you want to include multiple options from a single group, prefix with the desired number and $$.
+
+For instance: A {house|apartment|lodge|cottage} in {summer|winter|autumn|spring} designed in {2$$style1|style2|style3}.
+### How Dynamic Prompts Work
+
+Once a Dynamic Prompt is configured, the system generates an array of combinations using the options provided. Each group of options in curly braces is treated independently, with the system selecting one option from each group. For a prefixed set (e.g., 2$$), the system will select two distinct options.
+
+For example, the following prompts could be generated from the above Dynamic Prompt:
+
+    A house in summer designed in style1, style2
+    A lodge in autumn designed in style3, style1
+    A cottage in winter designed in style2, style3
+    And many more!
+
+When the `Combinatorial` setting is on, Invoke will disable the "Images" selection, and generate every combination up until the setting for Max Prompts is reached.
+When the `Combinatorial` setting is off, Invoke will randomly generate combinations up until the setting for Images has been reached.
+
+
+
+### Tips and Tricks for Using Dynamic Prompts
+
+Below are some useful strategies for creating Dynamic Prompts:
+
+    Utilize Dynamic Prompts to generate a wide spectrum of prompts, perfect for brainstorming and exploring diverse ideas.
+    Ensure that the options within a group are contextually relevant to the part of the sentence where they are used. For instance, group building types together, and seasons together.
+    Apply the 2$$ prefix when you want to incorporate more than one option from a single group. This becomes quite handy when mixing and matching different elements.
+    Experiment with different quantities for the prefix. For example, 3$$ will select three distinct options.
+    Be aware of coherence in your prompts. Although the system can generate all possible combinations, not all may semantically make sense. Therefore, carefully choose the options for each group.
+    Always review and fine-tune the generated prompts as needed. While Dynamic Prompts can help you generate a multitude of combinations, the final polishing and refining remain in your hands.

docs/features/TRAINING.md

@@ -1,9 +1,10 @@
 ---
-title: Textual-Inversion
+title: Training
 ---
 
-# :material-file-document: Textual Inversion
+# :material-file-document: Training
 
+# Textual Inversion Training
 ## **Personalizing Text-to-Image Generation**
 
 You may personalize the generated images to provide your own styles or objects
@@ -258,16 +259,6 @@ invokeai-ti \
        --only_save_embeds
 ```
 
-## Using Embeddings
-
-After training completes, the resultant embeddings will be saved into your `$INVOKEAI_ROOT/embeddings/<trigger word>/learned_embeds.bin`.
-
-These will be automatically loaded when you start InvokeAI.
-
-Add the trigger word, surrounded by angle brackets, to use that embedding. For example, if your trigger word was `terence`, use `<terence>` in prompts. This is the same syntax used by the HuggingFace concepts library.
-
-**Note:** `.pt` embeddings do not require the angle brackets.
-
 ## Troubleshooting
 
 ### `Cannot load embedding for <trigger>. It was trained on a model with token dimension 1024, but the current model has token dimension 768`

installer/templates/invoke.bat.in

@@ -20,7 +20,7 @@ echo 9. Update InvokeAI
 echo 10. Command-line help
 echo Q - Quit
 set /P choice="Please enter 1-10, Q: [2] "
-if not defined choice set choice=2
+if not defined choice set choice=1
 IF /I "%choice%" == "1" (
     echo Starting the InvokeAI browser-based UI..
     python .venv\Scripts\invokeai-web.exe %*
@@ -56,7 +56,7 @@ IF /I "%choice%" == "1" (
     call cmd /k
 ) ELSE IF /I "%choice%" == "9" (
    echo Running invokeai-update...
-   python .venv\Scripts\invokeai-update.exe %*
+   python -m invokeai.frontend.install.invokeai_update
 ) ELSE IF /I "%choice%" == "10" (
     echo Displaying command line help...
     python .venv\Scripts\invokeai.exe --help %*

installer/templates/invoke.sh.in

@@ -93,7 +93,7 @@ do_choice() {
     9)
         clear
         printf "Update InvokeAI\n"
-        invokeai-update
+        python -m invokeai.frontend.install.invokeai_update
         ;;
     10)
         clear

invokeai/app/api/dependencies.py

@@ -17,6 +17,7 @@ from invokeai.app.services.metadata import CoreMetadataService
 from invokeai.app.services.resource_name import SimpleNameService
 from invokeai.app.services.urls import LocalUrlService
 from invokeai.backend.util.logging import InvokeAILogger
+from invokeai.version.invokeai_version import __version__
 
 from ..services.default_graphs import create_system_graphs
 from ..services.latent_storage import DiskLatentsStorage, ForwardCacheLatentsStorage
@@ -58,7 +59,8 @@ class ApiDependencies:
 
     @staticmethod
     def initialize(config, event_handler_id: int, logger: Logger = logger):
-        logger.info(f"Internet connectivity is {config.internet_available}")
+        logger.debug(f'InvokeAI version {__version__}')
+        logger.debug(f"Internet connectivity is {config.internet_available}")
 
         events = FastAPIEventService(event_handler_id)
 

invokeai/app/api/routers/app_info.py

@@ -0,0 +1,18 @@
+from fastapi.routing import APIRouter
+from pydantic import BaseModel
+
+from invokeai.version import __version__
+
+app_router = APIRouter(prefix="/v1/app", tags=['app'])
+
+
+class AppVersion(BaseModel):
+    """App Version Response"""
+    version: str
+
+
+@app_router.get('/version', operation_id="app_version",
+                status_code=200,
+                response_model=AppVersion)
+async def get_version() -> AppVersion:
+    return AppVersion(version=__version__)

invokeai/app/api/routers/models.py

@@ -1,72 +1,30 @@
-# Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654) and 2023 Kent Keirsey (https://github.com/hipsterusername)
+# Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654), 2023 Kent Keirsey (https://github.com/hipsterusername), 2024 Lincoln Stein
 
-from typing import Literal, Optional, Union
 
-from fastapi import Query
-from fastapi.routing import APIRouter, HTTPException
-from pydantic import BaseModel, Field, parse_obj_as
-from ..dependencies import ApiDependencies
+from typing import Literal, List, Optional, Union
+
+from fastapi import Body, Path, Query, Response
+from fastapi.routing import APIRouter
+from pydantic import BaseModel, parse_obj_as
+from starlette.exceptions import HTTPException
+
 from invokeai.backend import BaseModelType, ModelType
-from invokeai.backend.model_management.models import OPENAPI_MODEL_CONFIGS, SchedulerPredictionType
-MODEL_CONFIGS = Union[tuple(OPENAPI_MODEL_CONFIGS)]
+from invokeai.backend.model_management.models import (
+    OPENAPI_MODEL_CONFIGS,
+    SchedulerPredictionType,
+)
+from invokeai.backend.model_management import MergeInterpolationMethod
+from ..dependencies import ApiDependencies
 
 models_router = APIRouter(prefix="/v1/models", tags=["models"])
 
-
-class VaeRepo(BaseModel):
-    repo_id: str = Field(description="The repo ID to use for this VAE")
-    path: Optional[str] = Field(description="The path to the VAE")
-    subfolder: Optional[str] = Field(description="The subfolder to use for this VAE")
-
-class ModelInfo(BaseModel):
-    description: Optional[str] = Field(description="A description of the model")
-    model_name: str = Field(description="The name of the model")
-    model_type: str = Field(description="The type of the model")
-    
-class DiffusersModelInfo(ModelInfo):
-    format: Literal['folder'] = 'folder'
-
-    vae: Optional[VaeRepo] = Field(description="The VAE repo to use for this model")
-    repo_id: Optional[str] = Field(description="The repo ID to use for this model")
-    path: Optional[str] = Field(description="The path to the model")
-    
-class CkptModelInfo(ModelInfo):
-    format: Literal['ckpt'] = 'ckpt'
-
-    config: str = Field(description="The path to the model config")
-    weights: str = Field(description="The path to the model weights")
-    vae: str = Field(description="The path to the model VAE")
-    width: Optional[int] = Field(description="The width of the model")
-    height: Optional[int] = Field(description="The height of the model")
-
-class SafetensorsModelInfo(CkptModelInfo):
-    format: Literal['safetensors'] = 'safetensors'
-
-class CreateModelRequest(BaseModel):
-    name: str = Field(description="The name of the model")
-    info: Union[CkptModelInfo, DiffusersModelInfo] = Field(discriminator="format", description="The model info")
-
-class CreateModelResponse(BaseModel):
-    name: str = Field(description="The name of the new model")
-    info: Union[CkptModelInfo, DiffusersModelInfo] = Field(discriminator="format", description="The model info")
-    status: str = Field(description="The status of the API response")
-
-class ImportModelRequest(BaseModel):
-    name: str = Field(description="A model path, repo_id or URL to import")
-    prediction_type: Optional[Literal['epsilon','v_prediction','sample']] = Field(description='Prediction type for SDv2 checkpoint files')
-
-class ConversionRequest(BaseModel):
-    name: str = Field(description="The name of the new model")
-    info: CkptModelInfo = Field(description="The converted model info")
-    save_location: str = Field(description="The path to save the converted model weights")
-
-class ConvertedModelResponse(BaseModel):
-    name: str = Field(description="The name of the new model")
-    info: DiffusersModelInfo = Field(description="The converted model info")
+UpdateModelResponse = Union[tuple(OPENAPI_MODEL_CONFIGS)]
+ImportModelResponse = Union[tuple(OPENAPI_MODEL_CONFIGS)]
+ConvertModelResponse = Union[tuple(OPENAPI_MODEL_CONFIGS)]
+MergeModelResponse = Union[tuple(OPENAPI_MODEL_CONFIGS)]
 
 class ModelsList(BaseModel):
-    models: list[MODEL_CONFIGS]
-
+    models: list[Union[tuple(OPENAPI_MODEL_CONFIGS)]]
 
 @models_router.get(
     "/",
@@ -74,65 +32,103 @@ class ModelsList(BaseModel):
     responses={200: {"model": ModelsList }},
 )
 async def list_models(
-    base_model: Optional[BaseModelType] = Query(
-        default=None, description="Base model"
-    ),
-    model_type: Optional[ModelType] = Query(
-        default=None, description="The type of model to get"
-    ),
+    base_model: Optional[BaseModelType] = Query(default=None, description="Base model"),
+    model_type: Optional[ModelType] = Query(default=None, description="The type of model to get"),
 ) -> ModelsList:
     """Gets a list of models"""
     models_raw = ApiDependencies.invoker.services.model_manager.list_models(base_model, model_type)
     models = parse_obj_as(ModelsList, { "models": models_raw })
     return models
 
-
-@models_router.post(
-    "/",
+@models_router.patch(
+    "/{base_model}/{model_type}/{model_name}",
     operation_id="update_model",
-    responses={200: {"status": "success"}},
+    responses={200: {"description" : "The model was updated successfully"},
+               404: {"description" : "The model could not be found"},
+               400: {"description" : "Bad request"}
+               },
+    status_code = 200,
+    response_model = UpdateModelResponse,
 )
 async def update_model(
-    model_request: CreateModelRequest
-) -> CreateModelResponse:
+        base_model: BaseModelType = Path(description="Base model"),
+        model_type: ModelType = Path(description="The type of model"),
+        model_name: str = Path(description="model name"),
+        info: Union[tuple(OPENAPI_MODEL_CONFIGS)] = Body(description="Model configuration"),
+) -> UpdateModelResponse:
     """ Add Model """
-    model_request_info = model_request.info
-    info_dict = model_request_info.dict()
-    model_response = CreateModelResponse(name=model_request.name, info=model_request.info, status="success")
-
-    ApiDependencies.invoker.services.model_manager.add_model(
-        model_name=model_request.name,
-        model_attributes=info_dict,
-        clobber=True,
-    )
+    try:
+        ApiDependencies.invoker.services.model_manager.update_model(
+            model_name=model_name,
+            base_model=base_model,
+            model_type=model_type,
+            model_attributes=info.dict()
+        )
+        model_raw = ApiDependencies.invoker.services.model_manager.list_model(
+            model_name=model_name,
+            base_model=base_model,
+            model_type=model_type,
+        )
+        model_response = parse_obj_as(UpdateModelResponse, model_raw)
+    except KeyError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
 
     return model_response
 
 @models_router.post(
     "/",
     operation_id="import_model",
-    responses={200: {"status": "success"}},
+    responses= {
+        201: {"description" : "The model imported successfully"},
+        404: {"description" : "The model could not be found"},
+        424: {"description" : "The model appeared to import successfully, but could not be found in the model manager"},
+        409: {"description" : "There is already a model corresponding to this path or repo_id"},
+    },
+    status_code=201,
+    response_model=ImportModelResponse
 )
 async def import_model(
-    model_request: ImportModelRequest
-) -> None:
-    """ Add Model """
-    items_to_import = set([model_request.name])
+        location: str = Body(description="A model path, repo_id or URL to import"),
+        prediction_type: Optional[Literal['v_prediction','epsilon','sample']] = \
+                Body(description='Prediction type for SDv2 checkpoint files', default="v_prediction"),
+) -> ImportModelResponse:
+    """ Add a model using its local path, repo_id, or remote URL """
+    
+    items_to_import = {location}
     prediction_types = { x.value: x for x in SchedulerPredictionType }
     logger = ApiDependencies.invoker.services.logger
+
+    try:
+        installed_models = ApiDependencies.invoker.services.model_manager.heuristic_import(
+            items_to_import = items_to_import,
+            prediction_type_helper = lambda x: prediction_types.get(prediction_type)
+        )
+        info = installed_models.get(location)
+
+        if not info:
+            logger.error("Import failed")
+            raise HTTPException(status_code=424)
+        
+        logger.info(f'Successfully imported {location}, got {info}')
+        model_raw = ApiDependencies.invoker.services.model_manager.list_model(
+            model_name=info.name,
+            base_model=info.base_model,
+            model_type=info.model_type
+        )
+        return parse_obj_as(ImportModelResponse, model_raw)
     
-    installed_models = ApiDependencies.invoker.services.model_manager.heuristic_import(
-        items_to_import = items_to_import,
-        prediction_type_helper = lambda x: prediction_types.get(model_request.prediction_type)
-    )
-    if len(installed_models) > 0:
-        logger.info(f'Successfully imported {model_request.name}')
-    else:
-        logger.error(f'Model {model_request.name} not imported')
-        raise HTTPException(status_code=500, detail=f'Model {model_request.name} not imported')
+    except KeyError as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=404, detail=str(e))
+    except ValueError as e:
+        logger.error(str(e))
+        raise HTTPException(status_code=409, detail=str(e))
+        
 
 @models_router.delete(
-    "/{model_name}",
+    "/{base_model}/{model_type}/{model_name}",
     operation_id="del_model",
     responses={
         204: {
@@ -143,144 +139,95 @@ async def import_model(
         }
     },
 )
-async def delete_model(model_name: str) -> None:
+async def delete_model(
+        base_model: BaseModelType = Path(description="Base model"),
+        model_type: ModelType = Path(description="The type of model"),
+        model_name: str = Path(description="model name"),
+) -> Response:
     """Delete Model"""
-    model_names = ApiDependencies.invoker.services.model_manager.model_names()
     logger = ApiDependencies.invoker.services.logger
-    model_exists = model_name in model_names
-
-    # check if model exists
-    logger.info(f"Checking for model {model_name}...")
-           
-    if model_exists:
-        logger.info(f"Deleting Model: {model_name}")
-        ApiDependencies.invoker.services.model_manager.del_model(model_name, delete_files=True)
-        logger.info(f"Model Deleted: {model_name}")
-        raise HTTPException(status_code=204, detail=f"Model '{model_name}' deleted successfully")
     
-    else:
-        logger.error("Model not found")
+    try:
+        ApiDependencies.invoker.services.model_manager.del_model(model_name,
+                                                                 base_model = base_model,
+                                                                 model_type = model_type
+                                                                 )
+        logger.info(f"Deleted model: {model_name}")
+        return Response(status_code=204)
+    except KeyError:
+        logger.error(f"Model not found: {model_name}")
         raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
-    
 
-            # @socketio.on("convertToDiffusers")
-        # def convert_to_diffusers(model_to_convert: dict):
-        #     try:
-        #         if model_info := self.generate.model_manager.model_info(
-        #             model_name=model_to_convert["model_name"]
-        #         ):
-        #             if "weights" in model_info:
-        #                 ckpt_path = Path(model_info["weights"])
-        #                 original_config_file = Path(model_info["config"])
-        #                 model_name = model_to_convert["model_name"]
-        #                 model_description = model_info["description"]
-        #             else:
-        #                 self.socketio.emit(
-        #                     "error", {"message": "Model is not a valid checkpoint file"}
-        #                 )
-        #         else:
-        #             self.socketio.emit(
-        #                 "error", {"message": "Could not retrieve model info."}
-        #             )
-
-        #         if not ckpt_path.is_absolute():
-        #             ckpt_path = Path(Globals.root, ckpt_path)
-
-        #         if original_config_file and not original_config_file.is_absolute():
-        #             original_config_file = Path(Globals.root, original_config_file)
-
-        #         diffusers_path = Path(
-        #             ckpt_path.parent.absolute(), f"{model_name}_diffusers"
-        #         )
-
-        #         if model_to_convert["save_location"] == "root":
-        #             diffusers_path = Path(
-        #                 global_converted_ckpts_dir(), f"{model_name}_diffusers"
-        #             )
-
-        #         if (
-        #             model_to_convert["save_location"] == "custom"
-        #             and model_to_convert["custom_location"] is not None
-        #         ):
-        #             diffusers_path = Path(
-        #                 model_to_convert["custom_location"], f"{model_name}_diffusers"
-        #             )
-
-        #         if diffusers_path.exists():
-        #             shutil.rmtree(diffusers_path)
-
-        #         self.generate.model_manager.convert_and_import(
-        #             ckpt_path,
-        #             diffusers_path,
-        #             model_name=model_name,
-        #             model_description=model_description,
-        #             vae=None,
-        #             original_config_file=original_config_file,
-        #             commit_to_conf=opt.conf,
-        #         )
-
-        #         new_model_list = self.generate.model_manager.list_models()
-        #         socketio.emit(
-        #             "modelConverted",
-        #             {
-        #                 "new_model_name": model_name,
-        #                 "model_list": new_model_list,
-        #                 "update": True,
-        #             },
-        #         )
-        #         print(f">> Model Converted: {model_name}")
-        #     except Exception as e:
-        #         self.handle_exceptions(e)
-
-        # @socketio.on("mergeDiffusersModels")
-        # def merge_diffusers_models(model_merge_info: dict):
-        #     try:
-        #         models_to_merge = model_merge_info["models_to_merge"]
-        #         model_ids_or_paths = [
-        #             self.generate.model_manager.model_name_or_path(x)
-        #             for x in models_to_merge
-        #         ]
-        #         merged_pipe = merge_diffusion_models(
-        #             model_ids_or_paths,
-        #             model_merge_info["alpha"],
-        #             model_merge_info["interp"],
-        #             model_merge_info["force"],
-        #         )
-
-        #         dump_path = global_models_dir() / "merged_models"
-        #         if model_merge_info["model_merge_save_path"] is not None:
-        #             dump_path = Path(model_merge_info["model_merge_save_path"])
-
-        #         os.makedirs(dump_path, exist_ok=True)
-        #         dump_path = dump_path / model_merge_info["merged_model_name"]
-        #         merged_pipe.save_pretrained(dump_path, safe_serialization=1)
-
-        #         merged_model_config = dict(
-        #             model_name=model_merge_info["merged_model_name"],
-        #             description=f'Merge of models {", ".join(models_to_merge)}',
-        #             commit_to_conf=opt.conf,
-        #         )
-
-        #         if vae := self.generate.model_manager.config[models_to_merge[0]].get(
-        #             "vae", None
-        #         ):
-        #             print(f">> Using configured VAE assigned to {models_to_merge[0]}")
-        #             merged_model_config.update(vae=vae)
-
-        #         self.generate.model_manager.import_diffuser_model(
-        #             dump_path, **merged_model_config
-        #         )
-        #         new_model_list = self.generate.model_manager.list_models()
-
-        #         socketio.emit(
-        #             "modelsMerged",
-        #             {
-        #                 "merged_models": models_to_merge,
-        #                 "merged_model_name": model_merge_info["merged_model_name"],
-        #                 "model_list": new_model_list,
-        #                 "update": True,
-        #             },
-        #         )
-        #         print(f">> Models Merged: {models_to_merge}")
-        #         print(f">> New Model Added: {model_merge_info['merged_model_name']}")
-        #     except Exception as e:
+@models_router.put(
+    "/convert/{base_model}/{model_type}/{model_name}",
+    operation_id="convert_model",
+    responses={
+        200: { "description": "Model converted successfully" },
+        400: {"description" : "Bad request"  },
+        404: { "description": "Model not found"  },
+    },
+    status_code = 200,
+    response_model = ConvertModelResponse,
+)
+async def convert_model(
+        base_model: BaseModelType = Path(description="Base model"),
+        model_type: ModelType = Path(description="The type of model"),
+        model_name: str = Path(description="model name"),
+) -> ConvertModelResponse:
+    """Convert a checkpoint model into a diffusers model"""
+    logger = ApiDependencies.invoker.services.logger
+    try:
+        logger.info(f"Converting model: {model_name}")
+        ApiDependencies.invoker.services.model_manager.convert_model(model_name,
+                                                                     base_model = base_model,
+                                                                     model_type = model_type
+                                                                     )
+        model_raw = ApiDependencies.invoker.services.model_manager.list_model(model_name,
+                                                                              base_model = base_model,
+                                                                              model_type = model_type)
+        response = parse_obj_as(ConvertModelResponse, model_raw)
+    except KeyError:
+        raise HTTPException(status_code=404, detail=f"Model '{model_name}' not found")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return response
+        
+@models_router.put(
+    "/merge/{base_model}",
+    operation_id="merge_models",
+    responses={
+        200: { "description": "Model converted successfully" },
+        400: { "description": "Incompatible models"  },
+        404: { "description": "One or more models not found"  },
+    },
+    status_code = 200,
+    response_model = MergeModelResponse,
+)
+async def merge_models(
+        base_model: BaseModelType                  = Path(description="Base model"),
+        model_names: List[str]                     = Body(description="model name", min_items=2, max_items=3),
+        merged_model_name: Optional[str]           = Body(description="Name of destination model"),
+        alpha: Optional[float]                     = Body(description="Alpha weighting strength to apply to 2d and 3d models", default=0.5),
+        interp: Optional[MergeInterpolationMethod] = Body(description="Interpolation method"),
+        force: Optional[bool]                      = Body(description="Force merging of models created with different versions of diffusers", default=False),
+) -> MergeModelResponse:
+    """Convert a checkpoint model into a diffusers model"""
+    logger = ApiDependencies.invoker.services.logger
+    try:
+        logger.info(f"Merging models: {model_names}")
+        result = ApiDependencies.invoker.services.model_manager.merge_models(model_names,
+                                                                             base_model,
+                                                                             merged_model_name or "+".join(model_names),
+                                                                             alpha,
+                                                                             interp,
+                                                                             force)
+        model_raw = ApiDependencies.invoker.services.model_manager.list_model(result.name,
+                                                                              base_model = base_model,
+                                                                              model_type = ModelType.Main,
+                                                                              )
+        response = parse_obj_as(ConvertModelResponse, model_raw)
+    except KeyError:
+        raise HTTPException(status_code=404, detail=f"One or more of the models '{model_names}' not found")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return response

invokeai/app/api_app.py

@@ -1,5 +1,6 @@
-# Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
+# Copyright (c) 2022-2023 Kyle Schouviller (https://github.com/kyle0654) and the InvokeAI Team
 import asyncio
+import sys
 from inspect import signature
 
 import uvicorn
@@ -20,13 +21,31 @@ from ..backend.util.logging import InvokeAILogger
 app_config = InvokeAIAppConfig.get_config()
 app_config.parse_args()
 logger = InvokeAILogger.getLogger(config=app_config)
+from invokeai.version.invokeai_version import __version__
+
+# we call this early so that the message appears before
+# other invokeai initialization messages
+if app_config.version:
+    print(f'InvokeAI version {__version__}')
+    sys.exit(0)
 
 import invokeai.frontend.web as web_dir
+import mimetypes
 
 from .api.dependencies import ApiDependencies
-from .api.routers import sessions, models, images, boards, board_images
+from .api.routers import sessions, models, images, boards, board_images, app_info
 from .api.sockets import SocketIO
 from .invocations.baseinvocation import BaseInvocation
+    
+
+import torch
+if torch.backends.mps.is_available():
+    import invokeai.backend.util.mps_fixes
+
+# fix for windows mimetypes registry entries being borked
+# see https://github.com/invoke-ai/InvokeAI/discussions/3684#discussioncomment-6391352
+mimetypes.add_type('application/javascript', '.js')
+mimetypes.add_type('text/css', '.css')
 
 # Create the app
 # TODO: create this all in a method so configuration/etc. can be passed in?
@@ -82,6 +101,8 @@ app.include_router(boards.boards_router, prefix="/api")
 
 app.include_router(board_images.board_images_router, prefix="/api")
 
+app.include_router(app_info.app_router, prefix='/api')
+
 # Build a custom OpenAPI to include all outputs
 # TODO: can outputs be included on metadata of invocation schemas somehow?
 def custom_openapi():

invokeai/app/cli/commands.py

@@ -47,7 +47,7 @@ def add_parsers(
     commands: list[type],
     command_field: str = "type",
     exclude_fields: list[str] = ["id", "type"],
-    add_arguments: Callable[[argparse.ArgumentParser], None]|None = None
+    add_arguments: Union[Callable[[argparse.ArgumentParser], None],None] = None
     ):
     """Adds parsers for each command to the subparsers"""
 
@@ -72,7 +72,7 @@ def add_parsers(
 def add_graph_parsers(
     subparsers,
     graphs: list[LibraryGraph],
-    add_arguments: Callable[[argparse.ArgumentParser], None]|None = None
+    add_arguments: Union[Callable[[argparse.ArgumentParser], None], None] = None
 ):
     for graph in graphs:
         command_parser = subparsers.add_parser(graph.name, help=graph.description)

invokeai/app/cli_app.py

@@ -1,12 +1,11 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
 import argparse
-import os
 import re
 import shlex
 import sys
 import time
-from typing import Union, get_type_hints
+from typing import Union, get_type_hints, Optional
 
 from pydantic import BaseModel, ValidationError
 from pydantic.fields import Field
@@ -17,6 +16,12 @@ from invokeai.backend.util.logging import InvokeAILogger
 config = InvokeAIAppConfig.get_config()
 config.parse_args()
 logger = InvokeAILogger().getLogger(config=config)
+from invokeai.version.invokeai_version import __version__
+
+# we call this early so that the message appears before other invokeai initialization messages
+if config.version:
+    print(f'InvokeAI version {__version__}')
+    sys.exit(0)
 
 from invokeai.app.services.board_image_record_storage import (
     SqliteBoardImageRecordStorage,
@@ -53,6 +58,10 @@ from .services.processor import DefaultInvocationProcessor
 from .services.restoration_services import RestorationServices
 from .services.sqlite import SqliteItemStorage
 
+import torch
+if torch.backends.mps.is_available():
+    import invokeai.backend.util.mps_fixes
+
 
 class CliCommand(BaseModel):
     command: Union[BaseCommand.get_commands() + BaseInvocation.get_invocations()] = Field(discriminator="type")  # type: ignore
@@ -205,6 +214,7 @@ def invoke_all(context: CliContext):
         raise SessionError()
 
 def invoke_cli():
+    logger.info(f'InvokeAI version {__version__}')
     # get the optional list of invocations to execute on the command line
     parser = config.get_parser()
     parser.add_argument('commands',nargs='*')
@@ -348,7 +358,7 @@ def invoke_cli():
 
                 # Parse invocation
                 command: CliCommand = None # type:ignore
-                system_graph: LibraryGraph|None = None
+                system_graph: Optional[LibraryGraph] = None
                 if args['type'] in system_graph_names:
                     system_graph = next(filter(lambda g: g.name == args['type'], system_graphs))
                     invocation = GraphInvocation(graph=system_graph.graph, id=str(current_id))

invokeai/app/invocations/baseinvocation.py

@@ -4,9 +4,10 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from inspect import signature
-from typing import get_args, get_type_hints, Dict, List, Literal, TypedDict, TYPE_CHECKING
+from typing import (TYPE_CHECKING, Dict, List, Literal, TypedDict, get_args,
+                    get_type_hints)
 
-from pydantic import BaseModel, Field
+from pydantic import BaseConfig, BaseModel, Field
 
 if TYPE_CHECKING:
     from ..services.invocation_services import InvocationServices
@@ -65,8 +66,13 @@ class BaseInvocation(ABC, BaseModel):
     @classmethod
     def get_invocations_map(cls):
         # Get the type strings out of the literals and into a dictionary
-        return dict(map(lambda t: (get_args(get_type_hints(t)['type'])[0], t),BaseInvocation.get_all_subclasses()))
-    
+        return dict(
+            map(
+                lambda t: (get_args(get_type_hints(t)["type"])[0], t),
+                BaseInvocation.get_all_subclasses(),
+            )
+        )
+
     @classmethod
     def get_output_type(cls):
         return signature(cls.invoke).return_annotation
@@ -75,11 +81,11 @@ class BaseInvocation(ABC, BaseModel):
     def invoke(self, context: InvocationContext) -> BaseInvocationOutput:
         """Invoke with provided context and return outputs."""
         pass
-    
-    #fmt: off
+
+    # fmt: off
     id: str = Field(description="The id of this node. Must be unique among all nodes.")
     is_intermediate: bool = Field(default=False, description="Whether or not this node is an intermediate node.")
-    #fmt: on
+    # fmt: on
 
 
 # TODO: figure out a better way to provide these hints
@@ -97,16 +103,20 @@ class UIConfig(TypedDict, total=False):
             "latents",
             "model",
             "control",
+            "image_collection",
+            "vae_model",
+            "lora_model",
         ],
     ]
     tags: List[str]
     title: str
 
+
 class CustomisedSchemaExtra(TypedDict):
     ui: UIConfig
 
 
-class InvocationConfig(BaseModel.Config):
+class InvocationConfig(BaseConfig):
     """Customizes pydantic's BaseModel.Config class for use by Invocations.
 
     Provide `schema_extra` a `ui` dict to add hints for generated UIs.

invokeai/app/invocations/collections.py

@@ -4,13 +4,16 @@ from typing import Literal
 
 import numpy as np
 from pydantic import Field, validator
+from invokeai.app.models.image import ImageField
 
 from invokeai.app.util.misc import SEED_MAX, get_random_seed
 
 from .baseinvocation import (
     BaseInvocation,
+    InvocationConfig,
     InvocationContext,
     BaseInvocationOutput,
+    UIConfig,
 )
 
 
@@ -22,6 +25,7 @@ class IntCollectionOutput(BaseInvocationOutput):
     # Outputs
     collection: list[int] = Field(default=[], description="The int collection")
 
+
 class FloatCollectionOutput(BaseInvocationOutput):
     """A collection of floats"""
 
@@ -31,6 +35,18 @@ class FloatCollectionOutput(BaseInvocationOutput):
     collection: list[float] = Field(default=[], description="The float collection")
 
 
+class ImageCollectionOutput(BaseInvocationOutput):
+    """A collection of images"""
+
+    type: Literal["image_collection"] = "image_collection"
+
+    # Outputs
+    collection: list[ImageField] = Field(default=[], description="The output images")
+
+    class Config:
+        schema_extra = {"required": ["type", "collection"]}
+
+
 class RangeInvocation(BaseInvocation):
     """Creates a range of numbers from start to stop with step"""
 
@@ -92,3 +108,27 @@ class RandomRangeInvocation(BaseInvocation):
         return IntCollectionOutput(
             collection=list(rng.integers(low=self.low, high=self.high, size=self.size))
         )
+
+
+class ImageCollectionInvocation(BaseInvocation):
+    """Load a collection of images and provide it as output."""
+
+    # fmt: off
+    type: Literal["image_collection"] = "image_collection"
+
+    # Inputs
+    images: list[ImageField] = Field(
+        default=[], description="The image collection to load"
+    )
+    # fmt: on
+    def invoke(self, context: InvocationContext) -> ImageCollectionOutput:
+        return ImageCollectionOutput(collection=self.images)
+
+    class Config(InvocationConfig):
+        schema_extra = {
+            "ui": {
+                "type_hints": {
+                    "images": "image_collection",
+                }
+            },
+        }

invokeai/app/invocations/compel.py

@@ -1,27 +1,25 @@
-from typing import Literal, Optional, Union
+from typing import Literal, Optional, Union, List
 from pydantic import BaseModel, Field
-from contextlib import ExitStack
 import re
-
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationContext, InvocationConfig
-from .model import ClipField
-
-from ...backend.util.devices import torch_dtype
-from ...backend.stable_diffusion.diffusion import InvokeAIDiffuserComponent
-from ...backend.model_management import BaseModelType, ModelType, SubModelType
-from ...backend.model_management.lora import ModelPatcher
-
+import torch
 from compel import Compel
-from compel.prompt_parser import (
-    Blend,
-    CrossAttentionControlSubstitute,
-    FlattenedPrompt,
-    Fragment, Conjunction,
-)
+from compel.prompt_parser import (Blend, Conjunction,
+                                  CrossAttentionControlSubstitute,
+                                  FlattenedPrompt, Fragment)
+from ...backend.util.devices import torch_dtype
+from ...backend.model_management import ModelType
+from ...backend.model_management.models import ModelNotFoundException
+from ...backend.model_management.lora import ModelPatcher
+from ...backend.stable_diffusion.diffusion import InvokeAIDiffuserComponent
+from .baseinvocation import (BaseInvocation, BaseInvocationOutput,
+                             InvocationConfig, InvocationContext)
+from .model import ClipField
 
 
 class ConditioningField(BaseModel):
-    conditioning_name: Optional[str] = Field(default=None, description="The name of conditioning data")
+    conditioning_name: Optional[str] = Field(
+        default=None, description="The name of conditioning data")
+
     class Config:
         schema_extra = {"required": ["conditioning_name"]}
 
@@ -51,83 +49,111 @@ class CompelInvocation(BaseInvocation):
                 "title": "Prompt (Compel)",
                 "tags": ["prompt", "compel"],
                 "type_hints": {
-                  "model": "model"
+                    "model": "model"
                 }
             },
         }
 
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> CompelOutput:
-
         tokenizer_info = context.services.model_manager.get_model(
             **self.clip.tokenizer.dict(),
         )
         text_encoder_info = context.services.model_manager.get_model(
             **self.clip.text_encoder.dict(),
         )
-        with tokenizer_info as orig_tokenizer,\
-             text_encoder_info as text_encoder:
 
-            loras = [(context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.clip.loras]
+        def _lora_loader():
+            for lora in self.clip.loras:
+                lora_info = context.services.model_manager.get_model(
+                    **lora.dict(exclude={"weight"}))
+                yield (lora_info.context.model, lora.weight)
+                del lora_info
+            return
 
-            ti_list = []
-            for trigger in re.findall(r"<[a-zA-Z0-9., _-]+>", self.prompt):
-                name = trigger[1:-1]
-                try:
-                    ti_list.append(
-                        context.services.model_manager.get_model(
-                            model_name=name,
-                            base_model=self.clip.text_encoder.base_model,
-                            model_type=ModelType.TextualInversion,
-                        ).context.model
-                    )
-                except Exception:
-                    #print(e)
-                    #import traceback
-                    #print(traceback.format_exc())
-                    print(f"Warn: trigger: \"{trigger}\" not found")
+        #loras = [(context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.clip.loras]
 
-            with ModelPatcher.apply_lora_text_encoder(text_encoder, loras),\
-                 ModelPatcher.apply_ti(orig_tokenizer, text_encoder, ti_list) as (tokenizer, ti_manager):
-
-                compel = Compel(
-                    tokenizer=tokenizer,
-                    text_encoder=text_encoder,
-                    textual_inversion_manager=ti_manager,
-                    dtype_for_device_getter=torch_dtype,
-                    truncate_long_prompts=True, # TODO:
+        ti_list = []
+        for trigger in re.findall(r"<[a-zA-Z0-9., _-]+>", self.prompt):
+            name = trigger[1:-1]
+            try:
+                ti_list.append(
+                    context.services.model_manager.get_model(
+                        model_name=name,
+                        base_model=self.clip.text_encoder.base_model,
+                        model_type=ModelType.TextualInversion,
+                    ).context.model
                 )
-                
-                conjunction = Compel.parse_prompt_string(self.prompt)
-                prompt: Union[FlattenedPrompt, Blend] = conjunction.prompts[0]
+            except ModelNotFoundException:
+                # print(e)
+                #import traceback
+                #print(traceback.format_exc())
+                print(f"Warn: trigger: \"{trigger}\" not found")
 
-                if context.services.configuration.log_tokenization:
-                    log_tokenization_for_prompt_object(prompt, tokenizer)
+        with ModelPatcher.apply_lora_text_encoder(text_encoder_info.context.model, _lora_loader()),\
+                ModelPatcher.apply_ti(tokenizer_info.context.model, text_encoder_info.context.model, ti_list) as (tokenizer, ti_manager),\
+                ModelPatcher.apply_clip_skip(text_encoder_info.context.model, self.clip.skipped_layers),\
+                text_encoder_info as text_encoder:
 
-                c, options = compel.build_conditioning_tensor_for_prompt_object(prompt)
-                
-                # TODO: long prompt support
-                #if not self.truncate_long_prompts:
-                #    [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc])
-                ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(
-                    tokens_count_including_eos_bos=get_max_token_count(tokenizer, conjunction),
-                    cross_attention_control_args=options.get("cross_attention_control", None),
-                )
-                
-            conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
-
-            # TODO: hacky but works ;D maybe rename latents somehow?
-            context.services.latents.save(conditioning_name, (c, ec))
-
-            return CompelOutput(
-                conditioning=ConditioningField(
-                    conditioning_name=conditioning_name,
-                ),
+            compel = Compel(
+                tokenizer=tokenizer,
+                text_encoder=text_encoder,
+                textual_inversion_manager=ti_manager,
+                dtype_for_device_getter=torch_dtype,
+                truncate_long_prompts=True,  # TODO:
             )
 
+            conjunction = Compel.parse_prompt_string(self.prompt)
+            prompt: Union[FlattenedPrompt, Blend] = conjunction.prompts[0]
+
+            if context.services.configuration.log_tokenization:
+                log_tokenization_for_prompt_object(prompt, tokenizer)
+
+            c, options = compel.build_conditioning_tensor_for_prompt_object(
+                prompt)
+
+            # TODO: long prompt support
+            # if not self.truncate_long_prompts:
+            #    [c, uc] = compel.pad_conditioning_tensors_to_same_length([c, uc])
+            ec = InvokeAIDiffuserComponent.ExtraConditioningInfo(
+                tokens_count_including_eos_bos=get_max_token_count(
+                    tokenizer, conjunction),
+                cross_attention_control_args=options.get(
+                    "cross_attention_control", None),)
+
+        conditioning_name = f"{context.graph_execution_state_id}_{self.id}_conditioning"
+
+        # TODO: hacky but works ;D maybe rename latents somehow?
+        context.services.latents.save(conditioning_name, (c, ec))
+
+        return CompelOutput(
+            conditioning=ConditioningField(
+                conditioning_name=conditioning_name,
+            ),
+        )
+
+class ClipSkipInvocationOutput(BaseInvocationOutput):
+    """Clip skip node output"""
+    type: Literal["clip_skip_output"] = "clip_skip_output"
+    clip: ClipField = Field(None, description="Clip with skipped layers")
+
+class ClipSkipInvocation(BaseInvocation):
+    """Skip layers in clip text_encoder model."""
+    type: Literal["clip_skip"] = "clip_skip"
+
+    clip: ClipField = Field(None, description="Clip to use")
+    skipped_layers: int = Field(0, description="Number of layers to skip in text_encoder")
+
+    def invoke(self, context: InvocationContext) -> ClipSkipInvocationOutput:
+        self.clip.skipped_layers += self.skipped_layers
+        return ClipSkipInvocationOutput(
+            clip=self.clip,
+        )
+
 
 def get_max_token_count(
-    tokenizer, prompt: Union[FlattenedPrompt, Blend, Conjunction], truncate_if_too_long=False
-) -> int:
+        tokenizer, prompt: Union[FlattenedPrompt, Blend, Conjunction],
+        truncate_if_too_long=False) -> int:
     if type(prompt) is Blend:
         blend: Blend = prompt
         return max(
@@ -146,13 +172,13 @@ def get_max_token_count(
         )
     else:
         return len(
-            get_tokens_for_prompt_object(tokenizer, prompt, truncate_if_too_long)
-        )
+            get_tokens_for_prompt_object(
+                tokenizer, prompt, truncate_if_too_long))
 
 
 def get_tokens_for_prompt_object(
     tokenizer, parsed_prompt: FlattenedPrompt, truncate_if_too_long=True
-) -> [str]:
+) -> List[str]:
     if type(parsed_prompt) is Blend:
         raise ValueError(
             "Blend is not supported here - you need to get tokens for each of its .children"
@@ -181,7 +207,7 @@ def log_tokenization_for_conjunction(
 ):
     display_label_prefix = display_label_prefix or ""
     for i, p in enumerate(c.prompts):
-        if len(c.prompts)>1:
+        if len(c.prompts) > 1:
             this_display_label_prefix = f"{display_label_prefix}(conjunction part {i + 1}, weight={c.weights[i]})"
         else:
             this_display_label_prefix = display_label_prefix
@@ -236,7 +262,8 @@ def log_tokenization_for_prompt_object(
             )
 
 
-def log_tokenization_for_text(text, tokenizer, display_label=None, truncate_if_too_long=False):
+def log_tokenization_for_text(
+        text, tokenizer, display_label=None, truncate_if_too_long=False):
     """shows how the prompt is tokenized
     # usually tokens have '</w>' to indicate end-of-word,
     # but for readability it has been replaced with ' '

invokeai/app/invocations/controlnet_image_processors.py

@@ -6,7 +6,7 @@ from builtins import float, bool
 import cv2
 import numpy as np
 from typing import Literal, Optional, Union, List, Dict
-from PIL import Image, ImageFilter, ImageOps
+from PIL import Image
 from pydantic import BaseModel, Field, validator
 
 from ..models.image import ImageField, ImageCategory, ResourceOrigin
@@ -422,9 +422,9 @@ class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation, PILInvoca
     # Inputs
     detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
     image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
-    h: Union[int, None] = Field(default=512, ge=0, description="Content shuffle `h` parameter")
-    w: Union[int, None] = Field(default=512, ge=0, description="Content shuffle `w` parameter")
-    f: Union[int, None] = Field(default=256, ge=0, description="Content shuffle `f` parameter")
+    h: Optional[int] = Field(default=512, ge=0, description="Content shuffle `h` parameter")
+    w: Optional[int] = Field(default=512, ge=0, description="Content shuffle `w` parameter")
+    f: Optional[int] = Field(default=256, ge=0, description="Content shuffle `f` parameter")
     # fmt: on
 
     def run_processor(self, image):

invokeai/app/invocations/generate.py

@@ -1,11 +1,10 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
 from functools import partial
-from typing import Literal, Optional, Union, get_args
+from typing import Literal, Optional, get_args
 
 import torch
-from diffusers import ControlNetModel
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from invokeai.app.models.image import (ColorField, ImageCategory, ImageField,
                                        ResourceOrigin)
@@ -18,7 +17,6 @@ from ..util.step_callback import stable_diffusion_step_callback
 from .baseinvocation import BaseInvocation, InvocationConfig, InvocationContext
 from .image import ImageOutput
 
-import re
 from ...backend.model_management.lora import ModelPatcher
 from ...backend.stable_diffusion.diffusers_pipeline import StableDiffusionGeneratorPipeline
 from .model import UNetField, VaeField
@@ -76,7 +74,7 @@ class InpaintInvocation(BaseInvocation):
     vae: VaeField = Field(default=None, description="Vae model")
 
     # Inputs
-    image: Union[ImageField, None] = Field(description="The input image")
+    image: Optional[ImageField] = Field(description="The input image")
     strength: float = Field(
         default=0.75, gt=0, le=1, description="The strength of the original image"
     )
@@ -86,7 +84,7 @@ class InpaintInvocation(BaseInvocation):
     )
 
     # Inputs
-    mask: Union[ImageField, None] = Field(description="The mask")
+    mask: Optional[ImageField] = Field(description="The mask")
     seam_size: int = Field(default=96, ge=1, description="The seam inpaint size (px)")
     seam_blur: int = Field(
         default=16, ge=0, description="The seam inpaint blur radius (px)"

invokeai/app/invocations/image.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
-import io
-from typing import Literal, Optional, Union
+from typing import Literal, Optional
 
 import numpy
 from PIL import Image, ImageFilter, ImageOps, ImageChops
@@ -67,7 +66,7 @@ class LoadImageInvocation(BaseInvocation):
     type: Literal["load_image"] = "load_image"
 
     # Inputs
-    image: Union[ImageField, None] = Field(
+    image: Optional[ImageField] = Field(
         default=None, description="The image to load"
     )
     # fmt: on
@@ -87,7 +86,7 @@ class ShowImageInvocation(BaseInvocation):
     type: Literal["show_image"] = "show_image"
 
     # Inputs
-    image: Union[ImageField, None] = Field(
+    image: Optional[ImageField] = Field(
         default=None, description="The image to show"
     )
 
@@ -112,7 +111,7 @@ class ImageCropInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_crop"] = "img_crop"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to crop")
+    image: Optional[ImageField]  = Field(default=None, description="The image to crop")
     x:      int = Field(default=0, description="The left x coordinate of the crop rectangle")
     y:      int = Field(default=0, description="The top y coordinate of the crop rectangle")
     width:  int = Field(default=512, gt=0, description="The width of the crop rectangle")
@@ -150,8 +149,8 @@ class ImagePasteInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_paste"] = "img_paste"
 
     # Inputs
-    base_image:     Union[ImageField, None]  = Field(default=None, description="The base image")
-    image:          Union[ImageField, None]  = Field(default=None, description="The image to paste")
+    base_image:     Optional[ImageField]  = Field(default=None, description="The base image")
+    image:          Optional[ImageField]  = Field(default=None, description="The image to paste")
     mask: Optional[ImageField] = Field(default=None, description="The mask to use when pasting")
     x:                     int = Field(default=0, description="The left x coordinate at which to paste the image")
     y:                     int = Field(default=0, description="The top y coordinate at which to paste the image")
@@ -203,7 +202,7 @@ class MaskFromAlphaInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["tomask"] = "tomask"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to create the mask from")
+    image: Optional[ImageField]  = Field(default=None, description="The image to create the mask from")
     invert:      bool = Field(default=False, description="Whether or not to invert the mask")
     # fmt: on
 
@@ -237,8 +236,8 @@ class ImageMultiplyInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_mul"] = "img_mul"
 
     # Inputs
-    image1: Union[ImageField, None]  = Field(default=None, description="The first image to multiply")
-    image2: Union[ImageField, None]  = Field(default=None, description="The second image to multiply")
+    image1: Optional[ImageField]  = Field(default=None, description="The first image to multiply")
+    image2: Optional[ImageField]  = Field(default=None, description="The second image to multiply")
     # fmt: on
 
     def invoke(self, context: InvocationContext) -> ImageOutput:
@@ -273,7 +272,7 @@ class ImageChannelInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_chan"] = "img_chan"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to get the channel from")
+    image: Optional[ImageField]  = Field(default=None, description="The image to get the channel from")
     channel: IMAGE_CHANNELS  = Field(default="A", description="The channel to get")
     # fmt: on
 
@@ -308,7 +307,7 @@ class ImageConvertInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_conv"] = "img_conv"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to convert")
+    image: Optional[ImageField]  = Field(default=None, description="The image to convert")
     mode: IMAGE_MODES  = Field(default="L", description="The mode to convert to")
     # fmt: on
 
@@ -340,7 +339,7 @@ class ImageBlurInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_blur"] = "img_blur"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to blur")
+    image: Optional[ImageField]  = Field(default=None, description="The image to blur")
     radius:     float = Field(default=8.0, ge=0, description="The blur radius")
     blur_type: Literal["gaussian", "box"] = Field(default="gaussian", description="The type of blur")
     # fmt: on
@@ -398,7 +397,7 @@ class ImageResizeInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_resize"] = "img_resize"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to resize")
+    image: Optional[ImageField]  = Field(default=None, description="The image to resize")
     width:                         int = Field(ge=64, multiple_of=8, description="The width to resize to (px)")
     height:                        int = Field(ge=64, multiple_of=8, description="The height to resize to (px)")
     resample_mode:  PIL_RESAMPLING_MODES = Field(default="bicubic", description="The resampling mode")
@@ -437,7 +436,7 @@ class ImageScaleInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_scale"] = "img_scale"
 
     # Inputs
-    image:       Union[ImageField, None] = Field(default=None, description="The image to scale")
+    image:       Optional[ImageField] = Field(default=None, description="The image to scale")
     scale_factor:                  float = Field(gt=0, description="The factor by which to scale the image")
     resample_mode:  PIL_RESAMPLING_MODES = Field(default="bicubic", description="The resampling mode")
     # fmt: on
@@ -477,7 +476,7 @@ class ImageLerpInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_lerp"] = "img_lerp"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to lerp")
+    image: Optional[ImageField]  = Field(default=None, description="The image to lerp")
     min: int = Field(default=0, ge=0, le=255, description="The minimum output value")
     max: int = Field(default=255, ge=0, le=255, description="The maximum output value")
     # fmt: on
@@ -513,7 +512,7 @@ class ImageInverseLerpInvocation(BaseInvocation, PILInvocationConfig):
     type: Literal["img_ilerp"] = "img_ilerp"
 
     # Inputs
-    image: Union[ImageField, None]  = Field(default=None, description="The image to lerp")
+    image: Optional[ImageField]  = Field(default=None, description="The image to lerp")
     min: int = Field(default=0, ge=0, le=255, description="The minimum input value")
     max: int = Field(default=255, ge=0, le=255, description="The maximum input value")
     # fmt: on

invokeai/app/invocations/infill.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654) and the InvokeAI Team
 
-from typing import Literal, Union, get_args
+from typing import Literal, Optional, get_args
 
 import numpy as np
 import math
@@ -68,7 +68,7 @@ def get_tile_images(image: np.ndarray, width=8, height=8):
 
 
 def tile_fill_missing(
-    im: Image.Image, tile_size: int = 16, seed: Union[int, None] = None
+    im: Image.Image, tile_size: int = 16, seed: Optional[int] = None
 ) -> Image.Image:
     # Only fill if there's an alpha layer
     if im.mode != "RGBA":
@@ -125,7 +125,7 @@ class InfillColorInvocation(BaseInvocation):
     """Infills transparent areas of an image with a solid color"""
 
     type: Literal["infill_rgba"] = "infill_rgba"
-    image: Union[ImageField, None] = Field(
+    image: Optional[ImageField] = Field(
         default=None, description="The image to infill"
     )
     color: ColorField = Field(
@@ -162,7 +162,7 @@ class InfillTileInvocation(BaseInvocation):
 
     type: Literal["infill_tile"] = "infill_tile"
 
-    image: Union[ImageField, None] = Field(
+    image: Optional[ImageField] = Field(
         default=None, description="The image to infill"
     )
     tile_size: int = Field(default=32, ge=1, description="The tile size (px)")
@@ -202,7 +202,7 @@ class InfillPatchMatchInvocation(BaseInvocation):
 
     type: Literal["infill_patchmatch"] = "infill_patchmatch"
 
-    image: Union[ImageField, None] = Field(
+    image: Optional[ImageField] = Field(
         default=None, description="The image to infill"
     )
 

invokeai/app/invocations/latent.py

@@ -1,21 +1,18 @@
 # Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654)
 
-from contextlib import ExitStack
 from typing import List, Literal, Optional, Union
 
 import einops
-
-from pydantic import BaseModel, Field, validator
 import torch
-from diffusers import ControlNetModel, DPMSolverMultistepScheduler
+from diffusers import ControlNetModel
 from diffusers.image_processor import VaeImageProcessor
 from diffusers.schedulers import SchedulerMixin as Scheduler
+from pydantic import BaseModel, Field, validator
 
-from invokeai.app.util.misc import SEED_MAX, get_random_seed
 from invokeai.app.util.step_callback import stable_diffusion_step_callback
 
 from ..models.image import ImageCategory, ImageField, ResourceOrigin
-from ...backend.image_util.seamless import configure_model_padding
+from ...backend.model_management.lora import ModelPatcher
 from ...backend.stable_diffusion import PipelineIntermediateState
 from ...backend.stable_diffusion.diffusers_pipeline import (
     ConditioningData, ControlNetData, StableDiffusionGeneratorPipeline,
@@ -24,7 +21,6 @@ from ...backend.stable_diffusion.diffusion.shared_invokeai_diffusion import \
     PostprocessingSettings
 from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP
 from ...backend.util.devices import torch_dtype
-from ...backend.model_management.lora import ModelPatcher
 from .baseinvocation import (BaseInvocation, BaseInvocationOutput,
                              InvocationConfig, InvocationContext)
 from .compel import ConditioningField
@@ -32,14 +28,17 @@ from .controlnet_image_processors import ControlField
 from .image import ImageOutput
 from .model import ModelInfo, UNetField, VaeField
 
+
 class LatentsField(BaseModel):
     """A latents field used for passing latents between invocations"""
 
-    latents_name: Optional[str] = Field(default=None, description="The name of the latents")
+    latents_name: Optional[str] = Field(
+        default=None, description="The name of the latents")
 
     class Config:
         schema_extra = {"required": ["latents_name"]}
 
+
 class LatentsOutput(BaseInvocationOutput):
     """Base class for invocations that output latents"""
     #fmt: off
@@ -53,11 +52,11 @@ class LatentsOutput(BaseInvocationOutput):
 
 
 def build_latents_output(latents_name: str, latents: torch.Tensor):
-      return LatentsOutput(
-          latents=LatentsField(latents_name=latents_name),
-          width=latents.size()[3] * 8,
-          height=latents.size()[2] * 8,
-      )
+    return LatentsOutput(
+        latents=LatentsField(latents_name=latents_name),
+        width=latents.size()[3] * 8,
+        height=latents.size()[2] * 8,
+    )
 
 
 SAMPLER_NAME_VALUES = Literal[
@@ -70,16 +69,19 @@ def get_scheduler(
     scheduler_info: ModelInfo,
     scheduler_name: str,
 ) -> Scheduler:
-    scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(scheduler_name, SCHEDULER_MAP['ddim'])
-    orig_scheduler_info = context.services.model_manager.get_model(**scheduler_info.dict())
+    scheduler_class, scheduler_extra_config = SCHEDULER_MAP.get(
+        scheduler_name, SCHEDULER_MAP['ddim'])
+    orig_scheduler_info = context.services.model_manager.get_model(
+        **scheduler_info.dict())
     with orig_scheduler_info as orig_scheduler:
         scheduler_config = orig_scheduler.config
-        
+
     if "_backup" in scheduler_config:
         scheduler_config = scheduler_config["_backup"]
-    scheduler_config = {**scheduler_config, **scheduler_extra_config, "_backup": scheduler_config}
+    scheduler_config = {**scheduler_config, **
+                        scheduler_extra_config, "_backup": scheduler_config}
     scheduler = scheduler_class.from_config(scheduler_config)
-    
+
     # hack copied over from generate.py
     if not hasattr(scheduler, 'uses_inpainting_model'):
         scheduler.uses_inpainting_model = lambda: False
@@ -124,18 +126,18 @@ class TextToLatentsInvocation(BaseInvocation):
             "ui": {
                 "tags": ["latents"],
                 "type_hints": {
-                  "model": "model",
-                  "control": "control",
-                  # "cfg_scale": "float",
-                  "cfg_scale": "number"
+                    "model": "model",
+                    "control": "control",
+                    # "cfg_scale": "float",
+                    "cfg_scale": "number"
                 }
             },
         }
 
     # TODO: pass this an emitter method or something? or a session for dispatching?
     def dispatch_progress(
-        self, context: InvocationContext, source_node_id: str, intermediate_state: PipelineIntermediateState
-    ) -> None:
+            self, context: InvocationContext, source_node_id: str,
+            intermediate_state: PipelineIntermediateState) -> None:
         stable_diffusion_step_callback(
             context=context,
             intermediate_state=intermediate_state,
@@ -143,9 +145,12 @@ class TextToLatentsInvocation(BaseInvocation):
             source_node_id=source_node_id,
         )
 
-    def get_conditioning_data(self, context: InvocationContext, scheduler) -> ConditioningData:
-        c, extra_conditioning_info = context.services.latents.get(self.positive_conditioning.conditioning_name)
-        uc, _ = context.services.latents.get(self.negative_conditioning.conditioning_name)
+    def get_conditioning_data(
+            self, context: InvocationContext, scheduler) -> ConditioningData:
+        c, extra_conditioning_info = context.services.latents.get(
+            self.positive_conditioning.conditioning_name)
+        uc, _ = context.services.latents.get(
+            self.negative_conditioning.conditioning_name)
 
         conditioning_data = ConditioningData(
             unconditioned_embeddings=uc,
@@ -153,10 +158,10 @@ class TextToLatentsInvocation(BaseInvocation):
             guidance_scale=self.cfg_scale,
             extra=extra_conditioning_info,
             postprocessing_settings=PostprocessingSettings(
-                threshold=0.0,#threshold,
-                warmup=0.2,#warmup,
-                h_symmetry_time_pct=None,#h_symmetry_time_pct,
-                v_symmetry_time_pct=None#v_symmetry_time_pct,
+                threshold=0.0,  # threshold,
+                warmup=0.2,  # warmup,
+                h_symmetry_time_pct=None,  # h_symmetry_time_pct,
+                v_symmetry_time_pct=None  # v_symmetry_time_pct,
             ),
         )
 
@@ -164,31 +169,32 @@ class TextToLatentsInvocation(BaseInvocation):
             scheduler,
 
             # for ddim scheduler
-            eta=0.0, #ddim_eta
+            eta=0.0,  # ddim_eta
 
             # for ancestral and sde schedulers
             generator=torch.Generator(device=uc.device).manual_seed(0),
         )
         return conditioning_data
 
-    def create_pipeline(self, unet, scheduler) -> StableDiffusionGeneratorPipeline:
+    def create_pipeline(
+            self, unet, scheduler) -> StableDiffusionGeneratorPipeline:
         # TODO:
-        #configure_model_padding(
+        # configure_model_padding(
         #    unet,
         #    self.seamless,
         #    self.seamless_axes,
-        #)
+        # )
 
         class FakeVae:
             class FakeVaeConfig:
                 def __init__(self):
                     self.block_out_channels = [0]
-            
+
             def __init__(self):
                 self.config = FakeVae.FakeVaeConfig()
 
         return StableDiffusionGeneratorPipeline(
-            vae=FakeVae(), # TODO: oh...
+            vae=FakeVae(),  # TODO: oh...
             text_encoder=None,
             tokenizer=None,
             unet=unet,
@@ -198,11 +204,12 @@ class TextToLatentsInvocation(BaseInvocation):
             requires_safety_checker=False,
             precision="float16" if unet.dtype == torch.float16 else "float32",
         )
-    
+
     def prep_control_data(
         self,
         context: InvocationContext,
-        model: StableDiffusionGeneratorPipeline, # really only need model for dtype and device
+        # really only need model for dtype and device
+        model: StableDiffusionGeneratorPipeline,
         control_input: List[ControlField],
         latents_shape: List[int],
         do_classifier_free_guidance: bool = True,
@@ -238,15 +245,17 @@ class TextToLatentsInvocation(BaseInvocation):
                     print("Using HF model subfolders")
                     print("    control_name: ", control_name)
                     print("    control_subfolder: ", control_subfolder)
-                    control_model = ControlNetModel.from_pretrained(control_name,
-                                                                    subfolder=control_subfolder,
-                                                                    torch_dtype=model.unet.dtype).to(model.device)
+                    control_model = ControlNetModel.from_pretrained(
+                        control_name, subfolder=control_subfolder,
+                        torch_dtype=model.unet.dtype).to(
+                        model.device)
                 else:
-                    control_model = ControlNetModel.from_pretrained(control_info.control_model,
-                                                                    torch_dtype=model.unet.dtype).to(model.device)
+                    control_model = ControlNetModel.from_pretrained(
+                        control_info.control_model, torch_dtype=model.unet.dtype).to(model.device)
                 control_models.append(control_model)
                 control_image_field = control_info.image
-                input_image = context.services.images.get_pil_image(control_image_field.image_name)
+                input_image = context.services.images.get_pil_image(
+                    control_image_field.image_name)
                 # self.image.image_type, self.image.image_name
                 # FIXME: still need to test with different widths, heights, devices, dtypes
                 #        and add in batch_size, num_images_per_prompt?
@@ -263,41 +272,50 @@ class TextToLatentsInvocation(BaseInvocation):
                     dtype=control_model.dtype,
                     control_mode=control_info.control_mode,
                 )
-                control_item = ControlNetData(model=control_model,
-                                              image_tensor=control_image,
-                                              weight=control_info.control_weight,
-                                              begin_step_percent=control_info.begin_step_percent,
-                                              end_step_percent=control_info.end_step_percent,
-                                              control_mode=control_info.control_mode,
-                                              )
+                control_item = ControlNetData(
+                    model=control_model, image_tensor=control_image,
+                    weight=control_info.control_weight,
+                    begin_step_percent=control_info.begin_step_percent,
+                    end_step_percent=control_info.end_step_percent,
+                    control_mode=control_info.control_mode,)
                 control_data.append(control_item)
                 # MultiControlNetModel has been refactored out, just need list[ControlNetData]
         return control_data
 
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         noise = context.services.latents.get(self.noise.latents_name)
 
         # Get the source node id (we are invoking the prepared node)
-        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
+        graph_execution_state = context.services.graph_execution_manager.get(
+            context.graph_execution_state_id)
         source_node_id = graph_execution_state.prepared_source_mapping[self.id]
 
         def step_callback(state: PipelineIntermediateState):
             self.dispatch_progress(context, source_node_id, state)
 
-        unet_info = context.services.model_manager.get_model(**self.unet.unet.dict())
-        with unet_info as unet:
+        def _lora_loader():
+            for lora in self.unet.loras:
+                lora_info = context.services.model_manager.get_model(
+                    **lora.dict(exclude={"weight"}))
+                yield (lora_info.context.model, lora.weight)
+                del lora_info
+            return
+
+        unet_info = context.services.model_manager.get_model(
+            **self.unet.unet.dict())
+        with ModelPatcher.apply_lora_unet(unet_info.context.model, _lora_loader()),\
+                unet_info as unet:
 
             scheduler = get_scheduler(
                 context=context,
                 scheduler_info=self.unet.scheduler,
                 scheduler_name=self.scheduler,
             )
-            
+
             pipeline = self.create_pipeline(unet, scheduler)
             conditioning_data = self.get_conditioning_data(context, scheduler)
 
-            loras = [(context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.unet.loras]
-
             control_data = self.prep_control_data(
                 model=pipeline, context=context, control_input=self.control,
                 latents_shape=noise.shape,
@@ -305,16 +323,15 @@ class TextToLatentsInvocation(BaseInvocation):
                 do_classifier_free_guidance=True,
             )
 
-            with ModelPatcher.apply_lora_unet(pipeline.unet, loras):
-                # TODO: Verify the noise is the right size
-                result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
-                    latents=torch.zeros_like(noise, dtype=torch_dtype(unet.device)),
-                    noise=noise,
-                    num_inference_steps=self.steps,
-                    conditioning_data=conditioning_data,
-                    control_data=control_data, # list[ControlNetData]
-                    callback=step_callback,
-                )
+            # TODO: Verify the noise is the right size
+            result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
+                latents=torch.zeros_like(noise, dtype=torch_dtype(unet.device)),
+                noise=noise,
+                num_inference_steps=self.steps,
+                conditioning_data=conditioning_data,
+                control_data=control_data,  # list[ControlNetData]
+                callback=step_callback,
+            )
 
         # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699
         torch.cuda.empty_cache()
@@ -323,14 +340,18 @@ class TextToLatentsInvocation(BaseInvocation):
         context.services.latents.save(name, result_latents)
         return build_latents_output(latents_name=name, latents=result_latents)
 
+
 class LatentsToLatentsInvocation(TextToLatentsInvocation):
     """Generates latents using latents as base image."""
 
     type: Literal["l2l"] = "l2l"
 
     # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to use as a base image")
-    strength: float = Field(default=0.7, ge=0, le=1, description="The strength of the latents to use")
+    latents: Optional[LatentsField] = Field(
+        description="The latents to use as a base image")
+    strength: float = Field(
+        default=0.7, ge=0, le=1,
+        description="The strength of the latents to use")
 
     # Schema customisation
     class Config(InvocationConfig):
@@ -345,22 +366,31 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
             },
         }
 
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         noise = context.services.latents.get(self.noise.latents_name)
         latent = context.services.latents.get(self.latents.latents_name)
 
         # Get the source node id (we are invoking the prepared node)
-        graph_execution_state = context.services.graph_execution_manager.get(context.graph_execution_state_id)
+        graph_execution_state = context.services.graph_execution_manager.get(
+            context.graph_execution_state_id)
         source_node_id = graph_execution_state.prepared_source_mapping[self.id]
 
         def step_callback(state: PipelineIntermediateState):
             self.dispatch_progress(context, source_node_id, state)
 
-        unet_info = context.services.model_manager.get_model(
-            **self.unet.unet.dict(),
-        )
+        def _lora_loader():
+            for lora in self.unet.loras:
+                lora_info = context.services.model_manager.get_model(
+                    **lora.dict(exclude={"weight"}))
+                yield (lora_info.context.model, lora.weight)
+                del lora_info
+            return
 
-        with unet_info as unet:
+        unet_info = context.services.model_manager.get_model(
+            **self.unet.unet.dict())
+        with ModelPatcher.apply_lora_unet(unet_info.context.model, _lora_loader()),\
+                unet_info as unet:
 
             scheduler = get_scheduler(
                 context=context,
@@ -370,7 +400,7 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
 
             pipeline = self.create_pipeline(unet, scheduler)
             conditioning_data = self.get_conditioning_data(context, scheduler)
-            
+
             control_data = self.prep_control_data(
                 model=pipeline, context=context, control_input=self.control,
                 latents_shape=noise.shape,
@@ -380,8 +410,7 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
 
             # TODO: Verify the noise is the right size
             initial_latents = latent if self.strength < 1.0 else torch.zeros_like(
-                latent, device=unet.device, dtype=latent.dtype
-            )
+                latent, device=unet.device, dtype=latent.dtype)
 
             timesteps, _ = pipeline.get_img2img_timesteps(
                 self.steps,
@@ -389,18 +418,15 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
                 device=unet.device,
             )
 
-            loras = [(context.services.model_manager.get_model(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.unet.loras]
-
-            with ModelPatcher.apply_lora_unet(pipeline.unet, loras):
-                result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
-                    latents=initial_latents,
-                    timesteps=timesteps,
-                    noise=noise,
-                    num_inference_steps=self.steps,
-                    conditioning_data=conditioning_data,
-                    control_data=control_data,  # list[ControlNetData]
-                    callback=step_callback
-                )
+            result_latents, result_attention_map_saver = pipeline.latents_from_embeddings(
+                latents=initial_latents,
+                timesteps=timesteps,
+                noise=noise,
+                num_inference_steps=self.steps,
+                conditioning_data=conditioning_data,
+                control_data=control_data,  # list[ControlNetData]
+                callback=step_callback
+            )
 
         # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699
         torch.cuda.empty_cache()
@@ -417,9 +443,12 @@ class LatentsToImageInvocation(BaseInvocation):
     type: Literal["l2i"] = "l2i"
 
     # Inputs
-    latents: Optional[LatentsField] = Field(description="The latents to generate an image from")
+    latents: Optional[LatentsField] = Field(
+        description="The latents to generate an image from")
     vae: VaeField = Field(default=None, description="Vae submodel")
-    tiled: bool = Field(default=False, description="Decode latents by overlaping tiles(less memory consumption)")
+    tiled: bool = Field(
+        default=False,
+        description="Decode latents by overlaping tiles(less memory consumption)")
 
     # Schema customisation
     class Config(InvocationConfig):
@@ -450,7 +479,7 @@ class LatentsToImageInvocation(BaseInvocation):
                 # copied from diffusers pipeline
                 latents = latents / vae.config.scaling_factor
                 image = vae.decode(latents, return_dict=False)[0]
-                image = (image / 2 + 0.5).clamp(0, 1) # denormalize
+                image = (image / 2 + 0.5).clamp(0, 1)  # denormalize
                 # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
                 np_image = image.cpu().permute(0, 2, 3, 1).float().numpy()
 
@@ -473,9 +502,9 @@ class LatentsToImageInvocation(BaseInvocation):
             height=image_dto.height,
         )
 
-LATENTS_INTERPOLATION_MODE = Literal[
-    "nearest", "linear", "bilinear", "bicubic", "trilinear", "area", "nearest-exact"
-]
+
+LATENTS_INTERPOLATION_MODE = Literal["nearest", "linear",
+                                     "bilinear", "bicubic", "trilinear", "area", "nearest-exact"]
 
 
 class ResizeLatentsInvocation(BaseInvocation):
@@ -484,21 +513,25 @@ class ResizeLatentsInvocation(BaseInvocation):
     type: Literal["lresize"] = "lresize"
 
     # Inputs
-    latents:    Optional[LatentsField] = Field(description="The latents to resize")
-    width:                         int = Field(ge=64, multiple_of=8, description="The width to resize to (px)")
-    height:                        int = Field(ge=64, multiple_of=8, description="The height to resize to (px)")
-    mode:   LATENTS_INTERPOLATION_MODE = Field(default="bilinear", description="The interpolation mode")
-    antialias:                    bool = Field(default=False, description="Whether or not to antialias (applied in bilinear and bicubic modes only)")
+    latents: Optional[LatentsField] = Field(
+        description="The latents to resize")
+    width:                         int = Field(
+        ge=64, multiple_of=8, description="The width to resize to (px)")
+    height:                        int = Field(
+        ge=64, multiple_of=8, description="The height to resize to (px)")
+    mode:   LATENTS_INTERPOLATION_MODE = Field(
+        default="bilinear", description="The interpolation mode")
+    antialias: bool = Field(
+        default=False,
+        description="Whether or not to antialias (applied in bilinear and bicubic modes only)")
 
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         latents = context.services.latents.get(self.latents.latents_name)
 
         resized_latents = torch.nn.functional.interpolate(
-            latents,
-            size=(self.height // 8, self.width // 8),
-            mode=self.mode,
-            antialias=self.antialias if self.mode in ["bilinear", "bicubic"] else False,
-        )
+            latents, size=(self.height // 8, self.width // 8),
+            mode=self.mode, antialias=self.antialias
+            if self.mode in ["bilinear", "bicubic"] else False,)
 
         # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699
         torch.cuda.empty_cache()
@@ -515,21 +548,24 @@ class ScaleLatentsInvocation(BaseInvocation):
     type: Literal["lscale"] = "lscale"
 
     # Inputs
-    latents:   Optional[LatentsField] = Field(description="The latents to scale")
-    scale_factor:               float = Field(gt=0, description="The factor by which to scale the latents")
-    mode:  LATENTS_INTERPOLATION_MODE = Field(default="bilinear", description="The interpolation mode")
-    antialias:                   bool = Field(default=False, description="Whether or not to antialias (applied in bilinear and bicubic modes only)")
+    latents: Optional[LatentsField] = Field(
+        description="The latents to scale")
+    scale_factor:               float = Field(
+        gt=0, description="The factor by which to scale the latents")
+    mode:  LATENTS_INTERPOLATION_MODE = Field(
+        default="bilinear", description="The interpolation mode")
+    antialias: bool = Field(
+        default=False,
+        description="Whether or not to antialias (applied in bilinear and bicubic modes only)")
 
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         latents = context.services.latents.get(self.latents.latents_name)
 
         # resizing
         resized_latents = torch.nn.functional.interpolate(
-            latents,
-            scale_factor=self.scale_factor,
-            mode=self.mode,
-            antialias=self.antialias if self.mode in ["bilinear", "bicubic"] else False,
-        )
+            latents, scale_factor=self.scale_factor, mode=self.mode,
+            antialias=self.antialias
+            if self.mode in ["bilinear", "bicubic"] else False,)
 
         # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699
         torch.cuda.empty_cache()
@@ -546,9 +582,11 @@ class ImageToLatentsInvocation(BaseInvocation):
     type: Literal["i2l"] = "i2l"
 
     # Inputs
-    image: Union[ImageField, None] = Field(description="The image to encode")
+    image: Optional[ImageField] = Field(description="The image to encode")
     vae: VaeField = Field(default=None, description="Vae submodel")
-    tiled: bool = Field(default=False, description="Encode latents by overlaping tiles(less memory consumption)")
+    tiled: bool = Field(
+        default=False,
+        description="Encode latents by overlaping tiles(less memory consumption)")
 
     # Schema customisation
     class Config(InvocationConfig):

invokeai/app/invocations/model.py

@@ -1,31 +1,39 @@
-from typing import Literal, Optional, Union, List
-from pydantic import BaseModel, Field
 import copy
+from typing import List, Literal, Optional, Union
 
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, InvocationContext, InvocationConfig
+from pydantic import BaseModel, Field
 
-from ...backend.util.devices import choose_torch_device, torch_dtype
 from ...backend.model_management import BaseModelType, ModelType, SubModelType
+from .baseinvocation import (BaseInvocation, BaseInvocationOutput,
+                             InvocationConfig, InvocationContext)
+
 
 class ModelInfo(BaseModel):
     model_name: str = Field(description="Info to load submodel")
     base_model: BaseModelType = Field(description="Base model")
     model_type: ModelType = Field(description="Info to load submodel")
-    submodel: Optional[SubModelType] = Field(description="Info to load submodel")
+    submodel: Optional[SubModelType] = Field(
+        default=None, description="Info to load submodel"
+    )
+
 
 class LoraInfo(ModelInfo):
     weight: float = Field(description="Lora's weight which to use when apply to model")
 
+
 class UNetField(BaseModel):
     unet: ModelInfo = Field(description="Info to load unet submodel")
     scheduler: ModelInfo = Field(description="Info to load scheduler submodel")
     loras: List[LoraInfo] = Field(description="Loras to apply on model loading")
 
+
 class ClipField(BaseModel):
     tokenizer: ModelInfo = Field(description="Info to load tokenizer submodel")
     text_encoder: ModelInfo = Field(description="Info to load text_encoder submodel")
+    skipped_layers: int = Field(description="Number of skipped layers in text_encoder")
     loras: List[LoraInfo] = Field(description="Loras to apply on model loading")
 
+
 class VaeField(BaseModel):
     # TODO: better naming?
     vae: ModelInfo = Field(description="Info to load vae submodel")
@@ -34,43 +42,48 @@ class VaeField(BaseModel):
 class ModelLoaderOutput(BaseInvocationOutput):
     """Model loader output"""
 
-    #fmt: off
+    # fmt: off
     type: Literal["model_loader_output"] = "model_loader_output"
 
     unet: UNetField = Field(default=None, description="UNet submodel")
     clip: ClipField = Field(default=None, description="Tokenizer and text_encoder submodels")
     vae: VaeField = Field(default=None, description="Vae submodel")
-    #fmt: on
+    # fmt: on
 
 
-class PipelineModelField(BaseModel):
-    """Pipeline model field"""
+class MainModelField(BaseModel):
+    """Main model field"""
 
     model_name: str = Field(description="Name of the model")
     base_model: BaseModelType = Field(description="Base model")
 
 
-class PipelineModelLoaderInvocation(BaseInvocation):
-    """Loads a pipeline model, outputting its submodels."""
+class LoRAModelField(BaseModel):
+    """LoRA model field"""
 
-    type: Literal["pipeline_model_loader"] = "pipeline_model_loader"
+    model_name: str = Field(description="Name of the LoRA model")
+    base_model: BaseModelType = Field(description="Base model")
 
-    model: PipelineModelField = Field(description="The model to load")
+
+class MainModelLoaderInvocation(BaseInvocation):
+    """Loads a main model, outputting its submodels."""
+
+    type: Literal["main_model_loader"] = "main_model_loader"
+
+    model: MainModelField = Field(description="The model to load")
     # TODO: precision?
 
     # Schema customisation
     class Config(InvocationConfig):
         schema_extra = {
             "ui": {
+                "title": "Model Loader",
                 "tags": ["model", "loader"],
-                "type_hints": {
-                  "model": "model"
-                }
+                "type_hints": {"model": "model"},
             },
         }
 
     def invoke(self, context: InvocationContext) -> ModelLoaderOutput:
-
         base_model = self.model.base_model
         model_name = self.model.model_name
         model_type = ModelType.Main
@@ -112,7 +125,6 @@ class PipelineModelLoaderInvocation(BaseInvocation):
             )
         """
 
-
         return ModelLoaderOutput(
             unet=UNetField(
                 unet=ModelInfo(
@@ -143,6 +155,7 @@ class PipelineModelLoaderInvocation(BaseInvocation):
                     submodel=SubModelType.TextEncoder,
                 ),
                 loras=[],
+                skipped_layers=0,
             ),
             vae=VaeField(
                 vae=ModelInfo(
@@ -151,47 +164,66 @@ class PipelineModelLoaderInvocation(BaseInvocation):
                     model_type=model_type,
                     submodel=SubModelType.Vae,
                 ),
-            )
+            ),
         )
 
+
 class LoraLoaderOutput(BaseInvocationOutput):
     """Model loader output"""
 
-    #fmt: off
+    # fmt: off
     type: Literal["lora_loader_output"] = "lora_loader_output"
 
     unet: Optional[UNetField] = Field(default=None, description="UNet submodel")
     clip: Optional[ClipField] = Field(default=None, description="Tokenizer and text_encoder submodels")
-    #fmt: on
+    # fmt: on
+
 
 class LoraLoaderInvocation(BaseInvocation):
     """Apply selected lora to unet and text_encoder."""
 
     type: Literal["lora_loader"] = "lora_loader"
 
-    lora_name: str = Field(description="Lora model name")
+    lora: Union[LoRAModelField, None] = Field(
+        default=None, description="Lora model name"
+    )
     weight: float = Field(default=0.75, description="With what weight to apply lora")
 
     unet: Optional[UNetField] = Field(description="UNet model for applying lora")
     clip: Optional[ClipField] = Field(description="Clip model for applying lora")
 
-    def invoke(self, context: InvocationContext) -> LoraLoaderOutput:
+    class Config(InvocationConfig):
+        schema_extra = {
+            "ui": {
+                "title": "Lora Loader",
+                "tags": ["lora", "loader"],
+                "type_hints": {"lora": "lora_model"},
+            },
+        }
 
-        # TODO: ui rewrite
-        base_model = BaseModelType.StableDiffusion1
+    def invoke(self, context: InvocationContext) -> LoraLoaderOutput:
+        if self.lora is None:
+            raise Exception("No LoRA provided")
+
+        base_model = self.lora.base_model
+        lora_name = self.lora.model_name
 
         if not context.services.model_manager.model_exists(
             base_model=base_model,
-            model_name=self.lora_name,
+            model_name=lora_name,
             model_type=ModelType.Lora,
         ):
-            raise Exception(f"Unkown lora name: {self.lora_name}!")
+            raise Exception(f"Unkown lora name: {lora_name}!")
 
-        if self.unet is not None and any(lora.model_name == self.lora_name for lora in self.unet.loras):
-            raise Exception(f"Lora \"{self.lora_name}\" already applied to unet")
+        if self.unet is not None and any(
+            lora.model_name == lora_name for lora in self.unet.loras
+        ):
+            raise Exception(f'Lora "{lora_name}" already applied to unet')
 
-        if self.clip is not None and any(lora.model_name == self.lora_name for lora in self.clip.loras):
-            raise Exception(f"Lora \"{self.lora_name}\" already applied to clip")
+        if self.clip is not None and any(
+            lora.model_name == lora_name for lora in self.clip.loras
+        ):
+            raise Exception(f'Lora "{lora_name}" already applied to clip')
 
         output = LoraLoaderOutput()
 
@@ -200,7 +232,7 @@ class LoraLoaderInvocation(BaseInvocation):
             output.unet.loras.append(
                 LoraInfo(
                     base_model=base_model,
-                    model_name=self.lora_name,
+                    model_name=lora_name,
                     model_type=ModelType.Lora,
                     submodel=None,
                     weight=self.weight,
@@ -212,7 +244,7 @@ class LoraLoaderInvocation(BaseInvocation):
             output.clip.loras.append(
                 LoraInfo(
                     base_model=base_model,
-                    model_name=self.lora_name,
+                    model_name=lora_name,
                     model_type=ModelType.Lora,
                     submodel=None,
                     weight=self.weight,
@@ -221,3 +253,58 @@ class LoraLoaderInvocation(BaseInvocation):
 
         return output
 
+
+class VAEModelField(BaseModel):
+    """Vae model field"""
+
+    model_name: str = Field(description="Name of the model")
+    base_model: BaseModelType = Field(description="Base model")
+
+
+class VaeLoaderOutput(BaseInvocationOutput):
+    """Model loader output"""
+
+    # fmt: off
+    type: Literal["vae_loader_output"] = "vae_loader_output"
+
+    vae: VaeField = Field(default=None, description="Vae model")
+    # fmt: on
+
+
+class VaeLoaderInvocation(BaseInvocation):
+    """Loads a VAE model, outputting a VaeLoaderOutput"""
+
+    type: Literal["vae_loader"] = "vae_loader"
+
+    vae_model: VAEModelField = Field(description="The VAE to load")
+
+    # Schema customisation
+    class Config(InvocationConfig):
+        schema_extra = {
+            "ui": {
+                "title": "VAE Loader",
+                "tags": ["vae", "loader"],
+                "type_hints": {"vae_model": "vae_model"},
+            },
+        }
+
+    def invoke(self, context: InvocationContext) -> VaeLoaderOutput:
+        base_model = self.vae_model.base_model
+        model_name = self.vae_model.model_name
+        model_type = ModelType.Vae
+
+        if not context.services.model_manager.model_exists(
+            base_model=base_model,
+            model_name=model_name,
+            model_type=model_type,
+        ):
+            raise Exception(f"Unkown vae name: {model_name}!")
+        return VaeLoaderOutput(
+            vae=VaeField(
+                vae=ModelInfo(
+                    model_name=model_name,
+                    base_model=base_model,
+                    model_type=model_type,
+                )
+            )
+        )

invokeai/app/invocations/noise.py

@@ -32,7 +32,7 @@ def get_noise(
     perlin: float = 0.0,
 ):
     """Generate noise for a given image size."""
-    noise_device_type = "cpu" if (use_cpu or device.type == "mps") else device.type
+    noise_device_type = "cpu" if use_cpu else device.type
 
     # limit noise to only the diffusion image channels, not the mask channels
     input_channels = min(latent_channels, 4)

invokeai/app/invocations/reconstruct.py

@@ -1,4 +1,4 @@
-from typing import Literal, Union
+from typing import Literal, Optional
 
 from pydantic import Field
 
@@ -15,7 +15,7 @@ class RestoreFaceInvocation(BaseInvocation):
     type:  Literal["restore_face"] = "restore_face"
 
     # Inputs
-    image: Union[ImageField, None] = Field(description="The input image")
+    image: Optional[ImageField] = Field(description="The input image")
     strength:                float = Field(default=0.75, gt=0, le=1, description="The strength of the restoration"  )
     # fmt: on
 

invokeai/app/invocations/upscale.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
-from typing import Literal, Union
+from typing import Literal, Optional
 
 from pydantic import Field
 
@@ -16,7 +16,7 @@ class UpscaleInvocation(BaseInvocation):
     type: Literal["upscale"] = "upscale"
 
     # Inputs
-    image: Union[ImageField, None] = Field(description="The input image", default=None)
+    image: Optional[ImageField] = Field(description="The input image", default=None)
     strength: float = Field(default=0.75, gt=0, le=1, description="The strength")
     level: Literal[2, 4] = Field(default=2, description="The upscale level")
     # fmt: on

invokeai/app/services/board_image_record_storage.py

@@ -1,8 +1,7 @@
 from abc import ABC, abstractmethod
 import sqlite3
 import threading
-from typing import Union, cast
-from invokeai.app.services.board_record_storage import BoardRecord
+from typing import Optional, cast
 
 from invokeai.app.services.image_record_storage import OffsetPaginatedResults
 from invokeai.app.services.models.image_record import (
@@ -44,7 +43,7 @@ class BoardImageRecordStorageBase(ABC):
     def get_board_for_image(
         self,
         image_name: str,
-    ) -> Union[str, None]:
+    ) -> Optional[str]:
         """Gets an image's board id, if it has one."""
         pass
 
@@ -215,7 +214,7 @@ class SqliteBoardImageRecordStorage(BoardImageRecordStorageBase):
     def get_board_for_image(
         self,
         image_name: str,
-    ) -> Union[str, None]:
+    ) -> Optional[str]:
         try:
             self._lock.acquire()
             self._cursor.execute(

invokeai/app/services/board_images.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from logging import Logger
-from typing import List, Union
+from typing import List, Union, Optional
 from invokeai.app.services.board_image_record_storage import BoardImageRecordStorageBase
 from invokeai.app.services.board_record_storage import (
     BoardRecord,
@@ -49,7 +49,7 @@ class BoardImagesServiceABC(ABC):
     def get_board_for_image(
         self,
         image_name: str,
-    ) -> Union[str, None]:
+    ) -> Optional[str]:
         """Gets an image's board id, if it has one."""
         pass
 
@@ -126,13 +126,13 @@ class BoardImagesService(BoardImagesServiceABC):
     def get_board_for_image(
         self,
         image_name: str,
-    ) -> Union[str, None]:
+    ) -> Optional[str]:
         board_id = self._services.board_image_records.get_board_for_image(image_name)
         return board_id
 
 
 def board_record_to_dto(
-    board_record: BoardRecord, cover_image_name: str | None, image_count: int
+    board_record: BoardRecord, cover_image_name: Optional[str], image_count: int
 ) -> BoardDTO:
     """Converts a board record to a board DTO."""
     return BoardDTO(

invokeai/app/services/config.py

@@ -168,9 +168,10 @@ from argparse import ArgumentParser
 from omegaconf import OmegaConf, DictConfig
 from pathlib import Path
 from pydantic import BaseSettings, Field, parse_obj_as
-from typing import ClassVar, Dict, List, Literal, Union, get_origin, get_type_hints, get_args
+from typing import ClassVar, Dict, List, Set, Literal, Union, get_origin, get_type_hints, get_args
 
 INIT_FILE = Path('invokeai.yaml')
+MODEL_CORE = Path('models/core')
 DB_FILE   = Path('invokeai.db')
 LEGACY_INIT_FILE = Path('invokeai.init')
 
@@ -228,10 +229,10 @@ class InvokeAISettings(BaseSettings):
         upcase_environ = dict()
         for key,value in os.environ.items():
             upcase_environ[key.upper()] = value
-        
+
         fields = cls.__fields__
         cls.argparse_groups = {}
-        
+
         for name, field in fields.items():
             if name not in cls._excluded():
                 current_default = field.default
@@ -268,8 +269,8 @@ class InvokeAISettings(BaseSettings):
         parser.add_parser(cls.cmd_name(), help=cls.__doc__)
 
     @classmethod
-    def _excluded(self)->List[str]:
-        return ['type','initconf']
+    def _excluded(self)->Set[str]:
+        return {'type','initconf','version'}
 
     class Config:
         env_file_encoding = 'utf-8'
@@ -324,16 +325,11 @@ class InvokeAISettings(BaseSettings):
                 help=field.field_info.description,
             )
 def _find_root()->Path:
+    venv = Path(os.environ.get("VIRTUAL_ENV") or ".")
     if os.environ.get("INVOKEAI_ROOT"):
         root = Path(os.environ.get("INVOKEAI_ROOT")).resolve()
-    elif (
-            os.environ.get("VIRTUAL_ENV")
-            and (Path(os.environ.get("VIRTUAL_ENV"), "..", INIT_FILE).exists()
-                 or
-                 Path(os.environ.get("VIRTUAL_ENV"), "..", LEGACY_INIT_FILE).exists()
-                 )
-    ):
-        root = Path(os.environ.get("VIRTUAL_ENV"), "..").resolve()
+    elif any([(venv.parent/x).exists() for x in [INIT_FILE, LEGACY_INIT_FILE, MODEL_CORE]]):
+        root = (venv.parent).resolve()
     else:
         root = Path("~/invokeai").expanduser().resolve()
     return root
@@ -348,7 +344,7 @@ setting environment variables INVOKEAI_<setting>.
      '''
     singleton_config: ClassVar[InvokeAIAppConfig] = None
     singleton_init: ClassVar[Dict] = None
-    
+
     #fmt: off
     type: Literal["InvokeAI"] = "InvokeAI"
     host                : str = Field(default="127.0.0.1", description="IP address to bind to", category='Web Server')
@@ -367,7 +363,8 @@ setting environment variables INVOKEAI_<setting>.
 
     always_use_cpu      : bool = Field(default=False, description="If true, use the CPU for rendering even if a GPU is available.", category='Memory/Performance')
     free_gpu_mem        : bool = Field(default=False, description="If true, purge model from GPU after each generation.", category='Memory/Performance')
-    max_loaded_models   : int = Field(default=3, gt=0, description="Maximum number of models to keep in memory for rapid switching", category='Memory/Performance')
+    max_loaded_models   : int = Field(default=3, gt=0, description="(DEPRECATED: use max_cache_size) Maximum number of models to keep in memory for rapid switching", category='Memory/Performance')
+    max_cache_size      : float = Field(default=6.0, gt=0, description="Maximum memory amount used by model cache for rapid switching", category='Memory/Performance')
     precision           : Literal[tuple(['auto','float16','float32','autocast'])] = Field(default='float16',description='Floating point precision', category='Memory/Performance')
     sequential_guidance : bool = Field(default=False, description="Whether to calculate guidance in serial instead of in parallel, lowering memory requirements", category='Memory/Performance')
     xformers_enabled    : bool = Field(default=True, description="Enable/disable memory-efficient attention", category='Memory/Performance')
@@ -385,18 +382,20 @@ setting environment variables INVOKEAI_<setting>.
     outdir              : Path = Field(default='outputs', description='Default folder for output images', category='Paths')
     from_file           : Path = Field(default=None, description='Take command input from the indicated file (command-line client only)', category='Paths')
     use_memory_db       : bool = Field(default=False, description='Use in-memory database for storing image metadata', category='Paths')
-    
+
     model               : str = Field(default='stable-diffusion-1.5', description='Initial model name', category='Models')
-    
+
     log_handlers        : List[str] = Field(default=["console"], description='Log handler. Valid options are "console", "file=<path>", "syslog=path|address:host:port", "http=<url>"', category="Logging")
     # note - would be better to read the log_format values from logging.py, but this creates circular dependencies issues
     log_format          : Literal[tuple(['plain','color','syslog','legacy'])] = Field(default="color", description='Log format. Use "plain" for text-only, "color" for colorized output, "legacy" for 2.3-style logging and "syslog" for syslog-style', category="Logging")
     log_level           : Literal[tuple(["debug","info","warning","error","critical"])] = Field(default="debug", description="Emit logging messages at this level or  higher", category="Logging")
+
+    version             : bool = Field(default=False, description="Show InvokeAI version and exit", category="Other")
     #fmt: on
 
     def parse_args(self, argv: List[str]=None, conf: DictConfig = None, clobber=False):
         '''
-        Update settings with contents of init file, environment, and 
+        Update settings with contents of init file, environment, and
         command-line settings.
         :param conf: alternate Omegaconf dictionary object
         :param argv: aternate sys.argv list
@@ -411,7 +410,7 @@ setting environment variables INVOKEAI_<setting>.
             except:
                 pass
         InvokeAISettings.initconf = conf
-        
+
         # parse args again in order to pick up settings in configuration file
         super().parse_args(argv)
 
@@ -431,7 +430,7 @@ setting environment variables INVOKEAI_<setting>.
             cls.singleton_config = cls(**kwargs)
             cls.singleton_init = kwargs
         return cls.singleton_config
-        
+
     @property
     def root_path(self)->Path:
         '''

invokeai/app/services/events.py

@@ -1,10 +1,9 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
-from typing import Any
+from typing import Any, Optional
 from invokeai.app.models.image import ProgressImage
 from invokeai.app.util.misc import get_timestamp
 from invokeai.app.services.model_manager_service import BaseModelType, ModelType, SubModelType, ModelInfo
-from invokeai.app.models.exceptions import CanceledException
 
 class EventServiceBase:
     session_event: str = "session_event"
@@ -28,7 +27,7 @@ class EventServiceBase:
         graph_execution_state_id: str,
         node: dict,
         source_node_id: str,
-        progress_image: ProgressImage | None,
+        progress_image: Optional[ProgressImage],
         step: int,
         total_steps: int,
     ) -> None:

invokeai/app/services/graph.py

@@ -3,7 +3,6 @@
 import copy
 import itertools
 import uuid
-from types import NoneType
 from typing import (
     Annotated,
     Any,
@@ -26,6 +25,8 @@ from ..invocations.baseinvocation import (
     InvocationContext,
 )
 
+# in 3.10 this would be "from types import NoneType"
+NoneType = type(None)
 
 class EdgeConnection(BaseModel):
     node_id: str = Field(description="The id of the node for this edge connection")
@@ -60,8 +61,6 @@ def get_input_field(node: BaseInvocation, field: str) -> Any:
     node_input_field = node_inputs.get(field) or None
     return node_input_field
 
-from typing import Optional, Union, List, get_args
-
 def is_union_subtype(t1, t2):
     t1_args = get_args(t1)
     t2_args = get_args(t2)
@@ -846,7 +845,7 @@ class GraphExecutionState(BaseModel):
             ]
         }
 
-    def next(self) -> BaseInvocation | None:
+    def next(self) -> Optional[BaseInvocation]:
         """Gets the next node ready to execute."""
 
         # TODO: enable multiple nodes to execute simultaneously by tracking currently executing nodes

invokeai/app/services/image_file_storage.py

@@ -2,13 +2,12 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 from queue import Queue
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from PIL.Image import Image as PILImageType
 from PIL import Image, PngImagePlugin
 from send2trash import send2trash
 
-from invokeai.app.models.image import ResourceOrigin
 from invokeai.app.models.metadata import ImageMetadata
 from invokeai.app.util.thumbnails import get_thumbnail_name, make_thumbnail
 
@@ -80,7 +79,7 @@ class DiskImageFileStorage(ImageFileStorageBase):
     __cache: Dict[Path, PILImageType]
     __max_cache_size: int
 
-    def __init__(self, output_folder: str | Path):
+    def __init__(self, output_folder: Union[str, Path]):
         self.__cache = dict()
         self.__cache_ids = Queue()
         self.__max_cache_size = 10  # TODO: get this from config
@@ -164,7 +163,7 @@ class DiskImageFileStorage(ImageFileStorageBase):
 
         return path
 
-    def validate_path(self, path: str | Path) -> bool:
+    def validate_path(self, path: Union[str, Path]) -> bool:
         """Validates the path given for an image or thumbnail."""
         path = path if isinstance(path, Path) else Path(path)
         return path.exists()
@@ -175,7 +174,7 @@ class DiskImageFileStorage(ImageFileStorageBase):
         for folder in folders:
             folder.mkdir(parents=True, exist_ok=True)
 
-    def __get_cache(self, image_name: Path) -> PILImageType | None:
+    def __get_cache(self, image_name: Path) -> Optional[PILImageType]:
         return None if image_name not in self.__cache else self.__cache[image_name]
 
     def __set_cache(self, image_name: Path, image: PILImageType):

invokeai/app/services/image_record_storage.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from typing import Generic, Optional, TypeVar, cast
 import sqlite3
 import threading
-from typing import Optional, Union
 
 from pydantic import BaseModel, Field
 from pydantic.generics import GenericModel
@@ -116,7 +115,7 @@ class ImageRecordStorageBase(ABC):
         pass
 
     @abstractmethod
-    def get_most_recent_image_for_board(self, board_id: str) -> ImageRecord | None:
+    def get_most_recent_image_for_board(self, board_id: str) -> Optional[ImageRecord]:
         """Gets the most recent image for a board."""
         pass
 
@@ -208,7 +207,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
             """
         )
 
-    def get(self, image_name: str) -> Union[ImageRecord, None]:
+    def get(self, image_name: str) -> Optional[ImageRecord]:
         try:
             self._lock.acquire()
 
@@ -220,7 +219,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                 (image_name,),
             )
 
-            result = cast(Union[sqlite3.Row, None], self._cursor.fetchone())
+            result = cast(Optional[sqlite3.Row], self._cursor.fetchone())
         except sqlite3.Error as e:
             self._conn.rollback()
             raise ImageRecordNotFoundException from e
@@ -475,7 +474,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
 
     def get_most_recent_image_for_board(
         self, board_id: str
-    ) -> Union[ImageRecord, None]:
+    ) -> Optional[ImageRecord]:
         try:
             self._lock.acquire()
             self._cursor.execute(
@@ -490,7 +489,7 @@ class SqliteImageRecordStorage(ImageRecordStorageBase):
                 (board_id,),
             )
 
-            result = cast(Union[sqlite3.Row, None], self._cursor.fetchone())
+            result = cast(Optional[sqlite3.Row], self._cursor.fetchone())
         finally:
             self._lock.release()
         if result is None:

invokeai/app/services/images.py

@@ -370,7 +370,7 @@ class ImageService(ImageServiceABC):
 
     def _get_metadata(
         self, session_id: Optional[str] = None, node_id: Optional[str] = None
-    ) -> Union[ImageMetadata, None]:
+    ) -> Optional[ImageMetadata]:
         """Get the metadata for a node."""
         metadata = None
 

invokeai/app/services/invocation_queue.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from queue import Queue
 
 from pydantic import BaseModel, Field
-
+from typing import Optional
 
 class InvocationQueueItem(BaseModel):
     graph_execution_state_id: str = Field(description="The ID of the graph execution state")
@@ -22,7 +22,7 @@ class InvocationQueueABC(ABC):
         pass
 
     @abstractmethod
-    def put(self, item: InvocationQueueItem | None) -> None:
+    def put(self, item: Optional[InvocationQueueItem]) -> None:
         pass
 
     @abstractmethod
@@ -57,7 +57,7 @@ class MemoryInvocationQueue(InvocationQueueABC):
 
         return item
 
-    def put(self, item: InvocationQueueItem | None) -> None:
+    def put(self, item: Optional[InvocationQueueItem]) -> None:
         self.__queue.put(item)
 
     def cancel(self, graph_execution_state_id: str) -> None:

invokeai/app/services/invocation_services.py

@@ -7,7 +7,7 @@ if TYPE_CHECKING:
     from invokeai.app.services.board_images import BoardImagesServiceABC
     from invokeai.app.services.boards import BoardServiceABC
     from invokeai.app.services.images import ImageServiceABC
-    from invokeai.backend import ModelManager
+    from invokeai.app.services.model_manager_service import ModelManagerServiceBase
     from invokeai.app.services.events import EventServiceBase
     from invokeai.app.services.latent_storage import LatentsStorageBase
     from invokeai.app.services.restoration_services import RestorationServices
@@ -22,46 +22,47 @@ class InvocationServices:
     """Services that can be used by invocations"""
 
     # TODO: Just forward-declared everything due to circular dependencies. Fix structure.
-    events: "EventServiceBase"
-    latents: "LatentsStorageBase"
-    queue: "InvocationQueueABC"
-    model_manager: "ModelManager"
-    restoration: "RestorationServices"
-    configuration: "InvokeAISettings"
-    images: "ImageServiceABC"
-    boards: "BoardServiceABC"
     board_images: "BoardImagesServiceABC"
-    graph_library: "ItemStorageABC"["LibraryGraph"]
+    boards: "BoardServiceABC"
+    configuration: "InvokeAISettings"
+    events: "EventServiceBase"
     graph_execution_manager: "ItemStorageABC"["GraphExecutionState"]
+    graph_library: "ItemStorageABC"["LibraryGraph"]
+    images: "ImageServiceABC"
+    latents: "LatentsStorageBase"
+    logger: "Logger"
+    model_manager: "ModelManagerServiceBase"
     processor: "InvocationProcessorABC"
+    queue: "InvocationQueueABC"
+    restoration: "RestorationServices"
 
     def __init__(
         self,
-        model_manager: "ModelManager",
-        events: "EventServiceBase",
-        logger: "Logger",
-        latents: "LatentsStorageBase",
-        images: "ImageServiceABC",
-        boards: "BoardServiceABC",
         board_images: "BoardImagesServiceABC",
-        queue: "InvocationQueueABC",
-        graph_library: "ItemStorageABC"["LibraryGraph"],
-        graph_execution_manager: "ItemStorageABC"["GraphExecutionState"],
-        processor: "InvocationProcessorABC",
-        restoration: "RestorationServices",
+        boards: "BoardServiceABC",
         configuration: "InvokeAISettings",
+        events: "EventServiceBase",
+        graph_execution_manager: "ItemStorageABC"["GraphExecutionState"],
+        graph_library: "ItemStorageABC"["LibraryGraph"],
+        images: "ImageServiceABC",
+        latents: "LatentsStorageBase",
+        logger: "Logger",
+        model_manager: "ModelManagerServiceBase",
+        processor: "InvocationProcessorABC",
+        queue: "InvocationQueueABC",
+        restoration: "RestorationServices",
     ):
-        self.model_manager = model_manager
-        self.events = events
-        self.logger = logger
-        self.latents = latents
-        self.images = images
-        self.boards = boards
         self.board_images = board_images
-        self.queue = queue
-        self.graph_library = graph_library
-        self.graph_execution_manager = graph_execution_manager
-        self.processor = processor
-        self.restoration = restoration
-        self.configuration = configuration
         self.boards = boards
+        self.boards = boards
+        self.configuration = configuration
+        self.events = events
+        self.graph_execution_manager = graph_execution_manager
+        self.graph_library = graph_library
+        self.images = images
+        self.latents = latents
+        self.logger = logger
+        self.model_manager = model_manager
+        self.processor = processor
+        self.queue = queue
+        self.restoration = restoration

invokeai/app/services/invoker.py

@@ -1,14 +1,11 @@
 # Copyright (c) 2022 Kyle Schouviller (https://github.com/kyle0654)
 
 from abc import ABC
-from threading import Event, Thread
+from typing import Optional
 
-from ..invocations.baseinvocation import InvocationContext
 from .graph import Graph, GraphExecutionState
-from .invocation_queue import InvocationQueueABC, InvocationQueueItem
+from .invocation_queue import InvocationQueueItem
 from .invocation_services import InvocationServices
-from .item_storage import ItemStorageABC
-
 
 class Invoker:
     """The invoker, used to execute invocations"""
@@ -21,7 +18,7 @@ class Invoker:
 
     def invoke(
         self, graph_execution_state: GraphExecutionState, invoke_all: bool = False
-    ) -> str | None:
+    ) -> Optional[str]:
         """Determines the next node to invoke and enqueues it, preparing if needed.
         Returns the id of the queued node, or `None` if there are no nodes left to enqueue."""
 
@@ -45,7 +42,7 @@ class Invoker:
 
         return invocation.id
 
-    def create_execution_state(self, graph: Graph | None = None) -> GraphExecutionState:
+    def create_execution_state(self, graph: Optional[Graph] = None) -> GraphExecutionState:
         """Creates a new execution state for the given graph"""
         new_state = GraphExecutionState(graph=Graph() if graph is None else graph)
         self.services.graph_execution_manager.set(new_state)

invokeai/app/services/latent_storage.py

@@ -3,7 +3,7 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 from queue import Queue
-from typing import Dict
+from typing import Dict, Union, Optional
 
 import torch
 
@@ -55,7 +55,7 @@ class ForwardCacheLatentsStorage(LatentsStorageBase):
         if name in self.__cache:
             del self.__cache[name]
 
-    def __get_cache(self, name: str) -> torch.Tensor|None:
+    def __get_cache(self, name: str) -> Optional[torch.Tensor]:
         return None if name not in self.__cache else self.__cache[name]
 
     def __set_cache(self, name: str, data: torch.Tensor):
@@ -69,9 +69,9 @@ class ForwardCacheLatentsStorage(LatentsStorageBase):
 class DiskLatentsStorage(LatentsStorageBase):
     """Stores latents in a folder on disk without caching"""
 
-    __output_folder: str | Path
+    __output_folder: Union[str, Path]
 
-    def __init__(self, output_folder: str | Path):
+    def __init__(self, output_folder: Union[str, Path]):
         self.__output_folder = output_folder if isinstance(output_folder, Path) else Path(output_folder)
         self.__output_folder.mkdir(parents=True, exist_ok=True)
 
@@ -91,4 +91,4 @@ class DiskLatentsStorage(LatentsStorageBase):
 
     def get_path(self, name: str) -> Path:
         return self.__output_folder / name
-    
+    

invokeai/app/services/metadata.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Union
+from typing import Any, Optional
 import networkx as nx
 
 from invokeai.app.models.metadata import ImageMetadata
@@ -34,7 +34,7 @@ class CoreMetadataService(MetadataServiceBase):
 
         return metadata
 
-    def _find_nearest_ancestor(self, G: nx.DiGraph, node_id: str) -> Union[str, None]:
+    def _find_nearest_ancestor(self, G: nx.DiGraph, node_id: str) -> Optional[str]:
         """
         Finds the id of the nearest ancestor (of a valid type) of a given node.
 
@@ -65,7 +65,7 @@ class CoreMetadataService(MetadataServiceBase):
 
     def _get_additional_metadata(
         self, graph: Graph, node_id: str
-    ) -> Union[dict[str, Any], None]:
+    ) -> Optional[dict[str, Any]]:
         """
         Returns additional metadata for a given node.
 

invokeai/app/services/model_manager_service.py

@@ -2,22 +2,29 @@
 
 from __future__ import annotations
 
-import torch
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Optional, Union, Callable, List, Tuple, types, TYPE_CHECKING
-from dataclasses import dataclass
+from pydantic import Field
+from typing import Optional, Union, Callable, List, Tuple, TYPE_CHECKING
+from types import ModuleType
 
-from invokeai.backend.model_management.model_manager import (
+from invokeai.backend.model_management import (
     ModelManager,
     BaseModelType,
     ModelType,
     SubModelType,
     ModelInfo,
+    AddModelResult,
+    SchedulerPredictionType,
+    ModelMerger,
+    MergeInterpolationMethod,
 )
+
+
+import torch
 from invokeai.app.models.exceptions import CanceledException
-from .config import InvokeAIAppConfig
 from ...backend.util import choose_precision, choose_torch_device
+from .config import InvokeAIAppConfig
 
 if TYPE_CHECKING:
     from ..invocations.baseinvocation import BaseInvocation, InvocationContext
@@ -30,16 +37,16 @@ class ModelManagerServiceBase(ABC):
     def __init__(
         self,
         config: InvokeAIAppConfig,
-        logger: types.ModuleType,
+        logger: ModuleType,
     ):
         """
-        Initialize with the path to the models.yaml config file. 
+        Initialize with the path to the models.yaml config file.
         Optional parameters are the torch device type, precision, max_models,
         and sequential_offload boolean. Note that the default device
         type and precision are set up for a CUDA system running at half precision.
         """
         pass
-    
+
     @abstractmethod
     def get_model(
         self,
@@ -50,8 +57,8 @@ class ModelManagerServiceBase(ABC):
         node: Optional[BaseInvocation] = None,
         context: Optional[InvocationContext] = None,
     ) -> ModelInfo:
-        """Retrieve the indicated model with name and type. 
-        submodel can be used to get a part (such as the vae) 
+        """Retrieve the indicated model with name and type.
+        submodel can be used to get a part (such as the vae)
         of a diffusers pipeline."""
         pass
 
@@ -73,13 +80,7 @@ class ModelManagerServiceBase(ABC):
     def model_info(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> dict:
         """
         Given a model name returns a dict-like (OmegaConf) object describing it.
-        """
-        pass
-
-    @abstractmethod
-    def model_names(self) -> List[Tuple[str, BaseModelType, ModelType]]:
-        """
-        Returns a list of all the model names known.
+        Uses the exact format as the omegaconf stanza.
         """
         pass
 
@@ -101,7 +102,20 @@ class ModelManagerServiceBase(ABC):
         }
         """
         pass
+    
+    @abstractmethod
+    def list_model(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> dict:
+        """
+        Return information about the model using the same format as list_models()
+        """
+        pass
 
+    @abstractmethod
+    def model_names(self) -> List[Tuple[str, BaseModelType, ModelType]]:
+        """
+        Returns a list of all the model names known.
+        """
+        pass
 
     @abstractmethod
     def add_model(
@@ -111,16 +125,34 @@ class ModelManagerServiceBase(ABC):
         model_type: ModelType,
         model_attributes: dict,
         clobber: bool = False
-    ) -> None:
+    ) -> AddModelResult:
         """
         Update the named model with a dictionary of attributes. Will fail with an
         assertion error if the name already exists. Pass clobber=True to overwrite.
+        On a successful update, the config will be changed in memory. Will fail
+        with an assertion error if provided attributes are incorrect or
+        the model name is missing. Call commit() to write changes to disk.
+        """
+        pass
+
+    @abstractmethod
+    def update_model(
+        self,
+        model_name: str,
+        base_model: BaseModelType,
+        model_type: ModelType,
+        model_attributes: dict,
+    ) -> AddModelResult:
+        """
+        Update the named model with a dictionary of attributes. Will fail with a
+        KeyErrorException if the name does not already exist.
+
         On a successful update, the config will be changed in memory. Will fail 
         with an assertion error if provided attributes are incorrect or 
         the model name is missing. Call commit() to write changes to disk.
         """
         pass
-
+    
     @abstractmethod
     def del_model(
         self,
@@ -129,14 +161,78 @@ class ModelManagerServiceBase(ABC):
         model_type: ModelType,
     ):
         """
-        Delete the named model from configuration. If delete_files is true, 
-        then the underlying weight file or diffusers directory will be deleted 
+        Delete the named model from configuration. If delete_files is true,
+        then the underlying weight file or diffusers directory will be deleted
         as well. Call commit() to write to disk.
         """
         pass
 
     @abstractmethod
-    def commit(self, conf_file: Path = None) -> None:
+    def convert_model(
+        self,
+        model_name: str,
+        base_model: BaseModelType,
+        model_type: Union[ModelType.Main,ModelType.Vae],
+    ) -> AddModelResult:
+        """
+        Convert a checkpoint file into a diffusers folder, deleting the cached
+        version and deleting the original checkpoint file if it is in the models
+        directory.
+        :param model_name: Name of the model to convert
+        :param base_model: Base model type
+        :param model_type: Type of model ['vae' or 'main']
+
+        This will raise a ValueError unless the model is not a checkpoint. It will
+        also raise a ValueError in the event that there is a similarly-named diffusers
+        directory already in place.
+        """
+        pass
+
+    @abstractmethod
+    def heuristic_import(self,
+                         items_to_import: set[str],
+                         prediction_type_helper: Optional[Callable[[Path],SchedulerPredictionType]]=None,
+                         )->dict[str, AddModelResult]:
+        '''Import a list of paths, repo_ids or URLs. Returns the set of
+        successfully imported items.
+        :param items_to_import: Set of strings corresponding to models to be imported.
+        :param prediction_type_helper: A callback that receives the Path of a Stable Diffusion 2 checkpoint model and returns a SchedulerPredictionType.
+
+        The prediction type helper is necessary to distinguish between
+        models based on Stable Diffusion 2 Base (requiring
+        SchedulerPredictionType.Epsilson) and Stable Diffusion 768
+        (requiring SchedulerPredictionType.VPrediction). It is
+        generally impossible to do this programmatically, so the
+        prediction_type_helper usually asks the user to choose.
+
+        The result is a set of successfully installed models. Each element
+        of the set is a dict corresponding to the newly-created OmegaConf stanza for
+        that model.
+        '''
+        pass
+
+    @abstractmethod
+    def merge_models(
+            self,
+            model_names: List[str] = Field(default=None, min_items=2, max_items=3, description="List of model names to merge"),
+            base_model: Union[BaseModelType,str] = Field(default=None, description="Base model shared by all models to be merged"),
+            merged_model_name: str = Field(default=None, description="Name of destination model after merging"),
+            alpha: Optional[float] = 0.5,
+            interp: Optional[MergeInterpolationMethod] = None,
+            force: Optional[bool] = False,
+    ) -> AddModelResult:
+        """
+        Merge two to three diffusrs pipeline models and save as a new model.
+        :param model_names: List of 2-3 models to merge
+        :param base_model: Base model to use for all models
+        :param merged_model_name: Name of destination merged model
+        :param alpha: Alpha strength to apply to 2d and 3d model
+        :param interp: Interpolation method. None (default) 
+        """
+        pass
+    
+    @abstractmethod
+    def commit(self, conf_file: Optional[Path] = None) -> None:
         """
         Write current configuration out to the indicated file.
         If no conf_file is provided, then replaces the
@@ -150,10 +246,10 @@ class ModelManagerService(ModelManagerServiceBase):
     def __init__(
         self,
         config: InvokeAIAppConfig,
-        logger: types.ModuleType,
+        logger: ModuleType,
     ):
         """
-        Initialize with the path to the models.yaml config file. 
+        Initialize with the path to the models.yaml config file.
         Optional parameters are the torch device type, precision, max_models,
         and sequential_offload boolean. Note that the default device
         type and precision are set up for a CUDA system running at half precision.
@@ -162,12 +258,12 @@ class ModelManagerService(ModelManagerServiceBase):
             config_file = config.model_conf_path
         else:
             config_file = config.root_dir / "configs/models.yaml"
-        if not config_file.exists():
-            raise IOError(f"The file {config_file} could not be found.")
-
+            
         logger.debug(f'config file={config_file}')
 
         device = torch.device(choose_torch_device())
+        logger.debug(f'GPU device = {device}')
+
         precision = config.precision
         if precision == "auto":
             precision = choose_precision(device)
@@ -183,6 +279,8 @@ class ModelManagerService(ModelManagerServiceBase):
             if hasattr(config,'max_cache_size') \
                else config.max_loaded_models * 2.5
 
+        logger.debug(f"Maximum RAM cache size: {max_cache_size} GiB")
+
         sequential_offload = config.sequential_guidance
 
         self.mgr = ModelManager(
@@ -238,7 +336,7 @@ class ModelManagerService(ModelManagerServiceBase):
                 submodel=submodel,
                 model_info=model_info
             )
-            
+
         return model_info
 
     def model_exists(
@@ -274,12 +372,19 @@ class ModelManagerService(ModelManagerServiceBase):
         base_model: Optional[BaseModelType] = None,
         model_type: Optional[ModelType] = None
     ) -> list[dict]:
-    # ) -> dict:
         """
         Return a list of models.
         """
         return self.mgr.list_models(base_model, model_type)
 
+    def list_model(self, model_name: str, base_model: BaseModelType, model_type: ModelType) -> dict:
+        """
+        Return information about the model using the same format as list_models()
+        """
+        return self.mgr.list_model(model_name=model_name,
+                                   base_model=base_model,
+                                   model_type=model_type)
+
     def add_model(
         self,
         model_name: str,
@@ -291,13 +396,32 @@ class ModelManagerService(ModelManagerServiceBase):
         """
         Update the named model with a dictionary of attributes. Will fail with an
         assertion error if the name already exists. Pass clobber=True to overwrite.
+        On a successful update, the config will be changed in memory. Will fail
+        with an assertion error if provided attributes are incorrect or
+        the model name is missing. Call commit() to write changes to disk.
+        """
+        self.logger.debug(f'add/update model {model_name}')        
+        return self.mgr.add_model(model_name, base_model, model_type, model_attributes, clobber)
+
+    def update_model(
+        self,
+        model_name: str,
+        base_model: BaseModelType,
+        model_type: ModelType,
+        model_attributes: dict,
+    ) -> AddModelResult:
+        """
+        Update the named model with a dictionary of attributes. Will fail with a
+        KeyError exception if the name does not already exist.
         On a successful update, the config will be changed in memory. Will fail 
         with an assertion error if provided attributes are incorrect or 
         the model name is missing. Call commit() to write changes to disk.
         """
-        return self.mgr.add_model(model_name, base_model, model_type, model_attributes, clobber)
-
-
+        self.logger.debug(f'update model {model_name}')
+        if not self.model_exists(model_name, base_model, model_type):
+            raise KeyError(f"Unknown model {model_name}")
+        return self.add_model(model_name, base_model, model_type, model_attributes, clobber=True)
+    
     def del_model(
         self,
         model_name: str,
@@ -305,12 +429,33 @@ class ModelManagerService(ModelManagerServiceBase):
         model_type: ModelType,
     ):
         """
-        Delete the named model from configuration. If delete_files is true, 
-        then the underlying weight file or diffusers directory will be deleted 
+        Delete the named model from configuration. If delete_files is true,
+        then the underlying weight file or diffusers directory will be deleted
         as well. Call commit() to write to disk.
         """
+        self.logger.debug(f'delete model {model_name}')
         self.mgr.del_model(model_name, base_model, model_type)
 
+    def convert_model(
+        self,
+        model_name: str,
+        base_model: BaseModelType,
+        model_type: Union[ModelType.Main,ModelType.Vae],
+    ) -> AddModelResult:
+        """
+        Convert a checkpoint file into a diffusers folder, deleting the cached
+        version and deleting the original checkpoint file if it is in the models
+        directory.
+        :param model_name: Name of the model to convert
+        :param base_model: Base model type
+        :param model_type: Type of model ['vae' or 'main']
+
+        This will raise a ValueError unless the model is not a checkpoint. It will
+        also raise a ValueError in the event that there is a similarly-named diffusers
+        directory already in place.
+        """
+        self.logger.debug(f'convert model {model_name}')        
+        return self.mgr.convert_model(model_name, base_model, model_type)
 
     def commit(self, conf_file: Optional[Path]=None):
         """
@@ -360,4 +505,56 @@ class ModelManagerService(ModelManagerServiceBase):
     @property
     def logger(self):
         return self.mgr.logger
-        
+
+    def heuristic_import(self,
+                         items_to_import: set[str],
+                         prediction_type_helper: Optional[Callable[[Path],SchedulerPredictionType]]=None,
+                         )->dict[str, AddModelResult]:
+        '''Import a list of paths, repo_ids or URLs. Returns the set of
+        successfully imported items.
+        :param items_to_import: Set of strings corresponding to models to be imported.
+        :param prediction_type_helper: A callback that receives the Path of a Stable Diffusion 2 checkpoint model and returns a SchedulerPredictionType.
+
+        The prediction type helper is necessary to distinguish between
+        models based on Stable Diffusion 2 Base (requiring
+        SchedulerPredictionType.Epsilson) and Stable Diffusion 768
+        (requiring SchedulerPredictionType.VPrediction). It is
+        generally impossible to do this programmatically, so the
+        prediction_type_helper usually asks the user to choose.
+
+        The result is a set of successfully installed models. Each element
+        of the set is a dict corresponding to the newly-created OmegaConf stanza for
+        that model.
+        '''
+        return self.mgr.heuristic_import(items_to_import, prediction_type_helper)        
+
+    def merge_models(
+            self,
+            model_names: List[str] = Field(default=None, min_items=2, max_items=3, description="List of model names to merge"),
+            base_model: Union[BaseModelType,str] = Field(default=None, description="Base model shared by all models to be merged"),
+            merged_model_name: str = Field(default=None, description="Name of destination model after merging"),
+            alpha: Optional[float] = 0.5,
+            interp: Optional[MergeInterpolationMethod] = None,
+            force: Optional[bool] = False,
+    ) -> AddModelResult:
+        """
+        Merge two to three diffusrs pipeline models and save as a new model.
+        :param model_names: List of 2-3 models to merge
+        :param base_model: Base model to use for all models
+        :param merged_model_name: Name of destination merged model
+        :param alpha: Alpha strength to apply to 2d and 3d model
+        :param interp: Interpolation method. None (default) 
+        """
+        merger = ModelMerger(self.mgr)
+        try:
+            result = merger.merge_diffusion_models_and_save(
+                model_names = model_names,
+                base_model = base_model,
+                merged_model_name = merged_model_name,
+                alpha = alpha,
+                interp = interp,
+                force = force,
+            )
+        except AssertionError as e:
+            raise ValueError(e)
+        return result

invokeai/app/services/models/image_record.py

@@ -88,7 +88,7 @@ class ImageUrlsDTO(BaseModel):
 class ImageDTO(ImageRecord, ImageUrlsDTO):
     """Deserialized image record, enriched for the frontend."""
 
-    board_id: Union[str, None] = Field(
+    board_id: Optional[str] = Field(
         description="The id of the board the image belongs to, if one exists."
     )
     """The id of the board the image belongs to, if one exists."""
@@ -96,7 +96,7 @@ class ImageDTO(ImageRecord, ImageUrlsDTO):
 
 
 def image_record_to_dto(
-    image_record: ImageRecord, image_url: str, thumbnail_url: str, board_id: Union[str, None]
+    image_record: ImageRecord, image_url: str, thumbnail_url: str, board_id: Optional[str]
 ) -> ImageDTO:
     """Converts an image record to an image DTO."""
     return ImageDTO(

invokeai/app/services/processor.py

@@ -104,6 +104,7 @@ class DefaultInvocationProcessor(InvocationProcessorABC):
 
                 except Exception as e:
                     error = traceback.format_exc()
+                    logger.error(error)
 
                     # Save error
                     graph_execution_state.set_node_error(invocation.id, error)

invokeai/app/services/sqlite.py

@@ -1,6 +1,6 @@
 import sqlite3
 from threading import Lock
-from typing import Generic, TypeVar, Union, get_args
+from typing import Generic, TypeVar, Optional, Union, get_args
 
 from pydantic import BaseModel, parse_raw_as
 
@@ -63,7 +63,7 @@ class SqliteItemStorage(ItemStorageABC, Generic[T]):
             self._lock.release()
         self._on_changed(item)
 
-    def get(self, id: str) -> Union[T, None]:
+    def get(self, id: str) -> Optional[T]:
         try:
             self._lock.acquire()
             self._cursor.execute(

invokeai/backend/generator/base.py

@@ -21,7 +21,7 @@ from PIL import Image, ImageChops, ImageFilter
 from accelerate.utils import set_seed
 from diffusers import DiffusionPipeline
 from tqdm import trange
-from typing import Callable, List, Iterator, Optional, Type
+from typing import Callable, List, Iterator, Optional, Type, Union
 from dataclasses import dataclass, field
 from diffusers.schedulers import SchedulerMixin as Scheduler
 
@@ -178,7 +178,7 @@ class InvokeAIGenerator(metaclass=ABCMeta):
 # ------------------------------------
 class Img2Img(InvokeAIGenerator):
     def generate(self,
-               init_image: Image.Image | torch.FloatTensor,
+               init_image: Union[Image.Image, torch.FloatTensor],
                strength: float=0.75,
                **keyword_args
                )->Iterator[InvokeAIGeneratorOutput]:
@@ -195,7 +195,7 @@ class Img2Img(InvokeAIGenerator):
 # Takes all the arguments of Img2Img and adds the mask image and the seam/infill stuff
 class Inpaint(Img2Img):
     def generate(self,
-                 mask_image: Image.Image | torch.FloatTensor,
+                 mask_image: Union[Image.Image, torch.FloatTensor],
                  # Seam settings - when 0, doesn't fill seam
                  seam_size: int = 96,
                  seam_blur: int = 16,
@@ -570,28 +570,16 @@ class Generator:
         device = self.model.device
         # limit noise to only the diffusion image channels, not the mask channels
         input_channels = min(self.latent_channels, 4)
-        if self.use_mps_noise or device.type == "mps":
-            x = torch.randn(
-                [
-                    1,
-                    input_channels,
-                    height // self.downsampling_factor,
-                    width // self.downsampling_factor,
-                ],
-                dtype=self.torch_dtype(),
-                device="cpu",
-            ).to(device)
-        else:
-            x = torch.randn(
-                [
-                    1,
-                    input_channels,
-                    height // self.downsampling_factor,
-                    width // self.downsampling_factor,
-                ],
-                dtype=self.torch_dtype(),
-                device=device,
-            )
+        x = torch.randn(
+            [
+                1,
+                input_channels,
+                height // self.downsampling_factor,
+                width // self.downsampling_factor,
+            ],
+            dtype=self.torch_dtype(),
+            device=device,
+        )
         if self.perlin > 0.0:
             perlin_noise = self.get_perlin_noise(
                 width // self.downsampling_factor, height // self.downsampling_factor

invokeai/backend/generator/img2img.py

@@ -88,10 +88,7 @@ class Img2Img(Generator):
 
     def get_noise_like(self, like: torch.Tensor):
         device = like.device
-        if device.type == "mps":
-            x = torch.randn_like(like, device="cpu").to(device)
-        else:
-            x = torch.randn_like(like, device=device)
+        x = torch.randn_like(like, device=device)
         if self.perlin > 0.0:
             shape = like.shape
             x = (1 - self.perlin) * x + self.perlin * self.get_perlin_noise(

invokeai/backend/generator/inpaint.py

@@ -4,11 +4,10 @@ invokeai.backend.generator.inpaint descends from .generator
 from __future__ import annotations
 
 import math
-from typing import Tuple, Union
+from typing import Tuple, Union, Optional
 
 import cv2
 import numpy as np
-import PIL
 import torch
 from PIL import Image, ImageChops, ImageFilter, ImageOps
 
@@ -76,7 +75,7 @@ class Inpaint(Img2Img):
         return im_patched
 
     def tile_fill_missing(
-        self, im: Image.Image, tile_size: int = 16, seed: Union[int, None] = None
+        self, im: Image.Image, tile_size: int = 16, seed: Optional[int] = None
     ) -> Image.Image:
         # Only fill if there's an alpha layer
         if im.mode != "RGBA":
@@ -203,8 +202,8 @@ class Inpaint(Img2Img):
         cfg_scale,
         ddim_eta,
         conditioning,
-        init_image: Image.Image | torch.FloatTensor,
-        mask_image: Image.Image | torch.FloatTensor,
+        init_image: Union[Image.Image, torch.FloatTensor],
+        mask_image: Union[Image.Image, torch.FloatTensor],
         strength: float,
         mask_blur_radius: int = 8,
         # Seam settings - when 0, doesn't fill seam

invokeai/backend/install/invokeai_configure.py

@@ -45,6 +45,7 @@ from invokeai.app.services.config import (
 from invokeai.backend.util.logging import InvokeAILogger
 from invokeai.frontend.install.model_install import addModelsForm, process_and_execute
 from invokeai.frontend.install.widgets import (
+    SingleSelectColumns,
     CenteredButtonPress,
     IntTitleSlider,
     set_min_terminal_size,
@@ -76,7 +77,7 @@ Weights_dir = "ldm/stable-diffusion-v1/"
 Default_config_file = config.model_conf_path
 SD_Configs = config.legacy_conf_path
 
-PRECISION_CHOICES = ['auto','float16','float32','autocast']
+PRECISION_CHOICES = ['auto','float16','float32']
 
 INIT_FILE_PREAMBLE = """# InvokeAI initialization file
 # This is the InvokeAI initialization file, which contains command-line default values.
@@ -359,9 +360,7 @@ Use cursor arrows to make a checkbox selection, and space to toggle.
             scroll_exit=True,
         )
         self.nextrely += 1
-        label = """If you have an account at HuggingFace you may optionally paste your access token here
-to allow InvokeAI to download restricted styles & subjects from the "Concept Library". See https://huggingface.co/settings/tokens.
-"""
+        label = """HuggingFace access token (OPTIONAL) for automatic model downloads. See https://huggingface.co/settings/tokens."""
         for line in textwrap.wrap(label,width=window_width-6):
             self.add_widget_intelligent(
                 npyscreen.FixedText,
@@ -423,6 +422,7 @@ to allow InvokeAI to download restricted styles & subjects from the "Concept Lib
         )
         self.precision = self.add_widget_intelligent(
             npyscreen.TitleSelectOne,
+            columns = 2,
             name="Precision",
             values=PRECISION_CHOICES,
             value=PRECISION_CHOICES.index(precision),
@@ -430,13 +430,13 @@ to allow InvokeAI to download restricted styles & subjects from the "Concept Lib
             max_height=len(PRECISION_CHOICES) + 1,
             scroll_exit=True,
         )
-        self.max_loaded_models = self.add_widget_intelligent(
+        self.max_cache_size = self.add_widget_intelligent(
             IntTitleSlider,
-            name="Number of models to cache in CPU memory (each will use 2-4 GB!)",
-            value=old_opts.max_loaded_models,
-            out_of=10,
-            lowest=1,
-            begin_entry_at=4,
+            name="Size of the RAM cache used for fast model switching (GB)",
+            value=old_opts.max_cache_size,
+            out_of=20,
+            lowest=3,
+            begin_entry_at=6,
             scroll_exit=True,
         )
         self.nextrely += 1
@@ -539,7 +539,7 @@ https://huggingface.co/spaces/CompVis/stable-diffusion-license
                 "outdir",
                 "nsfw_checker",
                 "free_gpu_mem",
-                "max_loaded_models",
+                "max_cache_size",
                 "xformers_enabled",
                 "always_use_cpu",
         ]:
@@ -555,9 +555,6 @@ https://huggingface.co/spaces/CompVis/stable-diffusion-license
         new_opts.license_acceptance = self.license_acceptance.value
         new_opts.precision = PRECISION_CHOICES[self.precision.value[0]]
         
-        # widget library workaround to make max_loaded_models an int rather than a float
-        new_opts.max_loaded_models = int(new_opts.max_loaded_models)
-
         return new_opts
 
 

invokeai/backend/install/legacy_arg_parsing.py

@@ -4,6 +4,8 @@ import argparse
 import shlex
 from argparse import ArgumentParser
 
+# note that this includes both old sampler names and new scheduler names
+# in order to be able to parse both 2.0 and 3.0-pre-nodes versions of invokeai.init
 SAMPLER_CHOICES = [
     "ddim",
     "ddpm",
@@ -27,6 +29,15 @@ SAMPLER_CHOICES = [
     "dpmpp_sde",
     "dpmpp_sde_k",
     "unipc",
+    "k_dpm_2_a",
+    "k_dpm_2",
+    "k_dpmpp_2_a",
+    "k_dpmpp_2",
+    "k_euler_a",
+    "k_euler",
+    "k_heun",
+    "k_lms",
+    "plms",
 ]
 
 PRECISION_CHOICES = [

invokeai/backend/install/migrate_to_3.py

@@ -3,7 +3,6 @@ Migrate the models directory and models.yaml file from an existing
 InvokeAI 2.3 installation to 3.0.0.
 '''
 
-import io
 import os
 import argparse
 import shutil
@@ -28,9 +27,10 @@ from transformers import (
 )
 
 import invokeai.backend.util.logging as logger
+from invokeai.app.services.config import InvokeAIAppConfig
 from invokeai.backend.model_management import ModelManager
 from invokeai.backend.model_management.model_probe import (
-    ModelProbe, ModelType, BaseModelType, SchedulerPredictionType, ModelProbeInfo
+    ModelProbe, ModelType, BaseModelType, ModelProbeInfo
     )
 
 warnings.filterwarnings("ignore")
@@ -47,48 +47,27 @@ class ModelPaths:
 
 class MigrateTo3(object):
     def __init__(self,
-                 root_directory: Path,
-                 dest_models: Path,
-                 yaml_file: io.TextIOBase,
+                 from_root: Path,
+                 to_models: Path,
+                 model_manager: ModelManager,
                  src_paths: ModelPaths,
                  ):
-        self.root_directory = root_directory
-        self.dest_models = dest_models
-        self.dest_yaml = yaml_file
-        self.model_names = set()
+        self.root_directory = from_root
+        self.dest_models = to_models
+        self.mgr = model_manager
         self.src_paths = src_paths
         
-        self._initialize_yaml()
-
-    def _initialize_yaml(self):
-        self.dest_yaml.write(
-            yaml.dump(
-                {
-                    '__metadata__':
+    @classmethod
+    def initialize_yaml(cls, yaml_file: Path):
+        with open(yaml_file, 'w') as file:
+            file.write(
+                yaml.dump(
                     {
-                        'version':'3.0.0'}
-                }
+                        '__metadata__': {'version':'3.0.0'}
+                    }
+                )
             )
-        )
     
-    def unique_name(self,name,info)->str:
-        '''
-        Create a unique name for a model for use within models.yaml.
-        '''
-        done = False
-        key = ModelManager.create_key(name,info.base_type,info.model_type)
-        unique_name = key
-        counter = 1
-        while not done:
-            if unique_name in self.model_names:
-                unique_name = f'{key}-{counter:0>2d}'
-                counter += 1
-            else:
-                done = True
-        self.model_names.add(unique_name)
-        name,_,_ = ModelManager.parse_key(unique_name)
-        return name
-
     def create_directory_structure(self):
         '''
         Create the basic directory structure for the models folder.
@@ -136,23 +115,8 @@ class MigrateTo3(object):
         that looks like a model, and copy the model into the
         appropriate location within the destination models directory.
         '''
+        directories_scanned = set()
         for root, dirs, files in os.walk(src_dir):
-            for f in files:
-                # hack - don't copy raw learned_embeds.bin, let them
-                # be copied as part of a tree copy operation
-                if f == 'learned_embeds.bin':
-                    continue
-                try:
-                    model = Path(root,f)
-                    info = ModelProbe().heuristic_probe(model)
-                    if not info:
-                        continue
-                    dest = self._model_probe_to_path(info) / f
-                    self.copy_file(model, dest)
-                except KeyboardInterrupt:
-                    raise
-                except Exception as e:
-                    logger.error(str(e))
             for d in dirs:
                 try:
                     model = Path(root,d)
@@ -161,6 +125,29 @@ class MigrateTo3(object):
                         continue
                     dest = self._model_probe_to_path(info) / model.name
                     self.copy_dir(model, dest)
+                    directories_scanned.add(model)
+                except Exception as e:
+                    logger.error(str(e))
+                except KeyboardInterrupt:
+                    raise
+                except Exception as e:
+                    logger.error(str(e))
+            for f in files:
+                # don't copy raw learned_embeds.bin or pytorch_lora_weights.bin
+                # let them be copied as part of a tree copy operation
+                try:
+                    if f in {'learned_embeds.bin','pytorch_lora_weights.bin'}:
+                        continue
+                    model = Path(root,f)
+                    if model.parent in directories_scanned:
+                        continue
+                    info = ModelProbe().heuristic_probe(model)
+                    if not info:
+                        continue
+                    dest = self._model_probe_to_path(info) / f
+                    self.copy_file(model, dest)
+                except Exception as e:
+                    logger.error(str(e))
                 except KeyboardInterrupt:
                     raise
                 except Exception as e:
@@ -219,11 +206,12 @@ class MigrateTo3(object):
             repo_id = 'openai/clip-vit-large-patch14'
             self._migrate_pretrained(CLIPTokenizer,
                                      repo_id= repo_id,
-                                     dest= target_dir / 'clip-vit-large-patch14' / 'tokenizer',
+                                     dest= target_dir / 'clip-vit-large-patch14',
                                      **kwargs)
             self._migrate_pretrained(CLIPTextModel,
                                      repo_id = repo_id,
-                                     dest = target_dir / 'clip-vit-large-patch14' / 'text_encoder',
+                                     dest = target_dir / 'clip-vit-large-patch14',
+                                     force = True,
                                      **kwargs)
 
             # sd-2
@@ -262,46 +250,24 @@ class MigrateTo3(object):
         except Exception as e:
             logger.error(str(e))
 
-    def write_yaml(self, model_name: str, path:Path, info:ModelProbeInfo, **kwargs):
-        '''
-        Write a stanza for a moved model into the new models.yaml file.
-        '''
-        name = self.unique_name(model_name, info)
-        stanza = {
-            f'{info.base_type.value}/{info.model_type.value}/{name}': {
-                'name': model_name,
-                'path': str(path),
-                'description': f'A {info.base_type.value} {info.model_type.value} model',
-                'format': info.format,
-                'image_size': info.image_size,
-                'base': info.base_type.value,
-                'variant': info.variant_type.value,
-                'prediction_type': info.prediction_type.value,
-                'upcast_attention': info.prediction_type == SchedulerPredictionType.VPrediction,
-                **kwargs,
-            }
-        }
-        self.dest_yaml.write(yaml.dump(stanza))
-        self.dest_yaml.flush()
-        
     def _model_probe_to_path(self, info: ModelProbeInfo)->Path:
         return Path(self.dest_models, info.base_type.value, info.model_type.value)
 
-    def _migrate_pretrained(self, model_class, repo_id: str, dest: Path, **kwargs):
-        if dest.exists():
+    def _migrate_pretrained(self, model_class, repo_id: str, dest: Path, force:bool=False, **kwargs):
+        if dest.exists() and not force:
             logger.info(f'Skipping existing {dest}')
             return
         model = model_class.from_pretrained(repo_id, **kwargs)
-        self._save_pretrained(model, dest)
+        self._save_pretrained(model, dest, overwrite=force)
 
-    def _save_pretrained(self, model, dest: Path):
-        if dest.exists():
-            logger.info(f'Skipping existing {dest}')
-            return
+    def _save_pretrained(self, model, dest: Path, overwrite: bool=False):
         model_name = dest.name
-        download_path = dest.with_name(f'{model_name}.downloading')
-        model.save_pretrained(download_path, safe_serialization=True)
-        download_path.replace(dest)
+        if overwrite:
+            model.save_pretrained(dest, safe_serialization=True)
+        else:
+            download_path = dest.with_name(f'{model_name}.downloading')
+            model.save_pretrained(download_path, safe_serialization=True)
+            download_path.replace(dest)
 
     def _download_vae(self, repo_id: str, subfolder:str=None)->Path:
         vae = AutoencoderKL.from_pretrained(repo_id, cache_dir=self.root_directory / 'models/hub', subfolder=subfolder)
@@ -327,6 +293,7 @@ class MigrateTo3(object):
             elif repo_id := vae.get('repo_id'):
                 if repo_id=='stabilityai/sd-vae-ft-mse':  # this guy is already downloaded
                     vae_path = 'models/core/convert/sd-vae-ft-mse'
+                    return vae_path
                 else:
                     vae_path = self._download_vae(repo_id, vae.get('subfolder'))
 
@@ -339,7 +306,10 @@ class MigrateTo3(object):
             info = ModelProbe().heuristic_probe(vae_path)
             dest = self._model_probe_to_path(info) / vae_path.name
             if not dest.exists():
-                self.copy_dir(vae_path,dest)
+                if vae_path.is_dir():
+                    self.copy_dir(vae_path,dest)
+                else:
+                    self.copy_file(vae_path,dest)
             vae_path = dest
 
         if vae_path.is_relative_to(self.dest_models):
@@ -348,7 +318,7 @@ class MigrateTo3(object):
         else:
             return vae_path
 
-    def migrate_repo_id(self, repo_id: str, model_name :str=None, **extra_config):
+    def migrate_repo_id(self, repo_id: str, model_name: str=None, **extra_config):
         '''
         Migrate a locally-cached diffusers pipeline identified with a repo_id
         '''
@@ -380,11 +350,15 @@ class MigrateTo3(object):
         if not info:
             return
 
-        dest = self._model_probe_to_path(info) / repo_name
+        if self.mgr.model_exists(model_name, info.base_type, info.model_type):
+            logger.warning(f'A model named {model_name} already exists at the destination. Skipping migration.')
+            return
+
+        dest = self._model_probe_to_path(info) / model_name
         self._save_pretrained(pipeline, dest)
             
         rel_path = Path('models',dest.relative_to(dest_dir))
-        self.write_yaml(model_name, path=rel_path, info=info, **extra_config)
+        self._add_model(model_name, info, rel_path, **extra_config)
 
     def migrate_path(self, location: Path, model_name: str=None, **extra_config):
         '''
@@ -394,20 +368,49 @@ class MigrateTo3(object):
         # handle relative paths
         dest_dir = self.dest_models
         location = self.root_directory / location
+        model_name = model_name or location.stem
         
         info = ModelProbe().heuristic_probe(location)
         if not info:
             return
+        
+        if self.mgr.model_exists(model_name, info.base_type, info.model_type):
+            logger.warning(f'A model named {model_name} already exists at the destination. Skipping migration.')
+            return
 
         # uh oh, weights is in the old models directory - move it into the new one
         if Path(location).is_relative_to(self.src_paths.models):
             dest = Path(dest_dir, info.base_type.value, info.model_type.value, location.name)
-            self.copy_dir(location,dest)
+            if location.is_dir():
+                self.copy_dir(location,dest)
+            else:
+                self.copy_file(location,dest)
             location = Path('models', info.base_type.value, info.model_type.value, location.name)
-        model_name = model_name or location.stem
-        model_name = self.unique_name(model_name, info)
-        self.write_yaml(model_name, path=location, info=info, **extra_config)
 
+        self._add_model(model_name, info, location, **extra_config)
+
+    def _add_model(self,
+                   model_name: str,
+                   info: ModelProbeInfo,
+                   location: Path,
+                   **extra_config):
+        if info.model_type != ModelType.Main:
+            return
+        
+        self.mgr.add_model(
+            model_name = model_name,
+            base_model = info.base_type,
+            model_type = info.model_type,
+            clobber = True,
+            model_attributes = {
+                'path': str(location),
+                'description': f'A {info.base_type.value} {info.model_type.value} model',
+                'model_format': info.format,
+                'variant': info.variant_type.value,
+                **extra_config,
+            }
+        )
+        
     def migrate_defined_models(self):
         '''
         Migrate models defined in models.yaml
@@ -429,6 +432,9 @@ class MigrateTo3(object):
 
                 if config := stanza.get('config'):
                     passthru_args['config'] = config
+
+                if description:= stanza.get('description'):
+                    passthru_args['description'] = description
                 
                 if repo_id := stanza.get('repo_id'):
                     logger.info(f'Migrating diffusers model {model_name}')
@@ -509,31 +515,50 @@ def get_legacy_embeddings(root: Path) -> ModelPaths:
         return _parse_legacy_yamlfile(root, path)
 
 def do_migrate(src_directory: Path, dest_directory: Path):
+    """
+    Migrate models from src to dest InvokeAI root directories
+    """
+    config_file = dest_directory / 'configs' / 'models.yaml.3'
+    dest_models = dest_directory / 'models.3'
     
-    dest_models = dest_directory / 'models-3.0'
-    dest_yaml = dest_directory / 'configs/models.yaml-3.0'
+    version_3 = (dest_directory / 'models' / 'core').exists()
 
+    # Here we create the destination models.yaml file.
+    # If we are writing into a version 3 directory and the
+    # file already exists, then we write into a copy of it to
+    # avoid deleting its previous customizations. Otherwise we
+    # create a new empty one.
+    if version_3:  # write into the dest directory
+        try:
+            shutil.copy(dest_directory / 'configs' / 'models.yaml', config_file)
+        except:
+            MigrateTo3.initialize_yaml(config_file)
+        mgr = ModelManager(config_file) # important to initialize BEFORE moving the models directory
+        (dest_directory / 'models').replace(dest_models)
+    else:
+        MigrateTo3.initialize_yaml(config_file)
+        mgr = ModelManager(config_file)
+    
     paths = get_legacy_embeddings(src_directory)
+    migrator = MigrateTo3(
+        from_root = src_directory,
+        to_models = dest_models,
+        model_manager = mgr,
+        src_paths = paths
+    )
+    migrator.migrate()
+    print("Migration successful.")
 
-    with open(dest_yaml,'w') as yaml_file:
-        migrator = MigrateTo3(src_directory,
-                              dest_models,
-                              yaml_file,
-                              src_paths = paths,
-                              )
-        migrator.migrate()
-
-    shutil.rmtree(dest_directory / 'models.orig', ignore_errors=True)
-    (dest_directory / 'models').replace(dest_directory / 'models.orig')
-    dest_models.replace(dest_directory / 'models')
-
-    (dest_directory /'configs/models.yaml').replace(dest_directory / 'configs/models.yaml.orig')
-    dest_yaml.replace(dest_directory / 'configs/models.yaml')
-    print(f"""Migration successful.
-Original models directory moved to {dest_directory}/models.orig
-Original models.yaml file moved to {dest_directory}/configs/models.yaml.orig
-""")
-
+    if not version_3:
+        (dest_directory / 'models').replace(src_directory / 'models.orig')
+        print(f'Original models directory moved to {dest_directory}/models.orig')
+        
+    (dest_directory / 'configs' / 'models.yaml').replace(src_directory / 'configs' / 'models.yaml.orig')
+    print(f'Original models.yaml file moved to {dest_directory}/configs/models.yaml.orig')
+    
+    config_file.replace(config_file.with_suffix(''))
+    dest_models.replace(dest_models.with_suffix(''))
+    
 def main():
     parser = argparse.ArgumentParser(prog="invokeai-migrate3",
                                      description="""
@@ -545,34 +570,34 @@ It is safe to provide the same directory for both arguments, but it is better to
 script, which will perform a full upgrade in place."""
                                      )
     parser.add_argument('--from-directory',
-                        dest='root_directory',
+                        dest='src_root',
                         type=Path,
                         required=True,
                         help='Source InvokeAI 2.3 root directory (containing "invokeai.init" or "invokeai.yaml")'
                         )
     parser.add_argument('--to-directory',
-                        dest='dest_directory',
+                        dest='dest_root',
                         type=Path,
                         required=True,
                         help='Destination InvokeAI 3.0 directory (containing "invokeai.yaml")'
                         )
-# TO DO: Implement full directory scanning
-#    parser.add_argument('--all-models',
-#                        action="store_true",
-#                        help='Migrate all models found in `models` directory, not just those mentioned in models.yaml',
-#                        )
     args = parser.parse_args()
-    root_directory = args.root_directory
-    assert root_directory.is_dir(), f"{root_directory} is not a valid directory"
-    assert (root_directory / 'models').is_dir(), f"{root_directory} does not contain a 'models' subdirectory"
-    assert (root_directory / 'invokeai.init').exists() or (root_directory / 'invokeai.yaml').exists(), f"{root_directory} does not contain an InvokeAI init file."
+    src_root = args.src_root
+    assert src_root.is_dir(), f"{src_root} is not a valid directory"
+    assert (src_root / 'models').is_dir(), f"{src_root} does not contain a 'models' subdirectory"
+    assert (src_root / 'models' / 'hub').exists(), f"{src_root} does not contain a version 2.3 models directory"
+    assert (src_root / 'invokeai.init').exists() or (src_root / 'invokeai.yaml').exists(), f"{src_root} does not contain an InvokeAI init file."
 
-    dest_directory = args.dest_directory
-    assert dest_directory.is_dir(), f"{dest_directory} is not a valid directory"
-    assert (dest_directory / 'models').is_dir(), f"{dest_directory} does not contain a 'models' subdirectory"
-    assert (dest_directory / 'invokeai.yaml').exists(), f"{dest_directory} does not contain an InvokeAI init file."
+    dest_root = args.dest_root
+    assert dest_root.is_dir(), f"{dest_root} is not a valid directory"
+    config = InvokeAIAppConfig.get_config()
+    config.parse_args(['--root',str(dest_root)])
 
-    do_migrate(root_directory,dest_directory)
+    # TODO: revisit
+    # assert (dest_root / 'models').is_dir(), f"{dest_root} does not contain a 'models' subdirectory"
+    # assert (dest_root / 'invokeai.yaml').exists(), f"{dest_root} does not contain an InvokeAI init file."
+
+    do_migrate(src_root,dest_root)
 
 if __name__ == '__main__':
     main()

invokeai/backend/install/model_install_backend.py

@@ -19,7 +19,7 @@ from tqdm import tqdm
 import invokeai.configs as configs
 
 from invokeai.app.services.config import InvokeAIAppConfig
-from invokeai.backend.model_management import ModelManager, ModelType, BaseModelType, ModelVariantType
+from invokeai.backend.model_management import ModelManager, ModelType, BaseModelType, ModelVariantType, AddModelResult
 from invokeai.backend.model_management.model_probe import ModelProbe, SchedulerPredictionType, ModelProbeInfo
 from invokeai.backend.util import download_with_resume
 from ..util.logging import InvokeAILogger
@@ -173,74 +173,78 @@ class ModelInstall(object):
         # add requested models
         for path in selections.install_models:
             logger.info(f'Installing {path} [{job}/{jobs}]')
-            self.heuristic_install(path)
+            try:
+                self.heuristic_import(path)
+            except (ValueError, KeyError) as e:
+                logger.error(str(e))
             job += 1
             
         dlogging.set_verbosity(verbosity)
         self.mgr.commit()
 
-    def heuristic_install(self,
-                          model_path_id_or_url: Union[str,Path],
-                          models_installed: Set[Path]=None)->Set[Path]:
+    def heuristic_import(self,
+                         model_path_id_or_url: Union[str,Path],
+                         models_installed: Set[Path]=None,
+                         )->Dict[str, AddModelResult]:
+        '''
+        :param model_path_id_or_url: A Path to a local model to import, or a string representing its repo_id or URL
+        :param models_installed: Set of installed models, used for recursive invocation
+        Returns a set of dict objects corresponding to newly-created stanzas in models.yaml.
+        '''
 
         if not models_installed:
-            models_installed = set()
+            models_installed = dict()
             
         # A little hack to allow nested routines to retrieve info on the requested ID
         self.current_id = model_path_id_or_url
         path = Path(model_path_id_or_url)
-        
-        try:
-            # checkpoint file, or similar
-            if path.is_file():
-                models_installed.add(self._install_path(path))
+        # checkpoint file, or similar
+        if path.is_file():
+            models_installed.update({str(path):self._install_path(path)})
 
-            # folders style or similar
-            elif path.is_dir() and any([(path/x).exists() for x in {'config.json','model_index.json','learned_embeds.bin'}]):
-                models_installed.add(self._install_path(path))
+        # folders style or similar
+        elif path.is_dir() and any([(path/x).exists() for x in \
+                                    {'config.json','model_index.json','learned_embeds.bin','pytorch_lora_weights.bin'}
+                                    ]
+                                   ):
+            models_installed.update(self._install_path(path))
 
-            # recursive scan
-            elif path.is_dir():
-                for child in path.iterdir():
-                    self.heuristic_install(child, models_installed=models_installed)
+        # recursive scan
+        elif path.is_dir():
+            for child in path.iterdir():
+                self.heuristic_import(child, models_installed=models_installed)
 
-            # huggingface repo
-            elif len(str(model_path_id_or_url).split('/')) == 2:
-                models_installed.add(self._install_repo(str(path)))
+        # huggingface repo
+        elif len(str(model_path_id_or_url).split('/')) == 2:
+            models_installed.update({str(model_path_id_or_url): self._install_repo(str(model_path_id_or_url))})
 
-            # a URL
-            elif model_path_id_or_url.startswith(("http:", "https:", "ftp:")):
-                models_installed.add(self._install_url(model_path_id_or_url))
+        # a URL
+        elif str(model_path_id_or_url).startswith(("http:", "https:", "ftp:")):
+            models_installed.update({str(model_path_id_or_url): self._install_url(model_path_id_or_url)})
 
-            else:
-                logger.warning(f'{str(model_path_id_or_url)} is not recognized as a local path, repo ID or URL. Skipping')
-            
-        except ValueError as e:
-            logger.error(str(e))
+        else:
+            raise KeyError(f'{str(model_path_id_or_url)} is not recognized as a local path, repo ID or URL. Skipping')
 
         return models_installed
 
     # install a model from a local path. The optional info parameter is there to prevent
     # the model from being probed twice in the event that it has already been probed.
-    def _install_path(self, path: Path, info: ModelProbeInfo=None)->Path:
-        try:
-            # logger.debug(f'Probing {path}')
-            info = info or ModelProbe().heuristic_probe(path,self.prediction_helper)
-            model_name = path.stem if info.format=='checkpoint' else path.name
-            if self.mgr.model_exists(model_name, info.base_type, info.model_type):
-                raise ValueError(f'A model named "{model_name}" is already installed.')
-            attributes = self._make_attributes(path,info)
-            self.mgr.add_model(model_name = model_name,
-                               base_model = info.base_type,
-                               model_type = info.model_type,
-                               model_attributes = attributes,
-                               )
-        except Exception as e:
-            logger.warning(f'{str(e)} Skipping registration.')
-        return path
+    def _install_path(self, path: Path, info: ModelProbeInfo=None)->AddModelResult:
+        info = info or ModelProbe().heuristic_probe(path,self.prediction_helper)
+        if not info:
+            logger.warning(f'Unable to parse format of {path}')
+            return None
+        model_name = path.stem if path.is_file() else path.name
+        if self.mgr.model_exists(model_name, info.base_type, info.model_type):
+            raise ValueError(f'A model named "{model_name}" is already installed.')
+        attributes = self._make_attributes(path,info)
+        return self.mgr.add_model(model_name = model_name,
+                                  base_model = info.base_type,
+                                  model_type = info.model_type,
+                                  model_attributes = attributes,
+                                  )
 
-    def _install_url(self, url: str)->Path:
-        # copy to a staging area, probe, import and delete
+    def _install_url(self, url: str)->AddModelResult:
         with TemporaryDirectory(dir=self.config.models_path) as staging:
             location = download_with_resume(url,Path(staging))
             if not location:
@@ -252,7 +256,7 @@ class ModelInstall(object):
         # staged version will be garbage-collected at this time
         return self._install_path(Path(models_path), info)
 
-    def _install_repo(self, repo_id: str)->Path:
+    def _install_repo(self, repo_id: str)->AddModelResult:
         hinfo = HfApi().model_info(repo_id)
         
         # we try to figure out how to download this most economically
@@ -278,16 +282,16 @@ class ModelInstall(object):
                         location = self._download_hf_model(repo_id, files, staging)
                         break
                     elif f'learned_embeds.{suffix}' in files:
-                        location = self._download_hf_model(repo_id, ['learned_embeds.suffix'], staging)
+                        location = self._download_hf_model(repo_id, [f'learned_embeds.{suffix}'], staging)
                         break
             if not location:
                 logger.warning(f'Could not determine type of repo {repo_id}. Skipping install.')
-                return
-            
+                return {}
+
             info = ModelProbe().heuristic_probe(location, self.prediction_helper)
             if not info:
                 logger.warning(f'Could not probe {location}. Skipping install.')
-                return
+                return {}
             dest = self.config.models_path / info.base_type.value / info.model_type.value / self._get_model_name(repo_id,location)
             if dest.exists():
                 shutil.rmtree(dest)

invokeai/backend/model_management/__init__.py

@@ -1,7 +1,8 @@
 """
 Initialization file for invokeai.backend.model_management
 """
-from .model_manager import ModelManager, ModelInfo
+from .model_manager import ModelManager, ModelInfo, AddModelResult, SchedulerPredictionType
 from .model_cache import ModelCache
 from .models import BaseModelType, ModelType, SubModelType, ModelVariantType
+from .model_merge import ModelMerger, MergeInterpolationMethod
 

invokeai/backend/model_management/convert_ckpt_to_diffusers.py

@@ -29,7 +29,7 @@ import invokeai.backend.util.logging as logger
 from invokeai.app.services.config import InvokeAIAppConfig
 
 from .model_manager import ModelManager
-from .model_cache import ModelCache
+from picklescan.scanner import scan_file_path
 from .models import BaseModelType, ModelVariantType
 
 try:
@@ -1014,7 +1014,10 @@ def load_pipeline_from_original_stable_diffusion_ckpt(
             checkpoint = load_file(checkpoint_path)
         else:
             if scan_needed:
-                ModelCache.scan_model(checkpoint_path, checkpoint_path)
+                # scan model
+                scan_result = scan_file_path(checkpoint_path)
+                if scan_result.infected_files != 0:
+                    raise "The model {checkpoint_path} is potentially infected by malware. Aborting import."
             checkpoint = torch.load(checkpoint_path)
 
         # sometimes there is a state_dict key and sometimes not

invokeai/backend/model_management/lora.py

@@ -1,18 +1,15 @@
 from __future__ import annotations
 
 import copy
-from pathlib import Path
 from contextlib import contextmanager
-from typing import Optional, Dict, Tuple, Any
+from typing import Optional, Dict, Tuple, Any, Union, List
+from pathlib import Path
 
 import torch
-from safetensors.torch import load_file
-from torch.utils.hooks import RemovableHandle
-
-from diffusers.models import UNet2DConditionModel
-from transformers import CLIPTextModel
-
 from compel.embeddings_provider import BaseTextualInversionManager
+from diffusers.models import UNet2DConditionModel
+from safetensors.torch import load_file
+from transformers import CLIPTextModel, CLIPTokenizer
 
 class LoRALayerBase:
     #rank: Optional[int]
@@ -124,8 +121,8 @@ class LoRALayer(LoRALayerBase):
 
     def get_weight(self):
         if self.mid is not None:
-            up = self.up.reshape(up.shape[0], up.shape[1])
-            down = self.down.reshape(up.shape[0], up.shape[1])
+            up = self.up.reshape(self.up.shape[0], self.up.shape[1])
+            down = self.down.reshape(self.down.shape[0], self.down.shape[1])
             weight = torch.einsum("m n w h, i m, n j -> i j w h", self.mid, up, down)
         else:
             weight = self.up.reshape(self.up.shape[0], -1) @ self.down.reshape(self.down.shape[0], -1)
@@ -411,7 +408,7 @@ class LoRAModel: #(torch.nn.Module):
             else:
                 # TODO: diff/ia3/... format
                 print(
-                    f">> Encountered unknown lora layer module in {self.name}: {layer_key}"
+                    f">> Encountered unknown lora layer module in {model.name}: {layer_key}"
                 )
                 return
 
@@ -539,9 +536,10 @@ class ModelPatcher:
                             original_weights[module_key] = module.weight.detach().to(device="cpu", copy=True)
 
                         # enable autocast to calc fp16 loras on cpu
-                        with torch.autocast(device_type="cpu"):
-                            layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
-                            layer_weight = layer.get_weight() * lora_weight * layer_scale
+                        #with torch.autocast(device_type="cpu"):
+                        layer.to(dtype=torch.float32)
+                        layer_scale = layer.alpha / layer.rank if (layer.alpha and layer.rank) else 1.0
+                        layer_weight = layer.get_weight() * lora_weight * layer_scale
 
                         if module.weight.shape != layer_weight.shape:
                             # TODO: debug on lycoris
@@ -617,6 +615,24 @@ class ModelPatcher:
                 text_encoder.resize_token_embeddings(init_tokens_count)
 
 
+    @classmethod
+    @contextmanager
+    def apply_clip_skip(
+        cls,
+        text_encoder: CLIPTextModel,
+        clip_skip: int,
+    ):
+        skipped_layers = []
+        try:
+            for i in range(clip_skip):
+                skipped_layers.append(text_encoder.text_model.encoder.layers.pop(-1))
+
+            yield
+
+        finally:
+            while len(skipped_layers) > 0:
+                text_encoder.text_model.encoder.layers.append(skipped_layers.pop())
+
 class TextualInversionModel:
     name: str
     embedding: torch.Tensor # [n, 768]|[n, 1280]
@@ -655,6 +671,9 @@ class TextualInversionModel:
         else:
             result.embedding = next(iter(state_dict.values()))
 
+            if len(result.embedding.shape) == 1:
+                result.embedding = result.embedding.unsqueeze(0)
+
             if not isinstance(result.embedding, torch.Tensor):
                 raise ValueError(f"Invalid embeddings file: {file_path.name}")
 

invokeai/backend/model_management/model_cache.py

@@ -8,7 +8,7 @@ The cache returns context manager generators designed to load the
 model into the GPU within the context, and unload outside the
 context. Use like this:
 
-   cache = ModelCache(max_models_cached=6)
+   cache = ModelCache(max_cache_size=7.5)
    with cache.get_model('runwayml/stable-diffusion-1-5') as SD1,
           cache.get_model('stabilityai/stable-diffusion-2') as SD2:
        do_something_in_GPU(SD1,SD2)
@@ -91,7 +91,7 @@ class ModelCache(object):
         logger: types.ModuleType = logger
     ):
         '''
-        :param max_models: Maximum number of models to cache in CPU RAM [4]
+        :param max_cache_size: Maximum size of the RAM cache [6.0 GB]
         :param execution_device: Torch device to load active model into [torch.device('cuda')]
         :param storage_device: Torch device to save inactive model in [torch.device('cpu')]
         :param precision: Precision for loaded models [torch.float16]
@@ -100,8 +100,6 @@ class ModelCache(object):
         :param sha_chunksize: Chunksize to use when calculating sha256 model hash
         '''
         #max_cache_size = 9999
-        execution_device = torch.device('cuda')
-
         self.model_infos: Dict[str, ModelBase] = dict()
         self.lazy_offloading = lazy_offloading
         #self.sequential_offload: bool=sequential_offload
@@ -128,16 +126,6 @@ class ModelCache(object):
             key += f":{submodel_type}"
         return key
 
-    #def get_model(
-    #    self,
-    #    repo_id_or_path: Union[str, Path],
-    #    model_type: ModelType = ModelType.Diffusers,
-    #    subfolder: Path = None,
-    #    submodel: ModelType = None,
-    #    revision: str = None,
-    #    attach_model_part: Tuple[ModelType, str] = (None, None),
-    #    gpu_load: bool = True,
-    #) -> ModelLocker:  # ?? what does it return
     def _get_model_info(
         self,
         model_path: str,
@@ -354,7 +342,9 @@ class ModelCache(object):
         for model_key, cache_entry in self._cached_models.items():
             if not cache_entry.locked and cache_entry.loaded:
                 self.logger.debug(f'Offloading {model_key} from {self.execution_device} into {self.storage_device}')
-                cache_entry.model.to(self.storage_device)
+                with VRAMUsage() as mem:
+                    cache_entry.model.to(self.storage_device)
+                self.logger.debug(f'GPU VRAM freed: {(mem.vram_used/GIG):.2f} GB')
         
     def _local_model_hash(self, model_path: Union[str, Path]) -> str:
         sha = hashlib.sha256()

invokeai/backend/model_management/model_manager.py

@@ -231,16 +231,17 @@ from __future__ import annotations
 import os
 import hashlib
 import textwrap
+import yaml
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional, List, Tuple, Union, Set, Callable, types
-from shutil import rmtree
+from typing import Optional, List, Tuple, Union, Dict, Set, Callable, types
+from shutil import rmtree, move
 
 import torch
 from omegaconf import OmegaConf
 from omegaconf.dictconfig import DictConfig
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 import invokeai.backend.util.logging as logger
 from invokeai.app.services.config import InvokeAIAppConfig
@@ -249,7 +250,7 @@ from .model_cache import ModelCache, ModelLocker
 from .models import (
     BaseModelType, ModelType, SubModelType,
     ModelError, SchedulerPredictionType, MODEL_CLASSES,
-    ModelConfigBase,
+    ModelConfigBase, ModelNotFoundException,
     )
 
 # We are only starting to number the config file with release 3.
@@ -278,8 +279,13 @@ class InvalidModelError(Exception):
     "Raised when an invalid model is requested"
     pass
 
-MAX_CACHE_SIZE = 6.0  # GB
+class AddModelResult(BaseModel):
+    name: str = Field(description="The name of the model after installation")
+    model_type: ModelType = Field(description="The type of model")
+    base_model: BaseModelType = Field(description="The base model")
+    config: ModelConfigBase = Field(description="The configuration of the model")
 
+MAX_CACHE_SIZE = 6.0  # GB
 
 class ConfigMeta(BaseModel):
     version: str
@@ -306,10 +312,12 @@ class ModelManager(object):
         and sequential_offload boolean. Note that the default device
         type and precision are set up for a CUDA system running at half precision.
         """
-
         self.config_path = None
         if isinstance(config, (str, Path)):
             self.config_path = Path(config)
+            if not self.config_path.exists():
+                logger.warning(f'The file {self.config_path} was not found. Initializing a new file')
+                self.initialize_model_config(self.config_path)
             config = OmegaConf.load(self.config_path)
 
         elif not isinstance(config, DictConfig):
@@ -382,6 +390,16 @@ class ModelManager(object):
     def _get_model_cache_path(self, model_path):
         return self.app_config.models_path / ".cache" / hashlib.md5(str(model_path).encode()).hexdigest()
 
+    @classmethod
+    def initialize_model_config(cls, config_path: Path):
+        """Create empty config file"""
+        with open(config_path,'w') as yaml_file:
+            yaml_file.write(yaml.dump({'__metadata__':
+                                       {'version':'3.0.0'}
+                                       }
+                                      )
+                            )
+
     def get_model(
         self,
         model_name: str,
@@ -404,7 +422,7 @@ class ModelManager(object):
         if model_key not in self.models:
             self.scan_models_directory(base_model=base_model, model_type=model_type)
             if model_key not in self.models:
-                raise Exception(f"Model not found - {model_key}")
+                raise ModelNotFoundException(f"Model not found - {model_key}")
 
         model_config = self.models[model_key]
         model_path = self.app_config.root_path / model_config.path
@@ -416,14 +434,14 @@ class ModelManager(object):
 
             else:
                 self.models.pop(model_key, None)
-                raise Exception(f"Model not found - {model_key}")
+                raise ModelNotFoundException(f"Model not found - {model_key}")
 
         # vae/movq override
         # TODO: 
         if submodel_type is not None and hasattr(model_config, submodel_type):
             override_path = getattr(model_config, submodel_type)
             if override_path:
-                model_path = override_path
+                model_path = self.app_config.root_path / override_path
                 model_type = submodel_type
                 submodel_type = None
                 model_class = MODEL_CLASSES[base_model][model_type]
@@ -431,6 +449,7 @@ class ModelManager(object):
         # TODO: path
         # TODO: is it accurate to use path as id
         dst_convert_path = self._get_model_cache_path(model_path)
+
         model_path = model_class.convert_if_required(
             base_model=base_model,
             model_path=str(model_path), # TODO: refactor str/Path types logic
@@ -485,17 +504,32 @@ class ModelManager(object):
         """
         return [(self.parse_key(x)) for x in self.models.keys()]
 
+    def list_model(
+            self,
+            model_name: str,
+            base_model: BaseModelType,
+            model_type: ModelType,
+    ) -> dict:
+        """
+        Returns a dict describing one installed model, using
+        the combined format of the list_models() method.
+        """
+        models = self.list_models(base_model,model_type,model_name)
+        return models[0] if models else None
+
     def list_models(
         self,
         base_model: Optional[BaseModelType] = None,
         model_type: Optional[ModelType] = None,
+        model_name: Optional[str] = None,
     ) -> list[dict]:
         """
         Return a list of models.
         """
 
+        model_keys = [self.create_key(model_name, base_model, model_type)] if model_name else sorted(self.models, key=str.casefold)
         models = []
-        for model_key in sorted(self.models, key=str.casefold):
+        for model_key in model_keys:
             model_config = self.models[model_key]
 
             cur_model_name, cur_base_model, cur_model_type = self.parse_key(model_key)
@@ -540,10 +574,7 @@ class ModelManager(object):
         model_cfg = self.models.pop(model_key, None)
 
         if model_cfg is None:
-            self.logger.error(
-                f"Unknown model {model_key}"
-            )
-            return
+            raise KeyError(f"Unknown model {model_key}")
 
         # note: it not garantie to release memory(model can has other references)
         cache_ids = self.cache_keys.pop(model_key, [])
@@ -570,13 +601,16 @@ class ModelManager(object):
         model_type: ModelType,
         model_attributes: dict,
         clobber: bool = False,
-    ) -> None:
+    ) -> AddModelResult:
         """
         Update the named model with a dictionary of attributes. Will fail with an
         assertion error if the name already exists. Pass clobber=True to overwrite.
         On a successful update, the config will be changed in memory and the
         method will return True. Will fail with an assertion error if provided
         attributes are incorrect or the model name is missing.
+
+        The returned dict has the same format as the dict returned by
+        model_info().
         """
 
         model_class = MODEL_CLASSES[base_model][model_type]
@@ -600,13 +634,74 @@ class ModelManager(object):
                     old_model_cache.unlink()
 
             # remove in-memory cache
-            # note: it not garantie to release memory(model can has other references)
+            # note: it not guaranteed to release memory(model can has other references)
             cache_ids = self.cache_keys.pop(model_key, [])
             for cache_id in cache_ids:
                 self.cache.uncache_model(cache_id)
 
         self.models[model_key] = model_config
+        self.commit()
+        return AddModelResult(
+            name = model_name,
+            model_type = model_type,
+            base_model = base_model,
+            config = model_config,
+        )
 
+    def convert_model (
+            self,
+            model_name: str,
+            base_model: BaseModelType,
+            model_type: Union[ModelType.Main,ModelType.Vae],
+    ) -> AddModelResult:
+        '''
+        Convert a checkpoint file into a diffusers folder, deleting the cached
+        version and deleting the original checkpoint file if it is in the models
+        directory.
+        :param model_name: Name of the model to convert
+        :param base_model: Base model type
+        :param model_type: Type of model ['vae' or 'main']
+
+        This will raise a ValueError unless the model is a checkpoint.
+        '''
+        info = self.model_info(model_name, base_model, model_type)
+        if info["model_format"] != "checkpoint":
+            raise ValueError(f"not a checkpoint format model: {model_name}")
+
+        # We are taking advantage of a side effect of get_model() that converts check points
+        # into cached diffusers directories stored at `location`. It doesn't matter
+        # what submodeltype we request here, so we get the smallest.
+        submodel = {"submodel_type": SubModelType.Tokenizer} if model_type==ModelType.Main else {}
+        model = self.get_model(model_name,
+                               base_model,
+                               model_type,
+                               **submodel,
+                               )
+        checkpoint_path = self.app_config.root_path / info["path"]
+        old_diffusers_path = self.app_config.models_path / model.location
+        new_diffusers_path = self.app_config.models_path / base_model.value / model_type.value / model_name
+        if new_diffusers_path.exists():
+            raise ValueError(f"A diffusers model already exists at {new_diffusers_path}")
+
+        try:
+            move(old_diffusers_path,new_diffusers_path)
+            info["model_format"] = "diffusers"
+            info["path"] = str(new_diffusers_path.relative_to(self.app_config.root_path))
+            info.pop('config')
+
+            result = self.add_model(model_name, base_model, model_type,
+                                    model_attributes = info,
+                                    clobber=True)
+        except:
+            # something went wrong, so don't leave dangling diffusers model in directory or it will cause a duplicate model error!
+            rmtree(new_diffusers_path)
+            raise
+        
+        if checkpoint_path.exists() and checkpoint_path.is_relative_to(self.app_config.models_path):
+            checkpoint_path.unlink()
+        
+        return result
+    
     def search_models(self, search_folder):
         self.logger.info(f"Finding Models In: {search_folder}")
         models_folder_ckpt = Path(search_folder).glob("**/*.ckpt")
@@ -717,19 +812,19 @@ class ModelManager(object):
 
                             if model_path.is_relative_to(self.app_config.root_path):
                                 model_path = model_path.relative_to(self.app_config.root_path)
-                                try:
-                                    model_config: ModelConfigBase = model_class.probe_config(str(model_path))
-                                    self.models[model_key] = model_config
-                                    new_models_found = True
-                                except NotImplementedError as e:
-                                    self.logger.warning(e)
+                            try:
+                                model_config: ModelConfigBase = model_class.probe_config(str(model_path))
+                                self.models[model_key] = model_config
+                                new_models_found = True
+                            except NotImplementedError as e:
+                                self.logger.warning(e)
 
         imported_models = self.autoimport()
 
         if (new_models_found or imported_models) and self.config_path:
             self.commit()
 
-    def autoimport(self)->set[Path]:
+    def autoimport(self)->Dict[str, AddModelResult]:
         '''
         Scan the autoimport directory (if defined) and import new models, delete defunct models.
         '''
@@ -742,7 +837,6 @@ class ModelManager(object):
                                  prediction_type_helper = ask_user_for_prediction_type,
                                  )
         
-        installed = set()
         scanned_dirs = set()
         
         config = self.app_config
@@ -756,13 +850,14 @@ class ModelManager(object):
                 continue
 
             self.logger.info(f'Scanning {autodir} for models to import')
+            installed = dict()
         
             autodir = self.app_config.root_path / autodir
             if not autodir.exists():
                 continue
 
             items_scanned = 0
-            new_models_found = set()
+            new_models_found = dict()
             
             for root, dirs, files in os.walk(autodir):
                 items_scanned += len(dirs) + len(files)
@@ -771,16 +866,23 @@ class ModelManager(object):
                     if path in known_paths or path.parent in scanned_dirs:
                         scanned_dirs.add(path)
                         continue
-                    if any([(path/x).exists() for x in {'config.json','model_index.json','learned_embeds.bin'}]):
-                        new_models_found.update(installer.heuristic_install(path))
-                        scanned_dirs.add(path)
+                    if any([(path/x).exists() for x in {'config.json','model_index.json','learned_embeds.bin','pytorch_lora_weights.bin'}]):
+                        try:
+                            new_models_found.update(installer.heuristic_import(path))
+                            scanned_dirs.add(path)
+                        except ValueError as e:
+                            self.logger.warning(str(e))
 
                 for f in files:
                     path = Path(root) / f
                     if path in known_paths or path.parent in scanned_dirs:
                         continue
                     if path.suffix in {'.ckpt','.bin','.pth','.safetensors','.pt'}:
-                        new_models_found.update(installer.heuristic_install(path))
+                        try:
+                            import_result = installer.heuristic_import(path)
+                            new_models_found.update(import_result)
+                        except ValueError as e:
+                            self.logger.warning(str(e))
 
             self.logger.info(f'Scanned {items_scanned} files and directories, imported {len(new_models_found)} models')
             installed.update(new_models_found)
@@ -790,7 +892,7 @@ class ModelManager(object):
     def heuristic_import(self,
                          items_to_import: Set[str],
                          prediction_type_helper: Callable[[Path],SchedulerPredictionType]=None,
-                         )->Set[str]:
+                         )->Dict[str, AddModelResult]:
         '''Import a list of paths, repo_ids or URLs. Returns the set of
         successfully imported items.
         :param items_to_import: Set of strings corresponding to models to be imported.
@@ -803,20 +905,23 @@ class ModelManager(object):
         generally impossible to do this programmatically, so the
         prediction_type_helper usually asks the user to choose.
 
+        The result is a set of successfully installed models. Each element
+        of the set is a dict corresponding to the newly-created OmegaConf stanza for
+        that model.
+
+        May return the following exceptions:
+        - KeyError   - one or more of the items to import is not a valid path, repo_id or URL
+        - ValueError - a corresponding model already exists
         '''
         # avoid circular import here
         from invokeai.backend.install.model_install_backend import ModelInstall
-        successfully_installed = set()
+        successfully_installed = dict()
         
         installer = ModelInstall(config = self.app_config,
                                  prediction_type_helper = prediction_type_helper,
                                  model_manager = self)
         for thing in items_to_import:
-            try:
-                installed = installer.heuristic_install(thing)
-                successfully_installed.update(installed)
-            except Exception as e:
-                self.logger.warning(f'{thing} could not be imported: {str(e)}')
-                
+            installed = installer.heuristic_import(thing)
+            successfully_installed.update(installed)
         self.commit()                
         return successfully_installed

invokeai/backend/model_management/model_merge.py

@@ -0,0 +1,131 @@
+"""
+invokeai.backend.model_management.model_merge exports:
+merge_diffusion_models() -- combine multiple models by location and return a pipeline object
+merge_diffusion_models_and_commit() -- combine multiple models by ModelManager ID and write to models.yaml
+
+Copyright (c) 2023 Lincoln Stein and the InvokeAI Development Team
+"""
+
+import warnings
+from enum import Enum
+from pathlib import Path
+from diffusers import DiffusionPipeline
+from diffusers import logging as dlogging
+from typing import List, Union
+
+import invokeai.backend.util.logging as logger
+
+from ...backend.model_management import ModelManager, ModelType, BaseModelType, ModelVariantType, AddModelResult
+
+class MergeInterpolationMethod(str, Enum):
+    WeightedSum = "weighted_sum"
+    Sigmoid = "sigmoid"
+    InvSigmoid = "inv_sigmoid"
+    AddDifference = "add_difference"
+
+class ModelMerger(object):
+    def __init__(self, manager: ModelManager):
+        self.manager = manager
+
+    def merge_diffusion_models(
+            self,
+            model_paths: List[Path],
+            alpha: float = 0.5,
+            interp: MergeInterpolationMethod = None,
+            force: bool = False,
+            **kwargs,
+    ) -> DiffusionPipeline:
+        """
+        :param model_paths:  up to three models, designated by their local paths or HuggingFace repo_ids
+        :param alpha: The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
+                   would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
+        :param interp: The interpolation method to use for the merging. Supports "sigmoid", "inv_sigmoid", "add_difference" and None.
+                   Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported.
+        :param force:  Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
+
+        **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
+             cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
+        """
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            verbosity = dlogging.get_verbosity()
+            dlogging.set_verbosity_error()
+
+            pipe = DiffusionPipeline.from_pretrained(
+                model_paths[0],
+                custom_pipeline="checkpoint_merger",
+            )
+            merged_pipe = pipe.merge(
+                pretrained_model_name_or_path_list=model_paths,
+                alpha=alpha,
+                interp=interp.value if interp else None, #diffusers API treats None as "weighted sum"
+                force=force,
+                **kwargs,
+            )
+            dlogging.set_verbosity(verbosity)
+        return merged_pipe
+
+
+    def merge_diffusion_models_and_save (
+            self,
+            model_names: List[str],
+            base_model: Union[BaseModelType,str],
+            merged_model_name: str,
+            alpha: float = 0.5,
+            interp: MergeInterpolationMethod = None,
+            force: bool = False,
+            **kwargs,
+    ) -> AddModelResult:
+        """
+        :param models: up to three models, designated by their InvokeAI models.yaml model name
+        :param base_model: base model (must be the same for all merged models!)
+        :param merged_model_name: name for new model
+        :param alpha: The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
+                   would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
+        :param interp: The interpolation method to use for the merging. Supports "weighted_average", "sigmoid", "inv_sigmoid", "add_difference" and None.
+                   Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported. Add_difference is A+(B-C).
+        :param force:  Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
+
+        **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
+             cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
+        """
+        model_paths = list()
+        config = self.manager.app_config
+        base_model = BaseModelType(base_model)
+        vae = None
+        
+        for mod in model_names:
+            info = self.manager.list_model(mod, base_model=base_model, model_type=ModelType.Main)
+            assert info,                                f"model {mod}, base_model {base_model}, is unknown"
+            assert info["model_format"] == "diffusers", f"{mod} is not a diffusers model. It must be optimized before merging"
+            assert info["variant"] == "normal",         f"{mod} is a {info['variant']} model, which cannot currently be merged"
+            assert len(model_names) <= 2 or \
+                interp==MergeInterpolationMethod.AddDifference, "When merging three models, only the 'add_difference' merge method is supported"
+            # pick up the first model's vae
+            if mod == model_names[0]:
+                vae = info.get("vae")
+            model_paths.extend([config.root_path / info["path"]])
+
+        merge_method = None if interp == 'weighted_sum' else MergeInterpolationMethod(interp)
+        logger.debug(f'interp = {interp}, merge_method={merge_method}')
+        merged_pipe = self.merge_diffusion_models(
+           model_paths, alpha, merge_method, force, **kwargs
+        )
+        dump_path = config.models_path / base_model.value / ModelType.Main.value
+        dump_path.mkdir(parents=True, exist_ok=True)
+        dump_path = dump_path / merged_model_name
+
+        merged_pipe.save_pretrained(dump_path, safe_serialization=1)
+        attributes = dict(
+            path = str(dump_path),
+            description = f"Merge of models {', '.join(model_names)}",
+            model_format = "diffusers",
+            variant = ModelVariantType.Normal.value,
+            vae = vae,
+        )
+        return self.manager.add_model(merged_model_name,
+                                      base_model = base_model,
+                                      model_type = ModelType.Main,
+                                      model_attributes = attributes,
+                                      clobber = True
+                                      )

invokeai/backend/model_management/model_probe.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 
 from diffusers import ModelMixin, ConfigMixin
 from pathlib import Path
-from typing import Callable, Literal, Union, Dict
+from typing import Callable, Literal, Union, Dict, Optional
 from picklescan.scanner import scan_file_path
 
 from .models import (
@@ -64,8 +64,8 @@ class ModelProbe(object):
     @classmethod
     def probe(cls,
               model_path: Path,
-              model: Union[Dict, ModelMixin] = None,
-              prediction_type_helper: Callable[[Path],SchedulerPredictionType] = None)->ModelProbeInfo:
+              model: Optional[Union[Dict, ModelMixin]] = None,
+              prediction_type_helper: Optional[Callable[[Path],SchedulerPredictionType]] = None)->ModelProbeInfo:
         '''
         Probe the model at model_path and return sufficient information about it
         to place it somewhere in the models directory hierarchy. If the model is
@@ -78,7 +78,6 @@ class ModelProbe(object):
             format_type = 'diffusers' if model_path.is_dir() else 'checkpoint'
         else:
             format_type = 'diffusers' if isinstance(model,(ConfigMixin,ModelMixin)) else 'checkpoint'
-
         model_info = None
         try:
             model_type = cls.get_model_type_from_folder(model_path, model) \
@@ -105,7 +104,7 @@ class ModelProbe(object):
                                      ) else 512,
             )
         except Exception:
-            return None
+            raise
 
         return model_info
 
@@ -127,6 +126,8 @@ class ModelProbe(object):
                 return ModelType.Vae
             elif any(key.startswith(v) for v in {"lora_te_", "lora_unet_"}):
                 return ModelType.Lora
+            elif any(key.endswith(v) for v in {"to_k_lora.up.weight", "to_q_lora.down.weight"}):
+                return ModelType.Lora
             elif any(key.startswith(v) for v in {"control_model", "input_blocks"}):
                 return ModelType.ControlNet
             elif key in {"emb_params", "string_to_param"}:
@@ -137,7 +138,7 @@ class ModelProbe(object):
             if len(ckpt) < 10 and all(isinstance(v, torch.Tensor) for v in ckpt.values()):
                 return ModelType.TextualInversion
         
-        raise ValueError("Unable to determine model type")
+        raise ValueError(f"Unable to determine model type for {model_path}")
 
     @classmethod
     def get_model_type_from_folder(cls, folder_path: Path, model: ModelMixin)->ModelType:
@@ -167,7 +168,7 @@ class ModelProbe(object):
             return type
 
         # give up
-        raise ValueError("Unable to determine model type")
+        raise ValueError(f"Unable to determine model type for {folder_path}")
 
     @classmethod
     def _scan_and_load_checkpoint(cls,model_path: Path)->dict:

invokeai/backend/model_management/models/__init__.py

@@ -2,7 +2,7 @@ import inspect
 from enum import Enum
 from pydantic import BaseModel
 from typing import Literal, get_origin
-from .base import BaseModelType, ModelType, SubModelType, ModelBase, ModelConfigBase, ModelVariantType, SchedulerPredictionType, ModelError, SilenceWarnings
+from .base import BaseModelType, ModelType, SubModelType, ModelBase, ModelConfigBase, ModelVariantType, SchedulerPredictionType, ModelError, SilenceWarnings, ModelNotFoundException
 from .stable_diffusion import StableDiffusion1Model, StableDiffusion2Model
 from .vae import VaeModel
 from .lora import LoRAModel
@@ -68,7 +68,11 @@ def get_model_config_enums():
     enums = list()
 
     for model_config in MODEL_CONFIGS:
-        fields = inspect.get_annotations(model_config)
+
+        if hasattr(inspect,'get_annotations'):
+            fields = inspect.get_annotations(model_config)
+        else:
+            fields = model_config.__annotations__
         try:
             field = fields["model_format"]
         except:

invokeai/backend/model_management/models/base.py

@@ -15,6 +15,9 @@ from contextlib import suppress
 from pydantic import BaseModel, Field
 from typing import List, Dict, Optional, Type, Literal, TypeVar, Generic, Callable, Any, Union
 
+class ModelNotFoundException(Exception):
+    pass
+
 class BaseModelType(str, Enum):
     StableDiffusion1 = "sd-1"
     StableDiffusion2 = "sd-2"

invokeai/backend/model_management/models/stable_diffusion.py

@@ -116,7 +116,7 @@ class StableDiffusion1Model(DiffusersModel):
                 version=BaseModelType.StableDiffusion1,
                 model_config=config,
                 output_path=output_path,
-            )
+           )
         else:
             return model_path
 

invokeai/backend/model_management/models/textual_inversion.py

@@ -8,6 +8,7 @@ from .base import (
     ModelType,
     SubModelType,
     classproperty,
+    ModelNotFoundException,
 )
 # TODO: naming
 from ..lora import TextualInversionModel as TextualInversionModelRaw
@@ -37,8 +38,15 @@ class TextualInversionModel(ModelBase):
         if child_type is not None:
             raise Exception("There is no child models in textual inversion")
 
+        checkpoint_path = self.model_path
+        if os.path.isdir(checkpoint_path):
+            checkpoint_path = os.path.join(checkpoint_path, "learned_embeds.bin")
+
+        if not os.path.exists(checkpoint_path):
+            raise ModelNotFoundException()
+
         model = TextualInversionModelRaw.from_checkpoint(
-            file_path=self.model_path,
+            file_path=checkpoint_path,
             dtype=torch_dtype,
         )
 

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -7,7 +7,7 @@ import secrets
 from collections.abc import Sequence
 from dataclasses import dataclass, field
 from typing import Any, Callable, Generic, List, Optional, Type, TypeVar, Union
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 import einops
 import PIL.Image
@@ -17,12 +17,11 @@ import psutil
 import torch
 import torchvision.transforms as T
 from diffusers.models import AutoencoderKL, UNet2DConditionModel
-from diffusers.models.controlnet import ControlNetModel, ControlNetOutput
+from diffusers.models.controlnet import ControlNetModel
 from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
 from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import (
     StableDiffusionPipeline,
 )
-from diffusers.pipelines.controlnet import MultiControlNetModel
 
 from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import (
     StableDiffusionImg2ImgPipeline,
@@ -46,7 +45,7 @@ from .diffusion import (
     InvokeAIDiffuserComponent,
     PostprocessingSettings,
 )
-from .offloading import FullyLoadedModelGroup, LazilyLoadedModelGroup, ModelGroup
+from .offloading import FullyLoadedModelGroup, ModelGroup
 
 @dataclass
 class PipelineIntermediateState:
@@ -105,7 +104,7 @@ class AddsMaskGuidance:
     _debug: Optional[Callable] = None
 
     def __call__(
-        self, step_output: BaseOutput | SchedulerOutput, t: torch.Tensor, conditioning
+        self, step_output: Union[BaseOutput, SchedulerOutput], t: torch.Tensor, conditioning
     ) -> BaseOutput:
         output_class = step_output.__class__  # We'll create a new one with masked data.
 
@@ -361,37 +360,34 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         ):
             self.enable_xformers_memory_efficient_attention()
         else:
-            if torch.backends.mps.is_available():
-                # until pytorch #91617 is fixed, slicing is borked on MPS
-                # https://github.com/pytorch/pytorch/issues/91617
-                # fix is in https://github.com/kulinseth/pytorch/pull/222 but no idea when it will get merged to pytorch mainline.
-                pass
+            if self.device.type == "cpu" or self.device.type == "mps":
+                mem_free = psutil.virtual_memory().free
+            elif self.device.type == "cuda":
+                mem_free, _ = torch.cuda.mem_get_info(normalize_device(self.device))
             else:
-                if self.device.type == "cpu" or self.device.type == "mps":
-                    mem_free = psutil.virtual_memory().free
-                elif self.device.type == "cuda":
-                    mem_free, _ = torch.cuda.mem_get_info(normalize_device(self.device))
-                else:
-                    raise ValueError(f"unrecognized device {self.device}")
-                # input tensor of [1, 4, h/8, w/8]
-                # output tensor of [16, (h/8 * w/8), (h/8 * w/8)]
-                bytes_per_element_needed_for_baddbmm_duplication = (
-                    latents.element_size() + 4
-                )
-                max_size_required_for_baddbmm = (
-                    16
-                    * latents.size(dim=2)
-                    * latents.size(dim=3)
-                    * latents.size(dim=2)
-                    * latents.size(dim=3)
-                    * bytes_per_element_needed_for_baddbmm_duplication
-                )
-                if max_size_required_for_baddbmm > (
-                    mem_free * 3.0 / 4.0
-                ):  # 3.3 / 4.0 is from old Invoke code
-                    self.enable_attention_slicing(slice_size="max")
-                else:
-                    self.disable_attention_slicing()
+                raise ValueError(f"unrecognized device {self.device}")
+            # input tensor of [1, 4, h/8, w/8]
+            # output tensor of [16, (h/8 * w/8), (h/8 * w/8)]
+            bytes_per_element_needed_for_baddbmm_duplication = (
+                latents.element_size() + 4
+            )
+            max_size_required_for_baddbmm = (
+                16
+                * latents.size(dim=2)
+                * latents.size(dim=3)
+                * latents.size(dim=2)
+                * latents.size(dim=3)
+                * bytes_per_element_needed_for_baddbmm_duplication
+            )
+            if max_size_required_for_baddbmm > (
+                mem_free * 3.0 / 4.0
+            ):  # 3.3 / 4.0 is from old Invoke code
+                self.enable_attention_slicing(slice_size="max")
+            elif torch.backends.mps.is_available():
+                # diffusers recommends always enabling for mps
+                self.enable_attention_slicing(slice_size="max")
+            else:
+                self.disable_attention_slicing()
 
     def to(self, torch_device: Optional[Union[str, torch.device]] = None, silence_dtype_warnings=False):
         # overridden method; types match the superclass.
@@ -917,20 +913,11 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
     def non_noised_latents_from_image(self, init_image, *, device: torch.device, dtype):
         init_image = init_image.to(device=device, dtype=dtype)
         with torch.inference_mode():
-            if device.type == "mps":
-                # workaround for torch MPS bug that has been fixed in https://github.com/kulinseth/pytorch/pull/222
-                # TODO remove this workaround once kulinseth#222 is merged to pytorch mainline
-                self.vae.to(CPU_DEVICE)
-                init_image = init_image.to(CPU_DEVICE)
-            else:
-                self._model_group.load(self.vae)
+            self._model_group.load(self.vae)
             init_latent_dist = self.vae.encode(init_image).latent_dist
             init_latents = init_latent_dist.sample().to(
                 dtype=dtype
             )  # FIXME: uses torch.randn. make reproducible!
-            if device.type == "mps":
-                self.vae.to(device)
-                init_latents = init_latents.to(device)
 
         init_latents = 0.18215 * init_latents
         return init_latents

invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py

@@ -248,9 +248,6 @@ class InvokeAIDiffuserComponent:
             x_twice, sigma_twice, both_conditionings, **kwargs,
         )
         unconditioned_next_x, conditioned_next_x = both_results.chunk(2)
-        if conditioned_next_x.device.type == "mps":
-            # prevent a result filled with zeros. seems to be a torch bug.
-            conditioned_next_x = conditioned_next_x.clone()
         return unconditioned_next_x, conditioned_next_x
 
     def _apply_standard_conditioning_sequentially(
@@ -264,9 +261,6 @@ class InvokeAIDiffuserComponent:
         # low-memory sequential path
         unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning, **kwargs)
         conditioned_next_x = self.model_forward_callback(x, sigma, conditioning, **kwargs)
-        if conditioned_next_x.device.type == "mps":
-            # prevent a result filled with zeros. seems to be a torch bug.
-            conditioned_next_x = conditioned_next_x.clone()
         return unconditioned_next_x, conditioned_next_x
 
     # TODO: looks unused

invokeai/backend/stable_diffusion/offloading.py

@@ -4,7 +4,7 @@ import warnings
 import weakref
 from abc import ABCMeta, abstractmethod
 from collections.abc import MutableMapping
-from typing import Callable
+from typing import Callable, Union
 
 import torch
 from accelerate.utils import send_to_device
@@ -117,7 +117,7 @@ class LazilyLoadedModelGroup(ModelGroup):
     """
 
     _hooks: MutableMapping[torch.nn.Module, RemovableHandle]
-    _current_model_ref: Callable[[], torch.nn.Module | _NoModel]
+    _current_model_ref: Callable[[], Union[torch.nn.Module, _NoModel]]
 
     def __init__(self, execution_device: torch.device):
         super().__init__(execution_device)

invokeai/backend/util/devices.py

@@ -4,6 +4,7 @@ from contextlib import nullcontext
 
 import torch
 from torch import autocast
+from typing import Union
 from invokeai.app.services.config import InvokeAIAppConfig
 
 CPU_DEVICE = torch.device("cpu")
@@ -28,6 +29,8 @@ def choose_precision(device: torch.device) -> str:
         device_name = torch.cuda.get_device_name(device)
         if not ("GeForce GTX 1660" in device_name or "GeForce GTX 1650" in device_name):
             return "float16"
+    elif device.type == "mps":
+        return "float16"
     return "float32"
 
 
@@ -49,7 +52,7 @@ def choose_autocast(precision):
     return nullcontext
 
 
-def normalize_device(device: str | torch.device) -> torch.device:
+def normalize_device(device: Union[str, torch.device]) -> torch.device:
     """Ensure device has a device index defined, if appropriate."""
     device = torch.device(device)
     if device.index is None:

invokeai/backend/util/mps_fixes.py

@@ -0,0 +1,63 @@
+import torch
+
+
+if torch.backends.mps.is_available():
+    torch.empty = torch.zeros
+
+
+_torch_layer_norm = torch.nn.functional.layer_norm
+def new_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-05):
+    if input.device.type == "mps" and input.dtype == torch.float16:
+        input = input.float()
+        if weight is not None:
+            weight = weight.float()
+        if bias is not None:
+            bias = bias.float()
+        return _torch_layer_norm(input, normalized_shape, weight, bias, eps).half()
+    else:
+        return _torch_layer_norm(input, normalized_shape, weight, bias, eps)
+
+torch.nn.functional.layer_norm = new_layer_norm
+
+
+_torch_tensor_permute = torch.Tensor.permute
+def new_torch_tensor_permute(input, *dims):
+    result = _torch_tensor_permute(input, *dims)
+    if input.device == "mps" and input.dtype == torch.float16:
+        result = result.contiguous()
+    return result
+
+torch.Tensor.permute = new_torch_tensor_permute
+
+
+_torch_lerp = torch.lerp
+def new_torch_lerp(input, end, weight, *, out=None):
+    if input.device.type == "mps" and input.dtype == torch.float16:
+        input = input.float()
+        end = end.float()
+        if isinstance(weight, torch.Tensor):
+            weight = weight.float()
+        if out is not None:
+            out_fp32 = torch.zeros_like(out, dtype=torch.float32)
+        else:
+            out_fp32 = None
+        result = _torch_lerp(input, end, weight, out=out_fp32)
+        if out is not None:
+            out.copy_(out_fp32.half())
+            del out_fp32
+        return result.half()
+
+    else:
+        return _torch_lerp(input, end, weight, out=out)
+
+torch.lerp = new_torch_lerp
+
+
+_torch_interpolate = torch.nn.functional.interpolate
+def new_torch_interpolate(input, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None, antialias=False):
+    if input.device.type == "mps" and input.dtype == torch.float16:
+        return _torch_interpolate(input.float(), size, scale_factor, mode, align_corners, recompute_scale_factor, antialias).half()
+    else:
+        return _torch_interpolate(input, size, scale_factor, mode, align_corners, recompute_scale_factor, antialias)
+
+torch.nn.functional.interpolate = new_torch_interpolate

invokeai/frontend/install/model_install.py

@@ -382,10 +382,21 @@ class addModelsForm(CyclingForm, npyscreen.FormMultiPage):
         )
         return min(cols, len(self.installed_models))
 
+    def confirm_deletions(self, selections: InstallSelections)->bool:
+        remove_models = selections.remove_models
+        if len(remove_models) > 0:
+            mods = "\n".join([ModelManager.parse_key(x)[0] for x in remove_models])
+            return npyscreen.notify_ok_cancel(f"These unchecked models will be deleted from disk. Continue?\n---------\n{mods}")
+        else:
+            return True
+
     def on_execute(self):
-        self.monitor.entry_widget.buffer(['Processing...'],scroll_end=True)
         self.marshall_arguments()
         app = self.parentApp
+        if not self.confirm_deletions(app.install_selections):
+            return
+            
+        self.monitor.entry_widget.buffer(['Processing...'],scroll_end=True)
         self.ok_button.hidden = True
         self.display()
         
@@ -417,6 +428,8 @@ class addModelsForm(CyclingForm, npyscreen.FormMultiPage):
         
     def on_done(self):
         self.marshall_arguments()
+        if not self.confirm_deletions(self.parentApp.install_selections):
+            return
         self.parentApp.setNextForm(None)
         self.parentApp.user_cancelled = False
         self.editing = False
@@ -678,9 +691,8 @@ def select_and_download_models(opt: Namespace):
 
     # this is where the TUI is called
     else:
-        # needed because the torch library is loaded, even though we don't use it
-        # currently commented out because it has started generating errors (?)
-        # torch.multiprocessing.set_start_method("spawn")
+        # needed to support the probe() method running under a subprocess
+        torch.multiprocessing.set_start_method("spawn")
 
         # the third argument is needed in the Windows 11 environment in
         # order to launch and resize a console window running this program
@@ -761,7 +773,7 @@ def main():
     config.parse_args(invoke_args)
     logger = InvokeAILogger().getLogger(config=config)
 
-    if not (config.root_dir / config.conf_path.parent).exists():
+    if not (config.conf_path / 'models.yaml').exists():
         logger.info(
             "Your InvokeAI root directory is not set up. Calling invokeai-configure."
         )

invokeai/frontend/install/widgets.py

@@ -18,7 +18,7 @@ from curses import BUTTON2_CLICKED,BUTTON3_CLICKED
 
 # minimum size for UIs
 MIN_COLS = 130
-MIN_LINES = 40
+MIN_LINES = 45
 
 # -------------------------------------
 def set_terminal_size(columns: int, lines: int, launch_command: str=None):

invokeai/frontend/legacy_launch_invokeai.py

@@ -0,0 +1,19 @@
+import os
+import sys
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--web', action='store_true')
+    opts,_ = parser.parse_known_args()
+
+    if opts.web:
+        sys.argv.pop(sys.argv.index('--web'))
+        from invokeai.app.api_app import invoke_api
+        invoke_api()
+    else:
+        from invokeai.app.cli_app import invoke_cli
+        invoke_cli()
+
+if __name__ == '__main__':
+    main()

invokeai/frontend/merge/__init__.py

@@ -1,4 +1,5 @@
 """
 Initialization file for invokeai.frontend.merge
 """
-from .merge_diffusers import main as invokeai_merge_diffusers, merge_diffusion_models
+from .merge_diffusers import main as invokeai_merge_diffusers
+

invokeai/frontend/merge/merge_diffusers.py

@@ -6,9 +6,7 @@ Copyright (c) 2023 Lincoln Stein and the InvokeAI Development Team
 """
 import argparse
 import curses
-import os
 import sys
-import warnings
 from argparse import Namespace
 from pathlib import Path
 from typing import List, Union
@@ -20,99 +18,15 @@ from npyscreen import widget
 from omegaconf import OmegaConf
 
 import invokeai.backend.util.logging as logger
-from invokeai.services.config import InvokeAIAppConfig
-from ...backend.model_management import ModelManager
-from ...frontend.install.widgets import FloatTitleSlider
+from invokeai.app.services.config import InvokeAIAppConfig
+from invokeai.backend.model_management import (
+    ModelMerger, MergeInterpolationMethod,
+    ModelManager, ModelType, BaseModelType,
+)
+from invokeai.frontend.install.widgets import FloatTitleSlider, TextBox, SingleSelectColumns
 
-DEST_MERGED_MODEL_DIR = "merged_models"
 config = InvokeAIAppConfig.get_config()
 
-def merge_diffusion_models(
-    model_ids_or_paths: List[Union[str, Path]],
-    alpha: float = 0.5,
-    interp: str = None,
-    force: bool = False,
-    **kwargs,
-) -> DiffusionPipeline:
-    """
-    model_ids_or_paths - up to three models, designated by their local paths or HuggingFace repo_ids
-    alpha  - The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
-               would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
-    interp - The interpolation method to use for the merging. Supports "sigmoid", "inv_sigmoid", "add_difference" and None.
-               Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported.
-    force  - Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
-
-    **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
-         cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
-    """
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        verbosity = dlogging.get_verbosity()
-        dlogging.set_verbosity_error()
-
-        pipe = DiffusionPipeline.from_pretrained(
-            model_ids_or_paths[0],
-            cache_dir=kwargs.get("cache_dir", config.cache_dir),
-            custom_pipeline="checkpoint_merger",
-        )
-        merged_pipe = pipe.merge(
-            pretrained_model_name_or_path_list=model_ids_or_paths,
-            alpha=alpha,
-            interp=interp,
-            force=force,
-            **kwargs,
-        )
-        dlogging.set_verbosity(verbosity)
-    return merged_pipe
-
-
-def merge_diffusion_models_and_commit(
-    models: List["str"],
-    merged_model_name: str,
-    alpha: float = 0.5,
-    interp: str = None,
-    force: bool = False,
-    **kwargs,
-):
-    """
-    models - up to three models, designated by their InvokeAI models.yaml model name
-    merged_model_name = name for new model
-    alpha  - The interpolation parameter. Ranges from 0 to 1.  It affects the ratio in which the checkpoints are merged. A 0.8 alpha
-               would mean that the first model checkpoints would affect the final result far less than an alpha of 0.2
-    interp - The interpolation method to use for the merging. Supports "weighted_average", "sigmoid", "inv_sigmoid", "add_difference" and None.
-               Passing None uses the default interpolation which is weighted sum interpolation. For merging three checkpoints, only "add_difference" is supported. Add_difference is A+(B-C).
-    force  - Whether to ignore mismatch in model_config.json for the current models. Defaults to False.
-
-    **kwargs - the default DiffusionPipeline.get_config_dict kwargs:
-         cache_dir, resume_download, force_download, proxies, local_files_only, use_auth_token, revision, torch_dtype, device_map
-    """
-    config_file = config.model_conf_path
-    model_manager = ModelManager(OmegaConf.load(config_file))
-    for mod in models:
-        assert mod in model_manager.model_names(), f'** Unknown model "{mod}"'
-        assert (
-            model_manager.model_info(mod).get("format", None) == "diffusers"
-        ), f"** {mod} is not a diffusers model. It must be optimized before merging."
-    model_ids_or_paths = [model_manager.model_name_or_path(x) for x in models]
-
-    merged_pipe = merge_diffusion_models(
-        model_ids_or_paths, alpha, interp, force, **kwargs
-    )
-    dump_path = config.models_dir / DEST_MERGED_MODEL_DIR
-
-    os.makedirs(dump_path, exist_ok=True)
-    dump_path = dump_path / merged_model_name
-    merged_pipe.save_pretrained(dump_path, safe_serialization=1)
-    import_args = dict(
-        model_name=merged_model_name, description=f'Merge of models {", ".join(models)}'
-    )
-    if vae := model_manager.config[models[0]].get("vae", None):
-        logger.info(f"Using configured VAE assigned to {models[0]}")
-        import_args.update(vae=vae)
-    model_manager.import_diffuser_model(dump_path, **import_args)
-    model_manager.commit(config_file)
-
-
 def _parse_args() -> Namespace:
     parser = argparse.ArgumentParser(description="InvokeAI model merging")
     parser.add_argument(
@@ -131,10 +45,17 @@ def _parse_args() -> Namespace:
     )
     parser.add_argument(
         "--models",
+        dest="model_names",
         type=str,
         nargs="+",
         help="Two to three model names to be merged",
     )
+    parser.add_argument(
+        "--base_model",
+        type=str,
+        choices=[x.value for x in BaseModelType],
+        help="The base model shared by the models to be merged",
+    )
     parser.add_argument(
         "--merged_model_name",
         "--destination",
@@ -192,6 +113,7 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
         window_height, window_width = curses.initscr().getmaxyx()
 
         self.model_names = self.get_model_names()
+        self.current_base = 0
         max_width = max([len(x) for x in self.model_names])
         max_width += 6
         horizontal_layout = max_width * 3 < window_width
@@ -208,12 +130,26 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             value="Use up and down arrows to move, <space> to select an item, <tab> and <shift-tab> to move from one field to the next.",
             editable=False,
         )
+        self.nextrely += 1
+        self.base_select = self.add_widget_intelligent(
+            SingleSelectColumns,
+            values=[
+                'Models Built on SD-1.x',
+                'Models Built on SD-2.x',
+            ],
+            value=[self.current_base],
+            columns = 4,
+            max_height = 2,
+            relx=8,
+            scroll_exit = True,
+        )
+        self.base_select.on_changed = self._populate_models
         self.add_widget_intelligent(
             npyscreen.FixedText,
             value="MODEL 1",
             color="GOOD",
             editable=False,
-            rely=4 if horizontal_layout else None,
+            rely=6 if horizontal_layout else None,
         )
         self.model1 = self.add_widget_intelligent(
             npyscreen.SelectOne,
@@ -222,7 +158,7 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             max_height=len(self.model_names),
             max_width=max_width,
             scroll_exit=True,
-            rely=5,
+            rely=7,
         )
         self.add_widget_intelligent(
             npyscreen.FixedText,
@@ -230,7 +166,7 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             color="GOOD",
             editable=False,
             relx=max_width + 3 if horizontal_layout else None,
-            rely=4 if horizontal_layout else None,
+            rely=6 if horizontal_layout else None,
         )
         self.model2 = self.add_widget_intelligent(
             npyscreen.SelectOne,
@@ -240,7 +176,7 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             max_height=len(self.model_names),
             max_width=max_width,
             relx=max_width + 3 if horizontal_layout else None,
-            rely=5 if horizontal_layout else None,
+            rely=7 if horizontal_layout else None,
             scroll_exit=True,
         )
         self.add_widget_intelligent(
@@ -249,7 +185,7 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             color="GOOD",
             editable=False,
             relx=max_width * 2 + 3 if horizontal_layout else None,
-            rely=4 if horizontal_layout else None,
+            rely=6 if horizontal_layout else None,
         )
         models_plus_none = self.model_names.copy()
         models_plus_none.insert(0, "None")
@@ -262,24 +198,26 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             max_width=max_width,
             scroll_exit=True,
             relx=max_width * 2 + 3 if horizontal_layout else None,
-            rely=5 if horizontal_layout else None,
+            rely=7 if horizontal_layout else None,
         )
         for m in [self.model1, self.model2, self.model3]:
             m.when_value_edited = self.models_changed
         self.merged_model_name = self.add_widget_intelligent(
-            npyscreen.TitleText,
+            TextBox,
             name="Name for merged model:",
             labelColor="CONTROL",
+            max_height=3,
             value="",
             scroll_exit=True,
         )
         self.force = self.add_widget_intelligent(
             npyscreen.Checkbox,
-            name="Force merge of incompatible models",
+            name="Force merge of models created by different diffusers library versions",
             labelColor="CONTROL",
-            value=False,
+            value=True,
             scroll_exit=True,
         )
+        self.nextrely += 1
         self.merge_method = self.add_widget_intelligent(
             npyscreen.TitleSelectOne,
             name="Merge Method:",
@@ -341,7 +279,8 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
             interp = self.interpolations[self.merge_method.value[0]]
 
         args = dict(
-            models=models,
+            model_names=models,
+            base_model=tuple(BaseModelType)[self.base_select.value[0]],
             alpha=self.alpha.value,
             interp=interp,
             force=self.force.value,
@@ -379,21 +318,30 @@ class mergeModelsForm(npyscreen.FormMultiPageAction):
         else:
             return True
 
-    def get_model_names(self) -> List[str]:
+    def get_model_names(self, base_model: BaseModelType=None) -> List[str]:
         model_names = [
-            name
-            for name in self.model_manager.model_names()
-            if self.model_manager.model_info(name).get("format") == "diffusers"
+            info["name"]
+            for info in self.model_manager.list_models(model_type=ModelType.Main, base_model=base_model)
+            if info["model_format"] == "diffusers"
         ]
         return sorted(model_names)
 
+    def _populate_models(self,value=None):
+        base_model = tuple(BaseModelType)[value[0]]
+        self.model_names = self.get_model_names(base_model)
+        
+        models_plus_none = self.model_names.copy()
+        models_plus_none.insert(0, "None")
+        self.model1.values = self.model_names
+        self.model2.values = self.model_names
+        self.model3.values = models_plus_none
+                  
+        self.display()
+
 class Mergeapp(npyscreen.NPSAppManaged):
-    def __init__(self):
+    def __init__(self, model_manager:ModelManager):
         super().__init__()
-        conf = OmegaConf.load(config.model_conf_path)
-        self.model_manager = ModelManager(
-            conf, "cpu", "float16"
-        )  # precision doesn't really matter here
+        self.model_manager = model_manager
 
     def onStart(self):
         npyscreen.setTheme(npyscreen.Themes.ElegantTheme)
@@ -401,44 +349,41 @@ class Mergeapp(npyscreen.NPSAppManaged):
 
 
 def run_gui(args: Namespace):
-    mergeapp = Mergeapp()
+    model_manager = ModelManager(config.model_conf_path)
+    mergeapp = Mergeapp(model_manager)
     mergeapp.run()
 
     args = mergeapp.merge_arguments
-    merge_diffusion_models_and_commit(**args)
+    merger = ModelMerger(model_manager)
+    merger.merge_diffusion_models_and_save(**args)
     logger.info(f'Models merged into new model: "{args["merged_model_name"]}".')
 
 
 def run_cli(args: Namespace):
     assert args.alpha >= 0 and args.alpha <= 1.0, "alpha must be between 0 and 1"
     assert (
-        args.models and len(args.models) >= 1 and len(args.models) <= 3
+        args.model_names and len(args.model_names) >= 1 and len(args.model_names) <= 3
     ), "Please provide the --models argument to list 2 to 3 models to merge. Use --help for full usage."
 
     if not args.merged_model_name:
-        args.merged_model_name = "+".join(args.models)
+        args.merged_model_name = "+".join(args.model_names)
         logger.info(
             f'No --merged_model_name provided. Defaulting to "{args.merged_model_name}"'
         )
 
-        model_manager = ModelManager(OmegaConf.load(config.model_conf_path))
-        assert (
-            args.clobber or args.merged_model_name not in model_manager.model_names()
-        ), f'A model named "{args.merged_model_name}" already exists. Use --clobber to overwrite.'
+    model_manager = ModelManager(config.model_conf_path)
+    assert (
+        not model_manager.model_exists(args.merged_model_name, args.base_model, ModelType.Main) or args.clobber
+    ), f'A model named "{args.merged_model_name}" already exists. Use --clobber to overwrite.'
 
-        merge_diffusion_models_and_commit(**vars(args))
-        logger.info(f'Models merged into new model: "{args.merged_model_name}".')
+    merger = ModelMerger(model_manager)
+    merger.merge_diffusion_models_and_save(**vars(args))
+    logger.info(f'Models merged into new model: "{args.merged_model_name}".')
 
 
 def main():
     args = _parse_args()
-    config.root = args.root_dir
-
-    cache_dir = config.cache_dir
-    os.environ[
-        "HF_HOME"
-    ] = cache_dir  # because not clear the merge pipeline is honoring cache_dir
-    args.cache_dir = cache_dir
+    config.parse_args(['--root',str(args.root_dir)])
 
     try:
         if args.front_end:

invokeai/frontend/web/.eslintrc.js

@@ -36,6 +36,12 @@ module.exports = {
     ],
     'prettier/prettier': ['error', { endOfLine: 'auto' }],
     '@typescript-eslint/ban-ts-comment': 'warn',
+    '@typescript-eslint/no-empty-interface': [
+      'error',
+      {
+        allowSingleExtends: true,
+      },
+    ],
   },
   settings: {
     react: {

invokeai/frontend/web/dist/index.html

@@ -12,7 +12,7 @@
         margin: 0;
       }
     </style>
-    <script type="module" crossorigin src="./assets/index-4dfaefdd.js"></script>
+    <script type="module" crossorigin src="./assets/index-f05723f9.js"></script>
   </head>
 
   <body dir="ltr">

invokeai/frontend/web/dist/locales/en.json

@@ -52,6 +52,8 @@
         "unifiedCanvas": "Unified Canvas",
         "linear": "Linear",
         "nodes": "Node Editor",
+        "batch": "Batch Manager",
+        "modelManager": "Model Manager",
         "postprocessing": "Post Processing",
         "nodesDesc": "A node based system for the generation of images is under development currently. Stay tuned for updates about this amazing feature.",
         "postProcessing": "Post Processing",
@@ -333,6 +335,7 @@
     "modelManager": {
         "modelManager": "Model Manager",
         "model": "Model",
+        "vae": "VAE",
         "allModels": "All Models",
         "checkpointModels": "Checkpoints",
         "diffusersModels": "Diffusers",
@@ -348,6 +351,7 @@
         "scanForModels": "Scan For Models",
         "addManually": "Add Manually",
         "manual": "Manual",
+        "baseModel": "Base Model",
         "name": "Name",
         "nameValidationMsg": "Enter a name for your model",
         "description": "Description",
@@ -360,6 +364,7 @@
         "repoIDValidationMsg": "Online repository of your model",
         "vaeLocation": "VAE Location",
         "vaeLocationValidationMsg": "Path to where your VAE is located.",
+        "variant": "Variant",
         "vaeRepoID": "VAE Repo ID",
         "vaeRepoIDValidationMsg": "Online repository of your VAE",
         "width": "Width",
@@ -522,7 +527,8 @@
         "showOptionsPanel": "Show Options Panel",
         "hidePreview": "Hide Preview",
         "showPreview": "Show Preview",
-        "controlNetControlMode": "Control Mode"
+        "controlNetControlMode": "Control Mode",
+        "clipSkip": "Clip Skip"
     },
     "settings": {
         "models": "Models",
@@ -546,7 +552,8 @@
         "generation": "Generation",
         "ui": "User Interface",
         "favoriteSchedulers": "Favorite Schedulers",
-        "favoriteSchedulersPlaceholder": "No schedulers favorited"
+        "favoriteSchedulersPlaceholder": "No schedulers favorited",
+        "showAdvancedOptions": "Show Advanced Options"
     },
     "toast": {
         "serverError": "Server Error",

invokeai/frontend/web/dist/locales/fi.json

@@ -0,0 +1,122 @@
+{
+    "accessibility": {
+        "reset": "Resetoi",
+        "useThisParameter": "Käytä tätä parametria",
+        "modelSelect": "Mallin Valinta",
+        "exitViewer": "Poistu katselimesta",
+        "uploadImage": "Lataa kuva",
+        "copyMetadataJson": "Kopioi metadata JSON:iin",
+        "invokeProgressBar": "Invoken edistymispalkki",
+        "nextImage": "Seuraava kuva",
+        "previousImage": "Edellinen kuva",
+        "zoomIn": "Lähennä",
+        "flipHorizontally": "Käännä vaakasuoraan",
+        "zoomOut": "Loitonna",
+        "rotateCounterClockwise": "Kierrä vastapäivään",
+        "rotateClockwise": "Kierrä myötäpäivään",
+        "flipVertically": "Käännä pystysuoraan",
+        "showGallery": "Näytä galleria",
+        "modifyConfig": "Muokkaa konfiguraatiota",
+        "toggleAutoscroll": "Kytke automaattinen vieritys",
+        "toggleLogViewer": "Kytke lokin katselutila",
+        "showOptionsPanel": "Näytä asetukset"
+    },
+    "common": {
+        "postProcessDesc2": "Erillinen käyttöliittymä tullaan julkaisemaan helpottaaksemme työnkulkua jälkikäsittelyssä.",
+        "training": "Kouluta",
+        "statusLoadingModel": "Ladataan mallia",
+        "statusModelChanged": "Malli vaihdettu",
+        "statusConvertingModel": "Muunnetaan mallia",
+        "statusModelConverted": "Malli muunnettu",
+        "langFrench": "Ranska",
+        "langItalian": "Italia",
+        "languagePickerLabel": "Kielen valinta",
+        "hotkeysLabel": "Pikanäppäimet",
+        "reportBugLabel": "Raportoi Bugista",
+        "langPolish": "Puola",
+        "themeLabel": "Teema",
+        "langDutch": "Hollanti",
+        "settingsLabel": "Asetukset",
+        "githubLabel": "Github",
+        "darkTheme": "Tumma",
+        "lightTheme": "Vaalea",
+        "greenTheme": "Vihreä",
+        "langGerman": "Saksa",
+        "langPortuguese": "Portugali",
+        "discordLabel": "Discord",
+        "langEnglish": "Englanti",
+        "oceanTheme": "Meren sininen",
+        "langRussian": "Venäjä",
+        "langUkranian": "Ukraina",
+        "langSpanish": "Espanja",
+        "upload": "Lataa",
+        "statusMergedModels": "Mallit yhdistelty",
+        "img2img": "Kuva kuvaksi",
+        "nodes": "Solmut",
+        "nodesDesc": "Solmupohjainen järjestelmä kuvien generoimiseen on parhaillaan kehitteillä. Pysy kuulolla päivityksistä tähän uskomattomaan ominaisuuteen liittyen.",
+        "postProcessDesc1": "Invoke AI tarjoaa monenlaisia jälkikäsittelyominaisuukisa. Kuvan laadun skaalaus sekä kasvojen korjaus ovat jo saatavilla WebUI:ssä. Voit ottaa ne käyttöön lisäasetusten valikosta teksti kuvaksi sekä kuva kuvaksi -välilehdiltä. Voit myös suoraan prosessoida kuvia käyttämällä kuvan toimintapainikkeita nykyisen kuvan yläpuolella tai tarkastelussa.",
+        "postprocessing": "Jälkikäsitellään",
+        "postProcessing": "Jälkikäsitellään",
+        "cancel": "Peruuta",
+        "close": "Sulje",
+        "accept": "Hyväksy",
+        "statusConnected": "Yhdistetty",
+        "statusError": "Virhe",
+        "statusProcessingComplete": "Prosessointi valmis",
+        "load": "Lataa",
+        "back": "Takaisin",
+        "statusGeneratingTextToImage": "Generoidaan tekstiä kuvaksi",
+        "trainingDesc2": "InvokeAI tukee jo mukautettujen upotusten kouluttamista tekstin inversiolla käyttäen pääskriptiä.",
+        "statusDisconnected": "Yhteys katkaistu",
+        "statusPreparing": "Valmistellaan",
+        "statusIterationComplete": "Iteraatio valmis",
+        "statusMergingModels": "Yhdistellään malleja",
+        "statusProcessingCanceled": "Valmistelu peruutettu",
+        "statusSavingImage": "Tallennetaan kuvaa",
+        "statusGeneratingImageToImage": "Generoidaan kuvaa kuvaksi",
+        "statusRestoringFacesGFPGAN": "Korjataan kasvoja (GFPGAN)",
+        "statusRestoringFacesCodeFormer": "Korjataan kasvoja (CodeFormer)",
+        "statusGeneratingInpainting": "Generoidaan sisällemaalausta",
+        "statusGeneratingOutpainting": "Generoidaan ulosmaalausta",
+        "statusRestoringFaces": "Korjataan kasvoja",
+        "pinOptionsPanel": "Kiinnitä asetukset -paneeli",
+        "loadingInvokeAI": "Ladataan Invoke AI:ta",
+        "loading": "Ladataan",
+        "statusGenerating": "Generoidaan",
+        "txt2img": "Teksti kuvaksi",
+        "trainingDesc1": "Erillinen työnkulku omien upotusten ja tarkastuspisteiden kouluttamiseksi käyttäen tekstin inversiota ja dreamboothia selaimen käyttöliittymässä.",
+        "postProcessDesc3": "Invoke AI:n komentorivi tarjoaa paljon muita ominaisuuksia, kuten esimerkiksi Embiggenin.",
+        "unifiedCanvas": "Yhdistetty kanvas",
+        "statusGenerationComplete": "Generointi valmis"
+    },
+    "gallery": {
+        "uploads": "Lataukset",
+        "showUploads": "Näytä lataukset",
+        "galleryImageResetSize": "Resetoi koko",
+        "maintainAspectRatio": "Säilytä kuvasuhde",
+        "galleryImageSize": "Kuvan koko",
+        "pinGallery": "Kiinnitä galleria",
+        "showGenerations": "Näytä generaatiot",
+        "singleColumnLayout": "Yhden sarakkeen asettelu",
+        "generations": "Generoinnit",
+        "gallerySettings": "Gallerian asetukset",
+        "autoSwitchNewImages": "Vaihda uusiin kuviin automaattisesti",
+        "allImagesLoaded": "Kaikki kuvat ladattu",
+        "noImagesInGallery": "Ei kuvia galleriassa",
+        "loadMore": "Lataa lisää"
+    },
+    "hotkeys": {
+        "keyboardShortcuts": "näppäimistön pikavalinnat",
+        "appHotkeys": "Sovelluksen pikanäppäimet",
+        "generalHotkeys": "Yleiset pikanäppäimet",
+        "galleryHotkeys": "Gallerian pikanäppäimet",
+        "unifiedCanvasHotkeys": "Yhdistetyn kanvaan pikanäppäimet",
+        "cancel": {
+            "desc": "Peruuta kuvan luominen",
+            "title": "Peruuta"
+        },
+        "invoke": {
+            "desc": "Luo kuva"
+        }
+    }
+}

invokeai/frontend/web/dist/locales/mn.json

@@ -0,0 +1 @@
+{}

invokeai/frontend/web/dist/locales/sv.json

@@ -0,0 +1,254 @@
+{
+    "accessibility": {
+        "copyMetadataJson": "Kopiera metadata JSON",
+        "zoomIn": "Zooma in",
+        "exitViewer": "Avslutningsvisare",
+        "modelSelect": "Välj modell",
+        "uploadImage": "Ladda upp bild",
+        "invokeProgressBar": "Invoke förloppsmätare",
+        "nextImage": "Nästa bild",
+        "toggleAutoscroll": "Växla automatisk rullning",
+        "flipHorizontally": "Vänd vågrätt",
+        "flipVertically": "Vänd lodrätt",
+        "zoomOut": "Zooma ut",
+        "toggleLogViewer": "Växla logvisare",
+        "reset": "Starta om",
+        "previousImage": "Föregående bild",
+        "useThisParameter": "Använd denna parametern",
+        "showGallery": "Visa galleri",
+        "rotateCounterClockwise": "Rotera moturs",
+        "rotateClockwise": "Rotera medurs",
+        "modifyConfig": "Ändra konfiguration",
+        "showOptionsPanel": "Visa inställningspanelen"
+    },
+    "common": {
+        "hotkeysLabel": "Snabbtangenter",
+        "reportBugLabel": "Rapportera bugg",
+        "githubLabel": "Github",
+        "discordLabel": "Discord",
+        "settingsLabel": "Inställningar",
+        "darkTheme": "Mörk",
+        "lightTheme": "Ljus",
+        "greenTheme": "Grön",
+        "oceanTheme": "Hav",
+        "langEnglish": "Engelska",
+        "langDutch": "Nederländska",
+        "langFrench": "Franska",
+        "langGerman": "Tyska",
+        "langItalian": "Italienska",
+        "langArabic": "العربية",
+        "langHebrew": "עברית",
+        "langPolish": "Polski",
+        "langPortuguese": "Português",
+        "langBrPortuguese": "Português do Brasil",
+        "langSimplifiedChinese": "简体中文",
+        "langJapanese": "日本語",
+        "langKorean": "한국어",
+        "langRussian": "Русский",
+        "unifiedCanvas": "Förenad kanvas",
+        "nodesDesc": "Ett nodbaserat system för bildgenerering är under utveckling. Håll utkik för uppdateringar om denna fantastiska funktion.",
+        "langUkranian": "Украї́нська",
+        "langSpanish": "Español",
+        "postProcessDesc2": "Ett dedikerat användargränssnitt kommer snart att släppas för att underlätta mer avancerade arbetsflöden av efterbehandling.",
+        "trainingDesc1": "Ett dedikerat arbetsflöde för träning av dina egna inbäddningar och kontrollpunkter genom Textual Inversion eller Dreambooth från webbgränssnittet.",
+        "trainingDesc2": "InvokeAI stöder redan träning av anpassade inbäddningar med hjälp av Textual Inversion genom huvudscriptet.",
+        "upload": "Ladda upp",
+        "close": "Stäng",
+        "cancel": "Avbryt",
+        "accept": "Acceptera",
+        "statusDisconnected": "Frånkopplad",
+        "statusGeneratingTextToImage": "Genererar text till bild",
+        "statusGeneratingImageToImage": "Genererar Bild till bild",
+        "statusGeneratingInpainting": "Genererar Måla i",
+        "statusGenerationComplete": "Generering klar",
+        "statusModelConverted": "Modell konverterad",
+        "statusMergingModels": "Sammanfogar modeller",
+        "pinOptionsPanel": "Nåla fast inställningspanelen",
+        "loading": "Laddar",
+        "loadingInvokeAI": "Laddar Invoke AI",
+        "statusRestoringFaces": "Återskapar ansikten",
+        "languagePickerLabel": "Språkväljare",
+        "themeLabel": "Tema",
+        "txt2img": "Text till bild",
+        "nodes": "Noder",
+        "img2img": "Bild till bild",
+        "postprocessing": "Efterbehandling",
+        "postProcessing": "Efterbehandling",
+        "load": "Ladda",
+        "training": "Träning",
+        "postProcessDesc1": "Invoke AI erbjuder ett brett utbud av efterbehandlingsfunktioner. Uppskalning och ansiktsåterställning finns redan tillgängligt i webbgränssnittet. Du kommer åt dem ifrån Avancerade inställningar-menyn under Bild till bild-fliken. Du kan också behandla bilder direkt genom att använda knappen bildåtgärder ovanför nuvarande bild eller i bildvisaren.",
+        "postProcessDesc3": "Invoke AI's kommandotolk erbjuder många olika funktioner, bland annat \"Förstora\".",
+        "statusGenerating": "Genererar",
+        "statusError": "Fel",
+        "back": "Bakåt",
+        "statusConnected": "Ansluten",
+        "statusPreparing": "Förbereder",
+        "statusProcessingCanceled": "Bearbetning avbruten",
+        "statusProcessingComplete": "Bearbetning färdig",
+        "statusGeneratingOutpainting": "Genererar Fyll ut",
+        "statusIterationComplete": "Itterering klar",
+        "statusSavingImage": "Sparar bild",
+        "statusRestoringFacesGFPGAN": "Återskapar ansikten (GFPGAN)",
+        "statusRestoringFacesCodeFormer": "Återskapar ansikten (CodeFormer)",
+        "statusUpscaling": "Skala upp",
+        "statusUpscalingESRGAN": "Uppskalning (ESRGAN)",
+        "statusModelChanged": "Modell ändrad",
+        "statusLoadingModel": "Laddar modell",
+        "statusConvertingModel": "Konverterar modell",
+        "statusMergedModels": "Modeller sammanfogade"
+    },
+    "gallery": {
+        "generations": "Generationer",
+        "showGenerations": "Visa generationer",
+        "uploads": "Uppladdningar",
+        "showUploads": "Visa uppladdningar",
+        "galleryImageSize": "Bildstorlek",
+        "allImagesLoaded": "Alla bilder laddade",
+        "loadMore": "Ladda mer",
+        "galleryImageResetSize": "Återställ storlek",
+        "gallerySettings": "Galleriinställningar",
+        "maintainAspectRatio": "Behåll bildförhållande",
+        "pinGallery": "Nåla fast galleri",
+        "noImagesInGallery": "Inga bilder i galleriet",
+        "autoSwitchNewImages": "Ändra automatiskt till nya bilder",
+        "singleColumnLayout": "Enkolumnslayout"
+    },
+    "hotkeys": {
+        "generalHotkeys": "Allmänna snabbtangenter",
+        "galleryHotkeys": "Gallerisnabbtangenter",
+        "unifiedCanvasHotkeys": "Snabbtangenter för sammanslagskanvas",
+        "invoke": {
+            "title": "Anropa",
+            "desc": "Genererar en bild"
+        },
+        "cancel": {
+            "title": "Avbryt",
+            "desc": "Avbryt bildgenerering"
+        },
+        "focusPrompt": {
+            "desc": "Fokusera området för promptinmatning",
+            "title": "Fokusprompt"
+        },
+        "pinOptions": {
+            "desc": "Nåla fast alternativpanelen",
+            "title": "Nåla fast alternativ"
+        },
+        "toggleOptions": {
+            "title": "Växla inställningar",
+            "desc": "Öppna och stäng alternativpanelen"
+        },
+        "toggleViewer": {
+            "title": "Växla visaren",
+            "desc": "Öppna och stäng bildvisaren"
+        },
+        "toggleGallery": {
+            "title": "Växla galleri",
+            "desc": "Öppna eller stäng galleribyrån"
+        },
+        "maximizeWorkSpace": {
+            "title": "Maximera arbetsyta",
+            "desc": "Stäng paneler och maximera arbetsyta"
+        },
+        "changeTabs": {
+            "title": "Växla flik",
+            "desc": "Byt till en annan arbetsyta"
+        },
+        "consoleToggle": {
+            "title": "Växla konsol",
+            "desc": "Öppna och stäng konsol"
+        },
+        "setSeed": {
+            "desc": "Använd seed för nuvarande bild",
+            "title": "välj seed"
+        },
+        "setParameters": {
+            "title": "Välj parametrar",
+            "desc": "Använd alla parametrar från nuvarande bild"
+        },
+        "setPrompt": {
+            "desc": "Använd prompt för nuvarande bild",
+            "title": "Välj prompt"
+        },
+        "restoreFaces": {
+            "title": "Återskapa ansikten",
+            "desc": "Återskapa nuvarande bild"
+        },
+        "upscale": {
+            "title": "Skala upp",
+            "desc": "Skala upp nuvarande bild"
+        },
+        "showInfo": {
+            "title": "Visa info",
+            "desc": "Visa metadata för nuvarande bild"
+        },
+        "sendToImageToImage": {
+            "title": "Skicka till Bild till bild",
+            "desc": "Skicka nuvarande bild till Bild till bild"
+        },
+        "deleteImage": {
+            "title": "Radera bild",
+            "desc": "Radera nuvarande bild"
+        },
+        "closePanels": {
+            "title": "Stäng paneler",
+            "desc": "Stäng öppna paneler"
+        },
+        "previousImage": {
+            "title": "Föregående bild",
+            "desc": "Visa föregående bild"
+        },
+        "nextImage": {
+            "title": "Nästa bild",
+            "desc": "Visa nästa bild"
+        },
+        "toggleGalleryPin": {
+            "title": "Växla gallerinål",
+            "desc": "Nålar fast eller nålar av galleriet i gränssnittet"
+        },
+        "increaseGalleryThumbSize": {
+            "title": "Förstora galleriets bildstorlek",
+            "desc": "Förstora miniatyrbildernas storlek"
+        },
+        "decreaseGalleryThumbSize": {
+            "title": "Minska gelleriets bildstorlek",
+            "desc": "Minska miniatyrbildernas storlek i galleriet"
+        },
+        "decreaseBrushSize": {
+            "desc": "Förminska storleken på kanvas- pensel eller suddgummi",
+            "title": "Minska penselstorlek"
+        },
+        "increaseBrushSize": {
+            "title": "Öka penselstorlek",
+            "desc": "Öka stoleken på kanvas- pensel eller suddgummi"
+        },
+        "increaseBrushOpacity": {
+            "title": "Öka penselns opacitet",
+            "desc": "Öka opaciteten för kanvaspensel"
+        },
+        "decreaseBrushOpacity": {
+            "desc": "Minska kanvaspenselns opacitet",
+            "title": "Minska penselns opacitet"
+        },
+        "moveTool": {
+            "title": "Flytta",
+            "desc": "Tillåt kanvasnavigation"
+        },
+        "fillBoundingBox": {
+            "title": "Fyll ram",
+            "desc": "Fyller ramen med pensels färg"
+        },
+        "keyboardShortcuts": "Snabbtangenter",
+        "appHotkeys": "Appsnabbtangenter",
+        "selectBrush": {
+            "desc": "Välj kanvaspensel",
+            "title": "Välj pensel"
+        },
+        "selectEraser": {
+            "desc": "Välj kanvassuddgummi",
+            "title": "Välj suddgummi"
+        },
+        "eraseBoundingBox": {
+            "title": "Ta bort ram"
+        }
+    }
+}

invokeai/frontend/web/dist/locales/tr.json

@@ -0,0 +1,64 @@
+{
+    "accessibility": {
+        "invokeProgressBar": "Invoke ilerleme durumu",
+        "nextImage": "Sonraki Resim",
+        "useThisParameter": "Kullanıcı parametreleri",
+        "copyMetadataJson": "Metadata verilerini kopyala (JSON)",
+        "exitViewer": "Görüntüleme Modundan Çık",
+        "zoomIn": "Yakınlaştır",
+        "zoomOut": "Uzaklaştır",
+        "rotateCounterClockwise": "Döndür (Saat yönünün tersine)",
+        "rotateClockwise": "Döndür (Saat yönünde)",
+        "flipHorizontally": "Yatay Çevir",
+        "flipVertically": "Dikey Çevir",
+        "modifyConfig": "Ayarları Değiştir",
+        "toggleAutoscroll": "Otomatik kaydırmayı aç/kapat",
+        "toggleLogViewer": "Günlük Görüntüleyici Aç/Kapa",
+        "showOptionsPanel": "Ayarlar Panelini Göster",
+        "modelSelect": "Model Seçin",
+        "reset": "Sıfırla",
+        "uploadImage": "Resim Yükle",
+        "previousImage": "Önceki Resim",
+        "menu": "Menü",
+        "showGallery": "Galeriyi Göster"
+    },
+    "common": {
+        "hotkeysLabel": "Kısayol Tuşları",
+        "themeLabel": "Tema",
+        "languagePickerLabel": "Dil Seçimi",
+        "reportBugLabel": "Hata Bildir",
+        "githubLabel": "Github",
+        "discordLabel": "Discord",
+        "settingsLabel": "Ayarlar",
+        "darkTheme": "Karanlık Tema",
+        "lightTheme": "Aydınlık Tema",
+        "greenTheme": "Yeşil Tema",
+        "oceanTheme": "Okyanus Tema",
+        "langArabic": "Arapça",
+        "langEnglish": "İngilizce",
+        "langDutch": "Hollandaca",
+        "langFrench": "Fransızca",
+        "langGerman": "Almanca",
+        "langItalian": "İtalyanca",
+        "langJapanese": "Japonca",
+        "langPolish": "Lehçe",
+        "langPortuguese": "Portekizce",
+        "langBrPortuguese": "Portekizcr (Brezilya)",
+        "langRussian": "Rusça",
+        "langSimplifiedChinese": "Çince (Basit)",
+        "langUkranian": "Ukraynaca",
+        "langSpanish": "İspanyolca",
+        "txt2img": "Metinden Resime",
+        "img2img": "Resimden Metine",
+        "linear": "Çizgisel",
+        "nodes": "Düğümler",
+        "postprocessing": "İşlem Sonrası",
+        "postProcessing": "İşlem Sonrası",
+        "postProcessDesc2": "Daha gelişmiş özellikler için ve iş akışını kolaylaştırmak için özel bir kullanıcı arayüzü çok yakında yayınlanacaktır.",
+        "postProcessDesc3": "Invoke AI komut satırı arayüzü, bir çok yeni özellik sunmaktadır.",
+        "langKorean": "Korece",
+        "unifiedCanvas": "Akıllı Tuval",
+        "nodesDesc": "Görüntülerin oluşturulmasında hazırladığımız yeni bir sistem geliştirme aşamasındadır. Bu harika özellikler ve çok daha fazlası için bizi takip etmeye devam edin.",
+        "postProcessDesc1": "Invoke AI son kullanıcıya yönelik bir çok özellik sunar. Görüntü kalitesi yükseltme, yüz restorasyonu WebUI üzerinden kullanılabilir. Metinden resime ve resimden metne araçlarına gelişmiş seçenekler menüsünden ulaşabilirsiniz. İsterseniz mevcut görüntü ekranının üzerindeki veya görüntüleyicideki görüntüyü doğrudan düzenleyebilirsiniz."
+    }
+}

invokeai/frontend/web/dist/locales/vi.json

@@ -0,0 +1 @@
+{}

invokeai/frontend/web/package.json

@@ -23,7 +23,7 @@
     "dev": "concurrently \"vite dev\" \"yarn run theme:watch\"",
     "dev:host": "concurrently \"vite dev --host\" \"yarn run theme:watch\"",
     "build": "yarn run lint && vite build",
-    "typegen": "npx openapi-typescript http://localhost:9090/openapi.json --output src/services/api/schema.d.ts -t",
+    "typegen": "npx ts-node scripts/typegen.ts",
     "preview": "vite preview",
     "lint:madge": "madge --circular src/main.tsx",
     "lint:eslint": "eslint --max-warnings=0 .",
@@ -53,7 +53,6 @@
     ]
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
     "@chakra-ui/anatomy": "^2.1.1",
     "@chakra-ui/icons": "^2.0.19",
     "@chakra-ui/react": "^2.7.1",
@@ -68,6 +67,7 @@
     "@fontsource-variable/inter": "^5.0.3",
     "@fontsource/inter": "^5.0.3",
     "@mantine/core": "^6.0.14",
+    "@mantine/form": "^6.0.15",
     "@mantine/hooks": "^6.0.14",
     "@reduxjs/toolkit": "^1.9.5",
     "@roarr/browser-log-writer": "^1.1.5",
@@ -83,7 +83,7 @@
     "konva": "^9.2.0",
     "lodash-es": "^4.17.21",
     "nanostores": "^0.9.2",
-    "openapi-fetch": "^0.4.0",
+    "openapi-fetch": "^0.6.1",
     "overlayscrollbars": "^2.2.0",
     "overlayscrollbars-react": "^0.5.0",
     "patch-package": "^7.0.0",
@@ -155,7 +155,6 @@
     "vite-plugin-css-injected-by-js": "^3.1.1",
     "vite-plugin-dts": "^2.3.0",
     "vite-plugin-eslint": "^1.8.1",
-    "vite-plugin-node-polyfills": "^0.9.0",
     "vite-tsconfig-paths": "^4.2.0",
     "yarn": "^1.22.19"
   }

invokeai/frontend/web/patches/openapi-fetch+0.4.0.patch

@@ -1,55 +0,0 @@
-diff --git a/node_modules/openapi-fetch/dist/index.js b/node_modules/openapi-fetch/dist/index.js
-index cd4528a..8976b51 100644
---- a/node_modules/openapi-fetch/dist/index.js
-+++ b/node_modules/openapi-fetch/dist/index.js
-@@ -1,5 +1,5 @@
- // settings & const
--const DEFAULT_HEADERS = {
-+const CONTENT_TYPE_APPLICATION_JSON = {
-     "Content-Type": "application/json",
- };
- const TRAILING_SLASH_RE = /\/*$/;
-@@ -29,18 +29,29 @@ export function createFinalURL(url, options) {
-     }
-     return finalURL;
- }
-+function stringifyBody(body) {
-+    if (body instanceof ArrayBuffer || body instanceof File || body instanceof DataView || body instanceof Blob || ArrayBuffer.isView(body) || body instanceof URLSearchParams || body instanceof FormData) {
-+      return;
-+    }
-+  
-+    if (typeof body === "string") {
-+      return body;
-+    }
-+  
-+    return JSON.stringify(body);
-+  }
-+  
- export default function createClient(clientOptions = {}) {
-     const { fetch = globalThis.fetch, ...options } = clientOptions;
--    const defaultHeaders = new Headers({
--        ...DEFAULT_HEADERS,
--        ...(options.headers ?? {}),
--    });
-+    const defaultHeaders = new Headers(options.headers ?? {});
-     async function coreFetch(url, fetchOptions) {
-         const { headers, body: requestBody, params = {}, parseAs = "json", querySerializer = defaultSerializer, ...init } = fetchOptions || {};
-         // URL
-         const finalURL = createFinalURL(url, { baseUrl: options.baseUrl, params, querySerializer });
-+        // Stringify body if needed
-+        const stringifiedBody = stringifyBody(requestBody);
-         // headers
--        const baseHeaders = new Headers(defaultHeaders); // clone defaults (don’t overwrite!)
-+        const baseHeaders = new Headers(stringifiedBody ? { ...CONTENT_TYPE_APPLICATION_JSON, ...defaultHeaders } : defaultHeaders); // clone defaults (don’t overwrite!)
-         const headerOverrides = new Headers(headers);
-         for (const [k, v] of headerOverrides.entries()) {
-             if (v === undefined || v === null)
-@@ -54,7 +65,7 @@ export default function createClient(clientOptions = {}) {
-             ...options,
-             ...init,
-             headers: baseHeaders,
--            body: typeof requestBody === "string" ? requestBody : JSON.stringify(requestBody),
-+            body: stringifiedBody ?? requestBody,
-         });
-         // handle empty content
-         // note: we return `{}` because we want user truthy checks for `.data` or `.error` to succeed

invokeai/frontend/web/public/locales/en.json

@@ -52,6 +52,8 @@
         "unifiedCanvas": "Unified Canvas",
         "linear": "Linear",
         "nodes": "Node Editor",
+        "batch": "Batch Manager",
+        "modelManager": "Model Manager",
         "postprocessing": "Post Processing",
         "nodesDesc": "A node based system for the generation of images is under development currently. Stay tuned for updates about this amazing feature.",
         "postProcessing": "Post Processing",
@@ -333,6 +335,7 @@
     "modelManager": {
         "modelManager": "Model Manager",
         "model": "Model",
+        "vae": "VAE",
         "allModels": "All Models",
         "checkpointModels": "Checkpoints",
         "diffusersModels": "Diffusers",
@@ -348,6 +351,7 @@
         "scanForModels": "Scan For Models",
         "addManually": "Add Manually",
         "manual": "Manual",
+        "baseModel": "Base Model",
         "name": "Name",
         "nameValidationMsg": "Enter a name for your model",
         "description": "Description",
@@ -360,6 +364,7 @@
         "repoIDValidationMsg": "Online repository of your model",
         "vaeLocation": "VAE Location",
         "vaeLocationValidationMsg": "Path to where your VAE is located.",
+        "variant": "Variant",
         "vaeRepoID": "VAE Repo ID",
         "vaeRepoIDValidationMsg": "Online repository of your VAE",
         "width": "Width",
@@ -522,7 +527,8 @@
         "showOptionsPanel": "Show Options Panel",
         "hidePreview": "Hide Preview",
         "showPreview": "Show Preview",
-        "controlNetControlMode": "Control Mode"
+        "controlNetControlMode": "Control Mode",
+        "clipSkip": "Clip Skip"
     },
     "settings": {
         "models": "Models",
@@ -546,7 +552,8 @@
         "generation": "Generation",
         "ui": "User Interface",
         "favoriteSchedulers": "Favorite Schedulers",
-        "favoriteSchedulersPlaceholder": "No schedulers favorited"
+        "favoriteSchedulersPlaceholder": "No schedulers favorited",
+        "showAdvancedOptions": "Show Advanced Options"
     },
     "toast": {
         "serverError": "Server Error",

invokeai/frontend/web/scripts/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

invokeai/frontend/web/scripts/typegen.ts

@@ -0,0 +1,23 @@
+import fs from 'node:fs';
+import openapiTS from 'openapi-typescript';
+
+const OPENAPI_URL = 'http://localhost:9090/openapi.json';
+const OUTPUT_FILE = 'src/services/api/schema.d.ts';
+
+async function main() {
+  process.stdout.write(
+    `Generating types "${OPENAPI_URL}" --> "${OUTPUT_FILE}"...`
+  );
+  const types = await openapiTS(OPENAPI_URL, {
+    exportType: true,
+    transform: (schemaObject, metadata) => {
+      if ('format' in schemaObject && schemaObject.format === 'binary') {
+        return schemaObject.nullable ? 'Blob | null' : 'Blob';
+      }
+    },
+  });
+  fs.writeFileSync(OUTPUT_FILE, types);
+  process.stdout.write(` OK!\r\n`);
+}
+
+main();