add button to set default settings in parameters

## What type of PR is this? (check all applicable)

- [ ] Refactor
- [X] Feature
- [ ] Bug Fix
- [ ] Optimization
- [ ] Documentation Update
- [ ] Community Node Submission


## Have you discussed this change with the InvokeAI team?
- [X] Yes
- [ ] No, because:

      
## Have you updated all relevant documentation?
- [ ] Yes
- [ ] No


## Description


## Related Tickets & Documents

<!--
For pull requests that relate or close an issue, please include them
below. 

For example having the text: "closes #1234" would connect the current
pull
request to issue 1234.  And when we merge the pull request, Github will
automatically close the issue.
-->

- Related Issue #
- Closes #

## QA Instructions, Screenshots, Recordings

<!-- 
Please provide steps on how to test changes, any hardware or 
software specifications as well as any other pertinent information. 
-->

## Merge Plan

<!--
A merge plan describes how this PR should be handled after it is
approved.

Example merge plans:
- "This PR can be merged when approved"
- "This must be squash-merged when approved"
- "DO NOT MERGE - I will rebase and tidy commits before merging"
- "#dev-chat on discord needs to be advised of this change when it is
merged"

A merge plan is particularly important for large PRs or PRs that touch
the
database in any way.
-->

## Added/updated tests?

- [ ] Yes
- [ ] No : _please replace this line with details on why tests
      have not been included_

## [optional] Are there any post deployment tasks we need to perform?

.github/actions/install-frontend-deps/action.yml

@@ -1,33 +0,0 @@
-name: install frontend dependencies
-description: Installs frontend dependencies with pnpm, with caching
-runs:
-  using: 'composite'
-  steps:
-    - name: setup node 18
-      uses: actions/setup-node@v4
-      with:
-        node-version: '18'
-
-    - name: setup pnpm
-      uses: pnpm/action-setup@v2
-      with:
-        version: 8
-        run_install: false
-
-    - name: get pnpm store directory
-      shell: bash
-      run: |
-        echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
-    - name: setup cache
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.STORE_PATH }}
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
-
-    - name: install frontend dependencies
-      run: pnpm install --prefer-frozen-lockfile
-      shell: bash
-      working-directory: invokeai/frontend/web

.github/pr_labels.yml

@@ -1,59 +1,59 @@
-root:
+Root:
 - changed-files:
   - any-glob-to-any-file: '*'
 
-python-deps:
+PythonDeps:
 - changed-files:
   - any-glob-to-any-file: 'pyproject.toml'
 
-python:
+Python:
 - changed-files:
   - all-globs-to-any-file: 
     - 'invokeai/**'
     - '!invokeai/frontend/web/**'
 
-python-tests:
+PythonTests:
 - changed-files:
   - any-glob-to-any-file: 'tests/**'
 
-ci-cd:
+CICD:
 - changed-files:
   - any-glob-to-any-file: .github/**
 
-docker:
+Docker:
 - changed-files:
   - any-glob-to-any-file: docker/**
 
-installer:
+Installer:
 - changed-files:
   - any-glob-to-any-file: installer/**
 
-docs:
+Documentation:
 - changed-files:
   - any-glob-to-any-file: docs/**
 
-invocations:
+Invocations:
 - changed-files:
   - any-glob-to-any-file: 'invokeai/app/invocations/**'
 
-backend:
+Backend:
 - changed-files:
   - any-glob-to-any-file: 'invokeai/backend/**'
 
-api:
+Api:
 - changed-files:
   - any-glob-to-any-file: 'invokeai/app/api/**'
 
-services:
+Services:
 - changed-files:
   - any-glob-to-any-file: 'invokeai/app/services/**'
 
-frontend-deps:
+FrontendDeps:
 - changed-files:
   - any-glob-to-any-file:
     - '**/*/package.json'
     - '**/*/pnpm-lock.yaml'
 
-frontend:
+Frontend:
 - changed-files:
   - any-glob-to-any-file: 'invokeai/frontend/web/**'

.github/workflows/build-container.yml

@@ -11,7 +11,7 @@ on:
       - 'docker/docker-entrypoint.sh'
       - 'workflows/build-container.yml'
     tags:
-      - 'v*.*.*'
+      - 'v*'
   workflow_dispatch:
 
 permissions:

.github/workflows/build-installer.yml

@@ -1,45 +0,0 @@
-# Builds and uploads the installer and python build artifacts.
-
-name: build installer
-
-on:
-  workflow_dispatch:
-  workflow_call:
-
-jobs:
-  build-installer:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5 # expected run time: <2 min
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-
-      - name: setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: pip
-          cache-dependency-path: pyproject.toml
-
-      - name: install pypa/build
-        run: pip install --upgrade build
-
-      - name: setup frontend
-        uses: ./.github/actions/install-frontend-deps
-
-      - name: create installer
-        id: create_installer
-        run: ./create_installer.sh
-        working-directory: installer
-
-      - name: upload python distribution artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: dist
-          path: ${{ steps.create_installer.outputs.DIST_PATH }}
-
-      - name: upload installer artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.create_installer.outputs.INSTALLER_FILENAME }}
-          path: ${{ steps.create_installer.outputs.INSTALLER_PATH }}

.github/workflows/frontend-checks.yml

@@ -1,68 +0,0 @@
-# Runs frontend code quality checks.
-#
-# Checks for changes to frontend files before running the checks.
-# When manually triggered or when called from another workflow, always runs the checks.
-
-name: 'frontend checks'
-
-on:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    types:
-      - 'ready_for_review'
-      - 'opened'
-      - 'synchronize'
-  merge_group:
-  workflow_dispatch:
-  workflow_call:
-
-defaults:
-  run:
-    working-directory: invokeai/frontend/web
-
-jobs:
-  frontend-checks:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10 # expected run time: <2 min
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: check for changed frontend files
-        if: ${{ github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' }}
-        id: changed-files
-        uses: tj-actions/changed-files@v42
-        with:
-          files_yaml: |
-            frontend:
-              - 'invokeai/frontend/web/**'
-
-      - name: install dependencies
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        uses: ./.github/actions/install-frontend-deps
-
-      - name: tsc
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm lint:tsc'
-        shell: bash
-
-      - name: dpdm
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm lint:dpdm'
-        shell: bash
-
-      - name: eslint
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm lint:eslint'
-        shell: bash
-
-      - name: prettier
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm lint:prettier'
-        shell: bash
-
-      - name: knip
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm lint:knip'
-        shell: bash

.github/workflows/frontend-tests.yml

@@ -1,48 +0,0 @@
-# Runs frontend tests.
-#
-# Checks for changes to frontend files before running the tests.
-# When manually triggered or called from another workflow, always runs the tests.
-
-name: 'frontend tests'
-
-on:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    types:
-      - 'ready_for_review'
-      - 'opened'
-      - 'synchronize'
-  merge_group:
-  workflow_dispatch:
-  workflow_call:
-
-defaults:
-  run:
-    working-directory: invokeai/frontend/web
-
-jobs:
-  frontend-tests:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10 # expected run time: <2 min
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: check for changed frontend files
-        if: ${{ github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' }}
-        id: changed-files
-        uses: tj-actions/changed-files@v42
-        with:
-          files_yaml: |
-            frontend:
-              - 'invokeai/frontend/web/**'
-
-      - name: install dependencies
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        uses: ./.github/actions/install-frontend-deps
-
-      - name: vitest
-        if: ${{ steps.changed-files.outputs.frontend_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: 'pnpm test:no-watch'
-        shell: bash

.github/workflows/label-pr.yml

@@ -1,6 +1,6 @@
-name: 'label PRs'
+name: "Pull Request Labeler"
 on:
-  - pull_request_target
+- pull_request_target
 
 jobs:
   labeler:
@@ -9,10 +9,8 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
+      - name: Checkout
         uses: actions/checkout@v4
-
-      - name: label PRs
-        uses: actions/labeler@v5
+      - uses: actions/labeler@v5
         with:
-          configuration-path: .github/pr_labels.yml
+          configuration-path: .github/pr_labels.yml

.github/workflows/lint-frontend.yml

@@ -0,0 +1,45 @@
+name: Lint frontend
+
+on:
+  pull_request:
+    types:
+      - 'ready_for_review'
+      - 'opened'
+      - 'synchronize'
+  push:
+    branches:
+      - 'main'
+  merge_group:
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: invokeai/frontend/web
+
+jobs:
+  lint-frontend:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Setup Node 18
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: '8.12.1'
+      - name: Install dependencies
+        run: 'pnpm install --prefer-frozen-lockfile'
+      - name: Typescript
+        run: 'pnpm run lint:tsc'
+      - name: Madge
+        run: 'pnpm run lint:dpdm'
+      - name: ESLint
+        run: 'pnpm run lint:eslint'
+      - name: Prettier
+        run: 'pnpm run lint:prettier'
+      - name: Knip
+        run: 'pnpm run lint:knip'

.github/workflows/mkdocs-material.yml

@@ -1,49 +1,51 @@
-# This is a mostly a copy-paste from https://github.com/squidfunk/mkdocs-material/blob/master/docs/publishing-your-site.md
-
-name: mkdocs
-
+name: mkdocs-material
 on:
   push:
     branches:
-      - main
-  workflow_dispatch:
+      - 'refs/heads/main'
 
 permissions:
-  contents: write
+    contents: write
 
 jobs:
-  deploy:
+  mkdocs-material:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     env:
       REPO_URL: '${{ github.server_url }}/${{ github.repository }}'
       REPO_NAME: '${{ github.repository }}'
       SITE_URL: 'https://${{ github.repository_owner }}.github.io/InvokeAI'
-
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
+      - name: checkout sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: setup python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
           cache: pip
           cache-dependency-path: pyproject.toml
 
-      - name: set cache id
-        run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - name: install requirements
+        env:
+          PIP_USE_PEP517: 1
+        run: |
+          python -m \
+            pip install ".[docs]"
 
-      - name: use cache
-        uses: actions/cache@v4
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
+      - name: confirm buildability
+        run: |
+          python -m \
+            mkdocs build \
+            --clean \
+            --verbose
 
-      - name: install dependencies
-        run: python -m pip install ".[docs]"
-
-      - name: build & deploy
-        run: mkdocs gh-deploy --force
+      - name: deploy to gh-pages
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          python -m \
+            mkdocs gh-deploy \
+            --clean \
+            --force

.github/workflows/pypi-release.yml

@@ -0,0 +1,67 @@
+name: PyPI Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      publish_package:
+        description: 'Publish build on PyPi? [true/false]'
+        required: true
+        default: 'false'
+
+jobs:
+  build-and-release:
+    if: github.repository == 'invoke-ai/InvokeAI'
+    runs-on: ubuntu-22.04
+    env:
+      TWINE_USERNAME: __token__
+      TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      TWINE_NON_INTERACTIVE: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node 18
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: '8.12.1'
+
+      - name: Install frontend dependencies
+        run: pnpm install --prefer-frozen-lockfile
+        working-directory: invokeai/frontend/web
+
+      - name: Build frontend
+        run: pnpm run build
+        working-directory: invokeai/frontend/web
+
+      - name: Install python dependencies
+        run: pip install --upgrade build twine
+
+      - name: Build python package
+        run: python3 -m build
+
+      - name: Upload build as workflow artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Check distribution
+        run: twine check dist/*
+
+      - name: Check PyPI versions
+        if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')
+        run: |
+          pip install --upgrade requests
+          python -c "\
+          import scripts.pypi_helper; \
+          EXISTS=scripts.pypi_helper.local_on_pypi(); \
+          print(f'PACKAGE_EXISTS={EXISTS}')" >> $GITHUB_ENV
+
+      - name: Publish build on PyPi
+        if: env.PACKAGE_EXISTS == 'False' && env.TWINE_PASSWORD != '' && github.event.inputs.publish_package == 'true'
+        run: twine upload dist/*

.github/workflows/python-checks.yml

@@ -1,64 +0,0 @@
-# Runs python code quality checks.
-#
-# Checks for changes to python files before running the checks.
-# When manually triggered or called from another workflow, always runs the tests.
-#
-# TODO: Add mypy or pyright to the checks.
-
-name: 'python checks'
-
-on:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    types:
-      - 'ready_for_review'
-      - 'opened'
-      - 'synchronize'
-  merge_group:
-  workflow_dispatch:
-  workflow_call:
-
-jobs:
-  python-checks:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5 # expected run time: <1 min
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-
-      - name: check for changed python files
-        if: ${{ github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' }}
-        id: changed-files
-        uses: tj-actions/changed-files@v42
-        with:
-          files_yaml: |
-            python:
-              - 'pyproject.toml'
-              - 'invokeai/**'
-              - '!invokeai/frontend/web/**'
-              - 'tests/**'
-
-      - name: setup python
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: pip
-          cache-dependency-path: pyproject.toml
-
-      - name: install ruff
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: pip install ruff
-        shell: bash
-
-      - name: ruff check
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: ruff check --output-format=github .
-        shell: bash
-
-      - name: ruff format
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: ruff format --check .
-        shell: bash

.github/workflows/python-tests.yml

@@ -1,94 +0,0 @@
-# Runs python tests on a matrix of python versions and platforms.
-#
-# Checks for changes to python files before running the tests.
-# When manually triggered or called from another workflow, always runs the tests.
-
-name: 'python tests'
-
-on:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    types:
-      - 'ready_for_review'
-      - 'opened'
-      - 'synchronize'
-  merge_group:
-  workflow_dispatch:
-  workflow_call:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  matrix:
-    strategy:
-      matrix:
-        python-version:
-          - '3.10'
-          - '3.11'
-        platform:
-          - linux-cuda-11_7
-          - linux-rocm-5_2
-          - linux-cpu
-          - macos-default
-          - windows-cpu
-        include:
-          - platform: linux-cuda-11_7
-            os: ubuntu-22.04
-            github-env: $GITHUB_ENV
-          - platform: linux-rocm-5_2
-            os: ubuntu-22.04
-            extra-index-url: 'https://download.pytorch.org/whl/rocm5.2'
-            github-env: $GITHUB_ENV
-          - platform: linux-cpu
-            os: ubuntu-22.04
-            extra-index-url: 'https://download.pytorch.org/whl/cpu'
-            github-env: $GITHUB_ENV
-          - platform: macos-default
-            os: macOS-12
-            github-env: $GITHUB_ENV
-          - platform: windows-cpu
-            os: windows-2022
-            github-env: $env:GITHUB_ENV
-    name: 'py${{ matrix.python-version }}: ${{ matrix.platform }}'
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 15 # expected run time: 2-6 min, depending on platform
-    env:
-      PIP_USE_PEP517: '1'
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-
-      - name: check for changed python files
-        if: ${{ github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' }}
-        id: changed-files
-        uses: tj-actions/changed-files@v42
-        with:
-          files_yaml: |
-            python:
-              - 'pyproject.toml'
-              - 'invokeai/**'
-              - '!invokeai/frontend/web/**'
-              - 'tests/**'
-
-      - name: setup python
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: pyproject.toml
-
-      - name: install dependencies
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        env:
-          PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }}
-        run: >
-          pip3 install --editable=".[test]"
-
-      - name: run pytest
-        if: ${{ steps.changed-files.outputs.python_any_changed == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' }}
-        run: pytest

.github/workflows/release.yml

@@ -1,96 +0,0 @@
-# Main release workflow. Triggered on tag push or manual trigger.
-#
-# - Runs all code checks and tests
-# - Verifies the app version matches the tag version.
-# - Builds the installer and build, uploading them as artifacts.
-# - Publishes to TestPyPI and PyPI. Both are conditional on the previous steps passing and require a manual approval.
-#
-# See docs/RELEASE.md for more information on the release process.
-
-name: release
-
-on:
-  push:
-    tags:
-      - 'v*'
-  workflow_dispatch:
-
-jobs:
-  check-version:
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-
-      - name: check python version
-        uses: samuelcolvin/check-python-version@v4
-        id: check-python-version
-        with:
-          version_file_path: invokeai/version/invokeai_version.py
-
-  frontend-checks:
-    uses: ./.github/workflows/frontend-checks.yml
-
-  frontend-tests:
-    uses: ./.github/workflows/frontend-tests.yml
-
-  python-checks:
-    uses: ./.github/workflows/python-checks.yml
-
-  python-tests:
-    uses: ./.github/workflows/python-tests.yml
-
-  build:
-    uses: ./.github/workflows/build-installer.yml
-
-  publish-testpypi:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5 # expected run time: <1 min
-    needs:
-      [
-        check-version,
-        frontend-checks,
-        frontend-tests,
-        python-checks,
-        python-tests,
-        build,
-      ]
-    environment:
-      name: testpypi
-      url: https://test.pypi.org/p/invokeai
-    steps:
-      - name: download distribution from build job
-        uses: actions/download-artifact@v4
-        with:
-          name: dist
-          path: dist/
-
-      - name: publish distribution to TestPyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          repository-url: https://test.pypi.org/legacy/
-
-  publish-pypi:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5 # expected run time: <1 min
-    needs:
-      [
-        check-version,
-        frontend-checks,
-        frontend-tests,
-        python-checks,
-        python-tests,
-        build,
-      ]
-    environment:
-      name: pypi
-      url: https://pypi.org/p/invokeai
-    steps:
-      - name: download distribution from build job
-        uses: actions/download-artifact@v4
-        with:
-          name: dist
-          path: dist/
-
-      - name: publish distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/style-checks.yml

@@ -0,0 +1,24 @@
+name: style checks
+
+on:
+  pull_request:
+  push:
+    branches: main
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies with pip
+        run: |
+          pip install ruff
+
+      - run: ruff check --output-format=github .
+      - run: ruff format --check .

.github/workflows/test-invoke-pip.yml

@@ -0,0 +1,129 @@
+name: Test invoke.py pip
+on:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+    types:
+      - 'ready_for_review'
+      - 'opened'
+      - 'synchronize'
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  matrix:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        python-version:
+          # - '3.9'
+          - '3.10'
+        pytorch:
+          - linux-cuda-11_7
+          - linux-rocm-5_2
+          - linux-cpu
+          - macos-default
+          - windows-cpu
+        include:
+          - pytorch: linux-cuda-11_7
+            os: ubuntu-22.04
+            github-env: $GITHUB_ENV
+          - pytorch: linux-rocm-5_2
+            os: ubuntu-22.04
+            extra-index-url: 'https://download.pytorch.org/whl/rocm5.2'
+            github-env: $GITHUB_ENV
+          - pytorch: linux-cpu
+            os: ubuntu-22.04
+            extra-index-url: 'https://download.pytorch.org/whl/cpu'
+            github-env: $GITHUB_ENV
+          - pytorch: macos-default
+            os: macOS-12
+            github-env: $GITHUB_ENV
+          - pytorch: windows-cpu
+            os: windows-2022
+            github-env: $env:GITHUB_ENV
+    name: ${{ matrix.pytorch }} on ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      PIP_USE_PEP517: '1'
+    steps:
+      - name: Checkout sources
+        id: checkout-sources
+        uses: actions/checkout@v3
+
+      - name: Check for changed python files
+        id: changed-files
+        uses: tj-actions/changed-files@v41
+        with:
+          files_yaml: |
+            python:
+              - 'pyproject.toml'
+              - 'invokeai/**'
+              - '!invokeai/frontend/web/**'
+              - 'tests/**'
+
+      - name: set test prompt to main branch validation
+        if: steps.changed-files.outputs.python_any_changed == 'true'
+        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
+
+      - name: setup python
+        if: steps.changed-files.outputs.python_any_changed == 'true'
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: install invokeai
+        if: steps.changed-files.outputs.python_any_changed == 'true'
+        env:
+          PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }}
+        run: >
+          pip3 install
+          --editable=".[test]"
+
+      - name: run pytest
+        if: steps.changed-files.outputs.python_any_changed == 'true'
+        id: run-pytest
+        run: pytest
+
+      # - name: run invokeai-configure
+      #   env:
+      #     HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+      #   run: >
+      #     invokeai-configure
+      #     --yes
+      #     --default_only
+      #     --full-precision
+      #   # can't use fp16 weights without a GPU
+
+      # - name: run invokeai
+      #   id: run-invokeai
+      #   env:
+      #     # Set offline mode to make sure configure preloaded successfully.
+      #     HF_HUB_OFFLINE: 1
+      #     HF_DATASETS_OFFLINE: 1
+      #     TRANSFORMERS_OFFLINE: 1
+      #     INVOKEAI_OUTDIR: ${{ github.workspace }}/results
+      #   run: >
+      #     invokeai
+      #     --no-patchmatch
+      #     --no-nsfw_checker
+      #     --precision=float32
+      #     --always_use_cpu
+      #     --use_memory_db
+      #     --outdir ${{ env.INVOKEAI_OUTDIR }}/${{ matrix.python-version }}/${{ matrix.pytorch }}
+      #     --from_file ${{ env.TEST_PROMPTS }}
+
+      # - name: Archive results
+      #   env:
+      #     INVOKEAI_OUTDIR: ${{ github.workspace }}/results
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: results
+      #     path: ${{ env.INVOKEAI_OUTDIR }}

.prettierrc.yaml

@@ -7,7 +7,7 @@ embeddedLanguageFormatting: auto
 overrides:
   - files: '*.md'
     options:
-      proseWrap: preserve
+      proseWrap: always
       printWidth: 80
       parser: markdown
       cursorOffset: -1

docs/RELEASE.md

@@ -1,142 +0,0 @@
-# Release Process
-
-The app is published in twice, in different build formats.
-
-- A [PyPI] distribution. This includes both a source distribution and built distribution (a wheel). Users install with `pip install invokeai`. The updater uses this build.
-- An installer on the [InvokeAI Releases Page]. This is a zip file with install scripts and a wheel. This is only used for new installs.
-
-## General Prep
-
-Make a developer call-out for PRs to merge. Merge and test things out.
-
-While the release workflow does not include end-to-end tests, it does pause before publishing so you can download and test the final build.
-
-## Release Workflow
-
-The `release.yml` workflow runs a number of jobs to handle code checks, tests, build and publish on PyPI.
-
-It is triggered on **tag push**, when the tag matches `v*`. It doesn't matter if you've prepped a release branch like `release/v3.5.0` or are releasing from `main` - it works the same.
-
-> Because commits are reference-counted, it is safe to create a release branch, tag it, let the workflow run, then delete the branch. So long as the tag exists, that commit will exist.
-
-### Triggering the Workflow
-
-Run `make tag-release` to tag the current commit and kick off the workflow.
-
-The release may also be dispatched [manually].
-
-### Workflow Jobs and Process
-
-The workflow consists of a number of concurrently-run jobs, and two final publish jobs.
-
-The publish jobs require manual approval and are only run if the other jobs succeed.
-
-#### `check-version` Job
-
-This job checks that the git ref matches the app version. It matches the ref against the `__version__` variable in `invokeai/version/invokeai_version.py`.
-
-When the workflow is triggered by tag push, the ref is the tag. If the workflow is run manually, the ref is the target selected from the **Use workflow from** dropdown.
-
-This job uses [samuelcolvin/check-python-version].
-
-> Any valid [version specifier] works, so long as the tag matches the version. The release workflow works exactly the same for `RC`, `post`, `dev`, etc.
-
-#### Check and Test Jobs
-
-- **`python-tests`**: runs `pytest` on matrix of platforms
-- **`python-checks`**: runs `ruff` (format and lint)
-- **`frontend-tests`**: runs `vitest`
-- **`frontend-checks`**: runs `prettier` (format), `eslint` (lint), `dpdm` (circular refs), `tsc` (static type check) and `knip` (unused imports)
-
-> **TODO** We should add `mypy` or `pyright` to the **`check-python`** job.
-
-> **TODO** We should add an end-to-end test job that generates an image.
-
-#### `build-installer` Job
-
-This sets up both python and frontend dependencies and builds the python package. Internally, this runs `installer/create_installer.sh` and uploads two artifacts:
-
-- **`dist`**: the python distribution, to be published on PyPI
-- **`InvokeAI-installer-${VERSION}.zip`**: the installer to be included in the GitHub release
-
-#### Sanity Check & Smoke Test
-
-At this point, the release workflow pauses as the remaining publish jobs require approval.
-
-A maintainer should go to the **Summary** tab of the workflow, download the installer and test it. Ensure the app loads and generates.
-
-> The same wheel file is bundled in the installer and in the `dist` artifact, which is uploaded to PyPI. You should end up with the exactly the same installation of the `invokeai` package from any of these methods.
-
-#### PyPI Publish Jobs
-
-The publish jobs will run if any of the previous jobs fail.
-
-They use [GitHub environments], which are configured as [trusted publishers] on PyPI.
-
-Both jobs require a maintainer to approve them from the workflow's **Summary** tab.
-
-- Click the **Review deployments** button
-- Select the environment (either `testpypi` or `pypi`)
-- Click **Approve and deploy**
-
-> **If the version already exists on PyPI, the publish jobs will fail.** PyPI only allows a given version to be published once - you cannot change it. If version published on PyPI has a problem, you'll need to "fail forward" by bumping the app version and publishing a followup release.
-
-#### `publish-testpypi` Job
-
-Publishes the distribution on the [Test PyPI] index, using the `testpypi` GitHub environment.
-
-This job is not required for the production PyPI publish, but included just in case you want to test the PyPI release.
-
-If approved and successful, you could try out the test release like this:
-
-```sh
-# Create a new virtual environment
-python -m venv ~/.test-invokeai-dist --prompt test-invokeai-dist
-# Install the distribution from Test PyPI
-pip install --index-url https://test.pypi.org/simple/ invokeai
-# Run and test the app
-invokeai-web
-# Cleanup
-deactivate
-rm -rf ~/.test-invokeai-dist
-```
-
-#### `publish-pypi` Job
-
-Publishes the distribution on the production PyPI index, using the `pypi` GitHub environment.
-
-## Publish the GitHub Release with installer
-
-Once the release is published to PyPI, it's time to publish the GitHub release.
-
-1. [Draft a new release] on GitHub, choosing the tag that triggered the release.
-2. Write the release notes, describing important changes. The **Generate release notes** button automatically inserts the changelog and new contributors, and you can copy/paste the intro from previous releases.
-3. Upload the zip file created in **`build`** job into the Assets section of the release notes. You can also upload the zip into the body of the release notes, since it can be hard for users to find the Assets section.
-4. Check the **Set as a pre-release** and **Create a discussion for this release** checkboxes at the bottom of the release page.
-5. Publish the pre-release.
-6. Announce the pre-release in Discord.
-
-> **TODO** Workflows can create a GitHub release from a template and upload release assets. One popular action to handle this is [ncipollo/release-action]. A future enhancement to the release process could set this up.
-
-## Manual Build
-
-The `build installer` workflow can be dispatched manually. This is useful to test the installer for a given branch or tag.
-
-No checks are run, it just builds.
-
-## Manual Release
-
-The `release` workflow can be dispatched manually. You must dispatch the workflow from the right tag, else it will fail the version check.
-
-This functionality is available as a fallback in case something goes wonky. Typically, releases should be triggered via tag push as described above.
-
-[InvokeAI Releases Page]: https://github.com/invoke-ai/InvokeAI/releases
-[PyPI]: https://pypi.org/
-[Draft a new release]: https://github.com/invoke-ai/InvokeAI/releases/new
-[Test PyPI]: https://test.pypi.org/
-[version specifier]: https://packaging.python.org/en/latest/specifications/version-specifiers/
-[ncipollo/release-action]: https://github.com/ncipollo/release-action
-[GitHub environments]: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment
-[trusted publishers]: https://docs.pypi.org/trusted-publishers/
-[samuelcolvin/check-python-version]: https://github.com/samuelcolvin/check-python-version
-[manually]: #manual-release

docs/nodes/INVOCATION_API.md

@@ -1,45 +0,0 @@
-# Invocation API
-
-Each invocation's `invoke` method is provided a single arg - the Invocation
-Context.
-
-This object provides access to various methods, used to interact with the
-application. Loading and saving images, logging messages, etc.
-
-!!! warning ""
-
-    This API may shift slightly until the release of v4.0.0 as we work through a few final updates to the Model Manager.
-
-```py
-class MyInvocation(BaseInvocation):
-  ...
-  def invoke(self, context: InvocationContext) -> ImageOutput:
-      image_pil = context.images.get_pil(image_name)
-      # Do something to the image
-      image_dto = context.images.save(image_pil)
-      # Log a message
-      context.logger.info(f"Did something cool, image saved!")
-      ...
-```
-
-<!-- prettier-ignore-start -->
-::: invokeai.app.services.shared.invocation_context.InvocationContext
-    options:
-        members: false
-
-::: invokeai.app.services.shared.invocation_context.ImagesInterface
-
-::: invokeai.app.services.shared.invocation_context.TensorsInterface
-
-::: invokeai.app.services.shared.invocation_context.ConditioningInterface
-
-::: invokeai.app.services.shared.invocation_context.ModelsInterface
-
-::: invokeai.app.services.shared.invocation_context.LoggerInterface
-
-::: invokeai.app.services.shared.invocation_context.ConfigInterface
-
-::: invokeai.app.services.shared.invocation_context.UtilInterface
-
-::: invokeai.app.services.shared.invocation_context.BoardsInterface
-<!-- prettier-ignore-end -->

docs/nodes/NODES_MIGRATION_V3_V4.md

@@ -1,148 +0,0 @@
-# Invoke v4.0.0 Nodes API Migration guide
-
-Invoke v4.0.0 is versioned as such due to breaking changes to the API utilized
-by nodes, both core and custom.
-
-## Motivation
-
-Prior to v4.0.0, the `invokeai` python package has not be set up to be utilized
-as a library. That is to say, it didn't have any explicitly public API, and node
-authors had to work with the unstable internal application API.
-
-v4.0.0 introduces a stable public API for nodes.
-
-## Changes
-
-There are two node-author-facing changes:
-
-1. Import Paths
-1. Invocation Context API
-
-### Import Paths
-
-All public objects are now exported from `invokeai.invocation_api`:
-
-```py
-# Old
-from invokeai.app.invocations.baseinvocation import (
-    BaseInvocation,
-    InputField,
-    InvocationContext,
-    invocation,
-)
-from invokeai.app.invocations.primitives import ImageField
-
-# New
-from invokeai.invocation_api import (
-    BaseInvocation,
-    ImageField,
-    InputField,
-    InvocationContext,
-    invocation,
-)
-```
-
-It's possible that we've missed some classes you need in your node. Please let
-us know if that's the case.
-
-### Invocation Context API
-
-Most nodes utilize the Invocation Context, an object that is passed to the
-`invoke` that provides access to data and services a node may need.
-
-Until now, that object and the services it exposed were internal. Exposing them
-to nodes means that changes to our internal implementation could break nodes.
-The methods on the services are also often fairly complicated and allowed nodes
-to footgun.
-
-In v4.0.0, this object has been refactored to be much simpler.
-
-See [INVOCATION_API](./INVOCATION_API.md) for full details of the API.
-
-!!! warning ""
-
-    This API may shift slightly until the release of v4.0.0 as we work through a few final updates to the Model Manager.
-
-#### Improved Service Methods
-
-The biggest offender was the image save method:
-
-```py
-# Old
-image_dto = context.services.images.create(
-    image=image,
-    image_origin=ResourceOrigin.INTERNAL,
-    image_category=ImageCategory.GENERAL,
-    node_id=self.id,
-    session_id=context.graph_execution_state_id,
-    is_intermediate=self.is_intermediate,
-    metadata=self.metadata,
-    workflow=context.workflow,
-)
-
-# New
-image_dto = context.images.save(image=image)
-```
-
-Other methods are simplified, or enhanced with additional functionality:
-
-```py
-# Old
-image = context.services.images.get_pil_image(image_name)
-
-# New
-image = context.images.get_pil(image_name)
-image_cmyk = context.images.get_pil(image_name, "CMYK")
-```
-
-We also had some typing issues around tensors:
-
-```py
-# Old
-# `latents` typed as `torch.Tensor`, but could be `ConditioningFieldData`
-latents = context.services.latents.get(self.latents.latents_name)
-# `data` typed as `torch.Tenssor,` but could be `ConditioningFieldData`
-context.services.latents.save(latents_name, data)
-
-# New - separate methods for tensors and conditioning data w/ correct typing
-# Also, the service generates the names
-tensor_name = context.tensors.save(tensor)
-tensor = context.tensors.load(tensor_name)
-# For conditioning
-cond_name = context.conditioning.save(cond_data)
-cond_data = context.conditioning.load(cond_name)
-```
-
-#### Output Construction
-
-Core Outputs have builder functions right on them - no need to manually
-construct these objects, or use an extra utility:
-
-```py
-# Old
-image_output = ImageOutput(
-    image=ImageField(image_name=image_dto.image_name),
-    width=image_dto.width,
-    height=image_dto.height,
-)
-latents_output = build_latents_output(latents_name=name, latents=latents, seed=None)
-noise_output = NoiseOutput(
-    noise=LatentsField(latents_name=latents_name, seed=seed),
-    width=latents.size()[3] * 8,
-    height=latents.size()[2] * 8,
-)
-cond_output = ConditioningOutput(
-    conditioning=ConditioningField(
-        conditioning_name=conditioning_name,
-    ),
-)
-
-# New
-image_output = ImageOutput.build(image_dto)
-latents_output = LatentsOutput.build(latents_name=name, latents=noise, seed=self.seed)
-noise_output = NoiseOutput.build(latents_name=name, latents=noise, seed=self.seed)
-cond_output = ConditioningOutput.build(conditioning_name)
-```
-
-You can still create the objects using constructors if you want, but we suggest
-using the builder methods.

docs/nodes/communityNodes.md

@@ -32,7 +32,6 @@ To use a community workflow, download the the `.json` node graph file and load i
     + [Image to Character Art Image Nodes](#image-to-character-art-image-nodes)
     + [Image Picker](#image-picker)
     + [Image Resize Plus](#image-resize-plus)
-    + [Latent Upscale](#latent-upscale)
     + [Load Video Frame](#load-video-frame)
     + [Make 3D](#make-3d)
     + [Mask Operations](#mask-operations)
@@ -291,13 +290,6 @@ View:
 </br><img src="https://raw.githubusercontent.com/VeyDlin/image-resize-plus-node/master/.readme/node.png" width="500" />
 
 
---------------------------------
-### Latent Upscale
-
-**Description:** This node uses a small (~2.4mb) model to upscale the latents used in a Stable Diffusion 1.5 or Stable Diffusion XL image generation, rather than the typical interpolation method, avoiding the traditional downsides of the latent upscale technique.
-
-**Node Link:** [https://github.com/gogurtenjoyer/latent-upscale](https://github.com/gogurtenjoyer/latent-upscale)
-
 --------------------------------
 ### Load Video Frame
 
@@ -354,21 +346,12 @@ See full docs here: https://github.com/skunkworxdark/Prompt-tools-nodes/edit/mai
 
 **Description:** A set of nodes for Metadata. Collect Metadata from within an `iterate` node & extract metadata from an image.
 
-- `Metadata Item Linked` - Allows collecting of metadata while within an iterate node with no need for a collect node or conversion to metadata node
-- `Metadata From Image` - Provides Metadata from an image
-- `Metadata To String` - Extracts a String value of a label from metadata
-- `Metadata To Integer` - Extracts an Integer value of a label from metadata
-- `Metadata To Float` - Extracts a Float value of a label from metadata
-- `Metadata To Scheduler` - Extracts a Scheduler value of a label from metadata
-- `Metadata To Bool` - Extracts Bool types from metadata
-- `Metadata To Model` - Extracts model types from metadata
-- `Metadata To SDXL Model` - Extracts SDXL model types from metadata
-- `Metadata To LoRAs` - Extracts Loras from metadata. 
-- `Metadata To SDXL LoRAs` - Extracts SDXL Loras from metadata
-- `Metadata To ControlNets` - Extracts ControNets from metadata
-- `Metadata To IP-Adapters` - Extracts IP-Adapters from metadata
-- `Metadata To T2I-Adapters` - Extracts T2I-Adapters from metadata
-- `Denoise Latents + Metadata` - This is an inherited version of the existing `Denoise Latents` node but with a metadata input and output. 
+- `Metadata Item Linked` - Allows collecting of metadata while within an iterate node with no need for a collect node or conversion to metadata node.
+- `Metadata From Image` - Provides Metadata from an image.
+- `Metadata To String` - Extracts a String value of a label from metadata.
+- `Metadata To Integer` - Extracts an Integer value of a label from metadata.
+- `Metadata To Float` - Extracts a Float value of a label from metadata.
+- `Metadata To Scheduler` - Extracts a Scheduler value of a label from metadata.
 
 **Node Link:** https://github.com/skunkworxdark/metadata-linked-nodes
 

docs/nodes/defaultNodes.md

@@ -19,8 +19,6 @@ their descriptions.
 | Conditioning Primitive                                        | A conditioning tensor primitive value                                                                                                                |
 | Content Shuffle Processor                                     | Applies content shuffle processing to image                                                                                                          |
 | ControlNet                                                    | Collects ControlNet info to pass to other nodes                                                                                                      |
-| Create Denoise Mask                                           | Converts a greyscale or transparency image into a mask for denoising.                                                                                |
-| Create Gradient Mask                                          | Creates a mask for Gradient ("soft", "differential") inpainting that gradually expands during denoising. Improves edge coherence.                    |
 | Denoise Latents                                               | Denoises noisy latents to decodable images                                                                                                           |
 | Divide Integers                                               | Divides two numbers                                                                                                                                  |
 | Dynamic Prompt                                                | Parses a prompt using adieyal/dynamicprompts' random or combinatorial generator                                                                      |

docs/requirements-mkdocs.txt

@@ -0,0 +1,5 @@
+mkdocs
+mkdocs-material>=8, <9
+mkdocs-git-revision-date-localized-plugin
+mkdocs-redirects==1.2.0
+

docs/stylesheets/extra.css

@@ -0,0 +1,5 @@
+:root {
+    --md-primary-fg-color:        #35A4DB;
+    --md-primary-fg-color--light: #35A4DB;
+    --md-primary-fg-color--dark:  #35A4DB;
+  }

installer/create_installer.sh

@@ -2,18 +2,22 @@
 
 set -e
 
-BCYAN="\033[1;36m"
-BYELLOW="\033[1;33m"
-BGREEN="\033[1;32m"
-BRED="\033[1;31m"
-RED="\033[31m"
-RESET="\033[0m"
+BCYAN="\e[1;36m"
+BYELLOW="\e[1;33m"
+BGREEN="\e[1;32m"
+BRED="\e[1;31m"
+RED="\e[31m"
+RESET="\e[0m"
+
+function is_bin_in_path {
+    builtin type -P "$1" &>/dev/null
+}
 
 function git_show {
     git show -s --format=oneline --abbrev-commit "$1" | cat
 }
 
-if [[ ! -z "${VIRTUAL_ENV}" ]]; then
+if [[ -v "VIRTUAL_ENV" ]]; then
     # we can't just call 'deactivate' because this function is not exported
     # to the environment of this script from the bash process that runs the script
     echo -e "${BRED}A virtual environment is activated. Please deactivate it before proceeding.${RESET}"
@@ -22,63 +26,31 @@ fi
 
 cd "$(dirname "$0")"
 
+echo
+echo -e "${BYELLOW}This script must be run from the installer directory!${RESET}"
+echo "The current working directory is $(pwd)"
+read -p "If that looks right, press any key to proceed, or CTRL-C to exit..."
+echo
+
+# Some machines only have `python3` in PATH, others have `python` - make an alias.
+# We can use a function to approximate an alias within a non-interactive shell.
+if ! is_bin_in_path python && is_bin_in_path python3; then
+    function python {
+        python3 "$@"
+    }
+fi
+
 VERSION=$(
     cd ..
-    python3 -c "from invokeai.version import __version__ as version; print(version)"
+    python -c "from invokeai.version import __version__ as version; print(version)"
 )
-VERSION="v${VERSION}"
-
-if [[ ! -z ${CI} ]]; then
-    echo
-    echo -e "${BCYAN}CI environment detected${RESET}"
-    echo
-else
-    echo
-    echo -e "${BYELLOW}This script must be run from the installer directory!${RESET}"
-    echo "The current working directory is $(pwd)"
-    read -p "If that looks right, press any key to proceed, or CTRL-C to exit..."
-    echo
-fi
+PATCH=""
+VERSION="v${VERSION}${PATCH}"
 
 echo -e "${BGREEN}HEAD${RESET}:"
 git_show HEAD
 echo
 
-# ---------------------- FRONTEND ----------------------
-
-pushd ../invokeai/frontend/web >/dev/null
-echo "Installing frontend dependencies..."
-echo
-pnpm i --frozen-lockfile
-echo
-if [[ ! -z ${CI} ]]; then
-    echo "Building frontend without checks..."
-    # In CI, we have already done the frontend checks and can just build
-    pnpm vite build
-else
-    echo "Running checks and building frontend..."
-    # This runs all the frontend checks and builds
-    pnpm build
-fi
-echo
-popd
-
-# ---------------------- BACKEND ----------------------
-
-echo
-echo "Building wheel..."
-echo
-
-# install the 'build' package in the user site packages, if needed
-# could be improved by using a temporary venv, but it's tiny and harmless
-if [[ $(python3 -c 'from importlib.util import find_spec; print(find_spec("build") is None)') == "True" ]]; then
-    pip install --user build
-fi
-
-rm -rf ../build
-
-python3 -m build --outdir dist/ ../.
-
 # ----------------------
 
 echo
@@ -106,28 +78,10 @@ chmod a+x InvokeAI-Installer/install.sh
 cp install.bat.in InvokeAI-Installer/install.bat
 cp WinLongPathsEnabled.reg InvokeAI-Installer/
 
-FILENAME=InvokeAI-installer-$VERSION.zip
-
 # Zip everything up
-zip -r ${FILENAME} InvokeAI-Installer
+zip -r InvokeAI-installer-$VERSION.zip InvokeAI-Installer
 
-echo
-echo -e "${BGREEN}Built installer: ./${FILENAME}${RESET}"
-echo -e "${BGREEN}Built PyPi distribution: ./dist${RESET}"
-
-# clean up, but only if we are not in a github action
-if [[ -z ${CI} ]]; then
-    echo
-    echo "Cleaning up intermediate build files..."
-    rm -rf InvokeAI-Installer tmp ../invokeai/frontend/web/dist/
-fi
-
-if [[ ! -z ${CI} ]]; then
-    echo
-    echo "Setting GitHub action outputs..."
-    echo "INSTALLER_FILENAME=${FILENAME}" >>$GITHUB_OUTPUT
-    echo "INSTALLER_PATH=installer/${FILENAME}" >>$GITHUB_OUTPUT
-    echo "DIST_PATH=installer/dist/" >>$GITHUB_OUTPUT
-fi
+# clean up
+rm -rf InvokeAI-Installer tmp dist ../invokeai/frontend/web/dist/
 
 exit 0

installer/tag_release.sh

@@ -2,12 +2,12 @@
 
 set -e
 
-BCYAN="\033[1;36m"
-BYELLOW="\033[1;33m"
-BGREEN="\033[1;32m"
-BRED="\033[1;31m"
-RED="\033[31m"
-RESET="\033[0m"
+BCYAN="\e[1;36m"
+BYELLOW="\e[1;33m"
+BGREEN="\e[1;32m"
+BRED="\e[1;31m"
+RED="\e[31m"
+RESET="\e[0m"
 
 function does_tag_exist {
     git rev-parse --quiet --verify "refs/tags/$1" >/dev/null
@@ -23,40 +23,49 @@ function git_show {
 
 VERSION=$(
     cd ..
-    python3 -c "from invokeai.version import __version__ as version; print(version)"
+    python -c "from invokeai.version import __version__ as version; print(version)"
 )
 PATCH=""
+MAJOR_VERSION=$(echo $VERSION | sed 's/\..*$//')
 VERSION="v${VERSION}${PATCH}"
+LATEST_TAG="v${MAJOR_VERSION}-latest"
 
 if does_tag_exist $VERSION; then
     echo -e "${BCYAN}${VERSION}${RESET} already exists:"
     git_show_ref tags/$VERSION
     echo
 fi
+if does_tag_exist $LATEST_TAG; then
+    echo -e "${BCYAN}${LATEST_TAG}${RESET} already exists:"
+    git_show_ref tags/$LATEST_TAG
+    echo
+fi
 
 echo -e "${BGREEN}HEAD${RESET}:"
 git_show
 echo
 
-echo -e "${BGREEN}git remote -v${RESET}:"
-git remote -v
-echo
-
-echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on origin remote${RESET}? "
+echo -e -n "Create tags ${BCYAN}${VERSION}${RESET} and ${BCYAN}${LATEST_TAG}${RESET} @ ${BGREEN}HEAD${RESET}, ${RED}deleting existing tags on remote${RESET}? "
 read -e -p 'y/n [n]: ' input
 RESPONSE=${input:='n'}
 if [ "$RESPONSE" == 'y' ]; then
     echo
-    echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on origin remote..."
-    git push origin :refs/tags/$VERSION
+    echo -e "Deleting ${BCYAN}${VERSION}${RESET} tag on remote..."
+    git push --delete origin $VERSION
 
-    echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} on locally..."
+    echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${VERSION}${RESET} locally..."
     if ! git tag -fa $VERSION; then
         echo "Existing/invalid tag"
         exit -1
     fi
 
-    echo -e "Pushing updated tags to origin remote..."
+    echo -e "Deleting ${BCYAN}${LATEST_TAG}${RESET} tag on remote..."
+    git push --delete origin $LATEST_TAG
+
+    echo -e "Tagging ${BGREEN}HEAD${RESET} with ${BCYAN}${LATEST_TAG}${RESET} locally..."
+    git tag -fa $LATEST_TAG
+
+    echo -e "Pushing updated tags to remote..."
     git push origin --tags
 fi
 exit 0

invokeai/app/api/routers/model_manager.py

@@ -5,6 +5,7 @@ import pathlib
 import shutil
 from hashlib import sha1
 from random import randbytes
+import traceback
 from typing import Any, Dict, List, Optional, Set
 
 from fastapi import Body, Path, Query, Response
@@ -244,7 +245,6 @@ async def get_model_metadata(
 
     return result
 
-
 @model_manager_router.patch(
     "/i/{key}/metadata",
     operation_id="update_model_metadata",
@@ -258,23 +258,24 @@ async def get_model_metadata(
 )
 async def update_model_metadata(
     key: str = Path(description="Key of the model repo metadata to fetch."),
-    changes: ModelMetadataChanges = Body(description="The changes"),
+    changes: ModelMetadataChanges = Body(description="The changes")
 ) -> Optional[AnyModelRepoMetadata]:
     """Updates or creates a model metadata object."""
     record_store = ApiDependencies.invoker.services.model_manager.store
     metadata_store = ApiDependencies.invoker.services.model_manager.store.metadata_store
-
+    
     try:
         original_metadata = record_store.get_metadata(key)
         if original_metadata:
+            if changes.trigger_phrases:
+                original_metadata.trigger_phrases = changes.trigger_phrases
+
             if changes.default_settings:
                 original_metadata.default_settings = changes.default_settings
 
             metadata_store.update_metadata(key, original_metadata)
         else:
-            metadata_store.add_metadata(
-                key, BaseMetadata(name="", author="", default_settings=changes.default_settings)
-            )
+            metadata_store.add_metadata(key, BaseMetadata(name="", author="",trigger_phrases=changes.trigger_phrases, default_settings=changes.default_settings))
     except Exception as e:
         raise HTTPException(
             status_code=500,
@@ -494,7 +495,6 @@ async def add_model_record(
 )
 async def install_model(
     source: str = Query(description="Model source to install, can be a local path, repo_id, or remote URL"),
-    inplace: Optional[bool] = Query(description="Whether or not to install a local model in place", default=False),
     # TODO(MM2): Can we type this?
     config: Optional[Dict[str, Any]] = Body(
         description="Dict of fields that override auto-probed values in the model config record, such as name, description and prediction_type ",
@@ -537,7 +537,6 @@ async def install_model(
             source=source,
             config=config,
             access_token=access_token,
-            inplace=bool(inplace),
         )
         logger.info(f"Started installation of {source}")
     except UnknownModelException as e:

invokeai/app/api_app.py

@@ -2,7 +2,6 @@
 # which are imported/used before parse_args() is called will get the default config values instead of the
 # values from the command line or config file.
 import sys
-from contextlib import asynccontextmanager
 
 from invokeai.app.api.no_cache_staticfiles import NoCacheStaticFiles
 from invokeai.version.invokeai_version import __version__
@@ -72,25 +71,9 @@ logger = InvokeAILogger.get_logger(config=app_config)
 mimetypes.add_type("application/javascript", ".js")
 mimetypes.add_type("text/css", ".css")
 
-
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    # Add startup event to load dependencies
-    ApiDependencies.initialize(config=app_config, event_handler_id=event_handler_id, logger=logger)
-    yield
-    # Shut down threads
-    ApiDependencies.shutdown()
-
-
 # Create the app
 # TODO: create this all in a method so configuration/etc. can be passed in?
-app = FastAPI(
-    title="Invoke - Community Edition",
-    docs_url=None,
-    redoc_url=None,
-    separate_input_output_schemas=False,
-    lifespan=lifespan,
-)
+app = FastAPI(title="Invoke - Community Edition", docs_url=None, redoc_url=None, separate_input_output_schemas=False)
 
 # Add event handler
 event_handler_id: int = id(app)
@@ -113,6 +96,18 @@ app.add_middleware(
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
+# Add startup event to load dependencies
+@app.on_event("startup")
+async def startup_event() -> None:
+    ApiDependencies.initialize(config=app_config, event_handler_id=event_handler_id, logger=logger)
+
+
+# Shut down threads
+@app.on_event("shutdown")
+async def shutdown_event() -> None:
+    ApiDependencies.shutdown()
+
+
 # Include all routers
 app.include_router(utilities.utilities_router, prefix="/api")
 app.include_router(model_manager.model_manager_router, prefix="/api")

invokeai/app/invocations/compel.py

@@ -1,23 +1,24 @@
-from typing import Iterator, List, Optional, Tuple, Union, cast
+from typing import Iterator, List, Optional, Tuple, Union
 
 import torch
 from compel import Compel, ReturnedEmbeddingsType
 from compel.prompt_parser import Blend, Conjunction, CrossAttentionControlSubstitute, FlattenedPrompt, Fragment
-from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
+from transformers import CLIPTokenizer
 
+import invokeai.backend.util.logging as logger
 from invokeai.app.invocations.fields import (
-    ConditioningField,
     FieldDescriptions,
     Input,
     InputField,
-    MaskField,
     OutputField,
     UIComponent,
 )
 from invokeai.app.invocations.primitives import ConditioningOutput
+from invokeai.app.services.model_records import UnknownModelException
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.app.util.ti_utils import generate_ti_list
+from invokeai.app.util.ti_utils import extract_ti_triggers_from_prompt
 from invokeai.backend.lora import LoRAModelRaw
+from invokeai.backend.model_manager import ModelType
 from invokeai.backend.model_patcher import ModelPatcher
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
     BasicConditioningInfo,
@@ -25,9 +26,15 @@ from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
     ExtraConditioningInfo,
     SDXLConditioningInfo,
 )
+from invokeai.backend.textual_inversion import TextualInversionModelRaw
 from invokeai.backend.util.devices import torch_dtype
 
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output
+from .baseinvocation import (
+    BaseInvocation,
+    BaseInvocationOutput,
+    invocation,
+    invocation_output,
+)
 from .model import ClipField
 
 # unconditioned: Optional[torch.Tensor]
@@ -44,7 +51,7 @@ from .model import ClipField
     title="Prompt",
     tags=["prompt", "compel"],
     category="conditioning",
-    version="1.2.0",
+    version="1.0.1",
 )
 class CompelInvocation(BaseInvocation):
     """Parse prompt using compel package to conditioning."""
@@ -59,19 +66,11 @@ class CompelInvocation(BaseInvocation):
         description=FieldDescriptions.clip,
         input=Input.Connection,
     )
-    mask: Optional[MaskField] = InputField(
-        default=None, description="A mask defining the region that this conditioning prompt applies to."
-    )
-    mask_weight: float = InputField(default=1.0, description="")
 
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> ConditioningOutput:
         tokenizer_info = context.models.load(**self.clip.tokenizer.model_dump())
-        tokenizer_model = tokenizer_info.model
-        assert isinstance(tokenizer_model, CLIPTokenizer)
         text_encoder_info = context.models.load(**self.clip.text_encoder.model_dump())
-        text_encoder_model = text_encoder_info.model
-        assert isinstance(text_encoder_model, CLIPTextModel)
 
         def _lora_loader() -> Iterator[Tuple[LoRAModelRaw, float]]:
             for lora in self.clip.loras:
@@ -83,10 +82,21 @@ class CompelInvocation(BaseInvocation):
 
         # loras = [(context.models.get(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.clip.loras]
 
-        ti_list = generate_ti_list(self.prompt, text_encoder_info.config.base, context)
+        ti_list = []
+        for trigger in extract_ti_triggers_from_prompt(self.prompt):
+            name = trigger[1:-1]
+            try:
+                loaded_model = context.models.load(key=name).model
+                assert isinstance(loaded_model, TextualInversionModelRaw)
+                ti_list.append((name, loaded_model))
+            except UnknownModelException:
+                # print(e)
+                # import traceback
+                # print(traceback.format_exc())
+                print(f'Warn: trigger: "{trigger}" not found')
 
         with (
-            ModelPatcher.apply_ti(tokenizer_model, text_encoder_model, ti_list) as (
+            ModelPatcher.apply_ti(tokenizer_info.model, text_encoder_info.model, ti_list) as (
                 tokenizer,
                 ti_manager,
             ),
@@ -94,9 +104,8 @@ class CompelInvocation(BaseInvocation):
             # Apply the LoRA after text_encoder has been moved to its target device for faster patching.
             ModelPatcher.apply_lora_text_encoder(text_encoder, _lora_loader()),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.
-            ModelPatcher.apply_clip_skip(text_encoder_model, self.clip.skipped_layers),
+            ModelPatcher.apply_clip_skip(text_encoder_info.model, self.clip.skipped_layers),
         ):
-            assert isinstance(text_encoder, CLIPTextModel)
             compel = Compel(
                 tokenizer=tokenizer,
                 text_encoder=text_encoder,
@@ -130,13 +139,7 @@ class CompelInvocation(BaseInvocation):
 
         conditioning_name = context.conditioning.save(conditioning_data)
 
-        return ConditioningOutput(
-            conditioning=ConditioningField(
-                conditioning_name=conditioning_name,
-                mask=self.mask,
-                mask_weight=self.mask_weight,
-            )
-        )
+        return ConditioningOutput.build(conditioning_name)
 
 
 class SDXLPromptInvocationBase:
@@ -152,11 +155,7 @@ class SDXLPromptInvocationBase:
         zero_on_empty: bool,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[ExtraConditioningInfo]]:
         tokenizer_info = context.models.load(**clip_field.tokenizer.model_dump())
-        tokenizer_model = tokenizer_info.model
-        assert isinstance(tokenizer_model, CLIPTokenizer)
         text_encoder_info = context.models.load(**clip_field.text_encoder.model_dump())
-        text_encoder_model = text_encoder_info.model
-        assert isinstance(text_encoder_model, (CLIPTextModel, CLIPTextModelWithProjection))
 
         # return zero on empty
         if prompt == "" and zero_on_empty:
@@ -190,10 +189,25 @@ class SDXLPromptInvocationBase:
 
         # loras = [(context.models.get(**lora.dict(exclude={"weight"})).context.model, lora.weight) for lora in self.clip.loras]
 
-        ti_list = generate_ti_list(prompt, text_encoder_info.config.base, context)
+        ti_list = []
+        for trigger in extract_ti_triggers_from_prompt(prompt):
+            name = trigger[1:-1]
+            try:
+                ti_model = context.models.load_by_attrs(
+                    model_name=name, base_model=text_encoder_info.config.base, model_type=ModelType.TextualInversion
+                ).model
+                assert isinstance(ti_model, TextualInversionModelRaw)
+                ti_list.append((name, ti_model))
+            except UnknownModelException:
+                # print(e)
+                # import traceback
+                # print(traceback.format_exc())
+                logger.warning(f'trigger: "{trigger}" not found')
+            except ValueError:
+                logger.warning(f'trigger: "{trigger}" more than one similarly-named textual inversion models')
 
         with (
-            ModelPatcher.apply_ti(tokenizer_model, text_encoder_model, ti_list) as (
+            ModelPatcher.apply_ti(tokenizer_info.model, text_encoder_info.model, ti_list) as (
                 tokenizer,
                 ti_manager,
             ),
@@ -201,10 +215,8 @@ class SDXLPromptInvocationBase:
             # Apply the LoRA after text_encoder has been moved to its target device for faster patching.
             ModelPatcher.apply_lora(text_encoder, _lora_loader(), lora_prefix),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.
-            ModelPatcher.apply_clip_skip(text_encoder_model, clip_field.skipped_layers),
+            ModelPatcher.apply_clip_skip(text_encoder_info.model, clip_field.skipped_layers),
         ):
-            assert isinstance(text_encoder, (CLIPTextModel, CLIPTextModelWithProjection))
-            text_encoder = cast(CLIPTextModel, text_encoder)
             compel = Compel(
                 tokenizer=tokenizer,
                 text_encoder=text_encoder,
@@ -250,7 +262,7 @@ class SDXLPromptInvocationBase:
     title="SDXL Prompt",
     tags=["sdxl", "compel", "prompt"],
     category="conditioning",
-    version="1.2.0",
+    version="1.0.1",
 )
 class SDXLCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
     """Parse prompt using compel package to conditioning."""
@@ -274,11 +286,6 @@ class SDXLCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
     clip: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection, title="CLIP 1")
     clip2: ClipField = InputField(description=FieldDescriptions.clip, input=Input.Connection, title="CLIP 2")
 
-    mask: Optional[MaskField] = InputField(
-        default=None, description="A mask defining the region that this conditioning prompt applies to."
-    )
-    mask_weight: float = InputField(default=1.0, description="")
-
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> ConditioningOutput:
         c1, c1_pooled, ec1 = self.run_clip_compel(
@@ -340,13 +347,7 @@ class SDXLCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase):
 
         conditioning_name = context.conditioning.save(conditioning_data)
 
-        return ConditioningOutput(
-            conditioning=ConditioningField(
-                conditioning_name=conditioning_name,
-                mask=self.mask,
-                mask_weight=self.mask_weight,
-            )
-        )
+        return ConditioningOutput.build(conditioning_name)
 
 
 @invocation(
@@ -395,7 +396,7 @@ class SDXLRefinerCompelPromptInvocation(BaseInvocation, SDXLPromptInvocationBase
 
         conditioning_name = context.conditioning.save(conditioning_data)
 
-        return ConditioningOutput(conditioning=ConditioningField(conditioning_name=conditioning_name, mask_weight=1.0))
+        return ConditioningOutput.build(conditioning_name)
 
 
 @invocation_output("clip_skip_output")

invokeai/app/invocations/conditioning.py

@@ -1,40 +0,0 @@
-import torch
-
-from invokeai.app.invocations.baseinvocation import (
-    BaseInvocation,
-    InvocationContext,
-    invocation,
-)
-from invokeai.app.invocations.fields import InputField, WithMetadata
-from invokeai.app.invocations.primitives import MaskField, MaskOutput
-
-
-@invocation(
-    "rectangle_mask",
-    title="Create Rectangle Mask",
-    tags=["conditioning"],
-    category="conditioning",
-    version="1.0.0",
-)
-class RectangleMaskInvocation(BaseInvocation, WithMetadata):
-    """Create a rectangular mask."""
-
-    height: int = InputField(description="The height of the entire mask.")
-    width: int = InputField(description="The width of the entire mask.")
-    y_top: int = InputField(description="The top y-coordinate of the rectangular masked region (inclusive).")
-    x_left: int = InputField(description="The left x-coordinate of the rectangular masked region (inclusive).")
-    rectangle_height: int = InputField(description="The height of the rectangular masked region.")
-    rectangle_width: int = InputField(description="The width of the rectangular masked region.")
-
-    def invoke(self, context: InvocationContext) -> MaskOutput:
-        mask = torch.zeros((1, self.height, self.width), dtype=torch.bool)
-        mask[
-            :, self.y_top : self.y_top + self.rectangle_height, self.x_left : self.x_left + self.rectangle_width
-        ] = True
-
-        mask_name = context.tensors.save(mask)
-        return MaskOutput(
-            mask=MaskField(mask_name=mask_name),
-            width=self.width,
-            height=self.height,
-        )

invokeai/app/invocations/fields.py

@@ -194,12 +194,6 @@ class BoardField(BaseModel):
     board_id: str = Field(description="The id of the board")
 
 
-class MaskField(BaseModel):
-    """A mask primitive field."""
-
-    mask_name: str = Field(description="The name of the mask.")
-
-
 class DenoiseMaskField(BaseModel):
     """An inpaint mask field"""
 
@@ -231,12 +225,7 @@ class ConditioningField(BaseModel):
     """A conditioning tensor primitive value"""
 
     conditioning_name: str = Field(description="The name of conditioning tensor")
-    mask: Optional[MaskField] = Field(
-        default=None,
-        description="The bool mask associated with this conditioning tensor. Excluded regions should be set to False, "
-        "included regions should be set to True.",
-    )
-    mask_weight: float = Field(description="")
+    # endregion
 
 
 class MetadataField(RootModel):

invokeai/app/invocations/ip_adapter.py

@@ -93,7 +93,7 @@ class IPAdapterInvocation(BaseInvocation):
         image_encoder_model_id = ip_adapter_info.image_encoder_model_id
         image_encoder_model_name = image_encoder_model_id.split("/")[-1].strip()
         image_encoder_models = context.models.search_by_attrs(
-            name=image_encoder_model_name, base=BaseModelType.Any, type=ModelType.CLIPVision
+            model_name=image_encoder_model_name, base_model=BaseModelType.Any, model_type=ModelType.CLIPVision
         )
         assert len(image_encoder_models) == 1
         image_encoder_model = CLIPVisionModelField(key=image_encoder_models[0].key)

invokeai/app/invocations/latent.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2023 Kyle Schouviller (https://github.com/kyle0654)
-import inspect
+
 import math
 from contextlib import ExitStack
 from functools import singledispatchmethod
@@ -9,7 +9,6 @@ import einops
 import numpy as np
 import numpy.typing as npt
 import torch
-import torchvision
 import torchvision.transforms as T
 from diffusers import AutoencoderKL, AutoencoderTiny
 from diffusers.configuration_utils import ConfigMixin
@@ -56,14 +55,7 @@ from invokeai.backend.lora import LoRAModelRaw
 from invokeai.backend.model_manager import BaseModelType, LoadedModel
 from invokeai.backend.model_patcher import ModelPatcher
 from invokeai.backend.stable_diffusion import PipelineIntermediateState, set_seamless
-from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
-    BasicConditioningInfo,
-    IPAdapterConditioningInfo,
-    Range,
-    SDXLConditioningInfo,
-    TextConditioningData,
-    TextConditioningRegions,
-)
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningData, IPAdapterConditioningInfo
 from invokeai.backend.util.silence_warnings import SilenceWarnings
 
 from ...backend.stable_diffusion.diffusers_pipeline import (
@@ -73,6 +65,7 @@ from ...backend.stable_diffusion.diffusers_pipeline import (
     T2IAdapterData,
     image_resized_to_grid_as_tensor,
 )
+from ...backend.stable_diffusion.diffusion.shared_invokeai_diffusion import PostprocessingSettings
 from ...backend.stable_diffusion.schedulers import SCHEDULER_MAP
 from ...backend.util.devices import choose_precision, choose_torch_device
 from .baseinvocation import (
@@ -180,16 +173,6 @@ class CreateDenoiseMaskInvocation(BaseInvocation):
         )
 
 
-@invocation_output("gradient_mask_output")
-class GradientMaskOutput(BaseInvocationOutput):
-    """Outputs a denoise mask and an image representing the total gradient of the mask."""
-
-    denoise_mask: DenoiseMaskField = OutputField(description="Mask for denoise model run")
-    expanded_mask_area: ImageField = OutputField(
-        description="Image representing the total gradient area of the mask. For paste-back purposes."
-    )
-
-
 @invocation(
     "create_gradient_mask",
     title="Create Gradient Mask",
@@ -210,42 +193,38 @@ class CreateGradientMaskInvocation(BaseInvocation):
     )
 
     @torch.no_grad()
-    def invoke(self, context: InvocationContext) -> GradientMaskOutput:
+    def invoke(self, context: InvocationContext) -> DenoiseMaskOutput:
         mask_image = context.images.get_pil(self.mask.image_name, mode="L")
-        if self.edge_radius > 0:
-            if self.coherence_mode == "Box Blur":
-                blur_mask = mask_image.filter(ImageFilter.BoxBlur(self.edge_radius))
-            else:  # Gaussian Blur OR Staged
-                # Gaussian Blur uses standard deviation. 1/2 radius is a good approximation
-                blur_mask = mask_image.filter(ImageFilter.GaussianBlur(self.edge_radius / 2))
+        if self.coherence_mode == "Box Blur":
+            blur_mask = mask_image.filter(ImageFilter.BoxBlur(self.edge_radius))
+        else:  # Gaussian Blur OR Staged
+            # Gaussian Blur uses standard deviation. 1/2 radius is a good approximation
+            blur_mask = mask_image.filter(ImageFilter.GaussianBlur(self.edge_radius / 2))
 
-            blur_tensor: torch.Tensor = image_resized_to_grid_as_tensor(blur_mask, normalize=False)
+        mask_tensor: torch.Tensor = image_resized_to_grid_as_tensor(mask_image, normalize=False)
+        blur_tensor: torch.Tensor = image_resized_to_grid_as_tensor(blur_mask, normalize=False)
 
-            # redistribute blur so that the original edges are 0 and blur outwards to 1
-            blur_tensor = (blur_tensor - 0.5) * 2
+        # redistribute blur so that the edges are 0 and blur out to 1
+        blur_tensor = (blur_tensor - 0.5) * 2
 
-            threshold = 1 - self.minimum_denoise
-
-            if self.coherence_mode == "Staged":
-                # wherever the blur_tensor is less than fully masked, convert it to threshold
-                blur_tensor = torch.where((blur_tensor < 1) & (blur_tensor > 0), threshold, blur_tensor)
-            else:
-                # wherever the blur_tensor is above threshold but less than 1, drop it to threshold
-                blur_tensor = torch.where((blur_tensor > threshold) & (blur_tensor < 1), threshold, blur_tensor)
+        threshold = 1 - self.minimum_denoise
 
+        if self.coherence_mode == "Staged":
+            # wherever the blur_tensor is masked to any degree, convert it to threshold
+            blur_tensor = torch.where((blur_tensor < 1), threshold, blur_tensor)
         else:
-            blur_tensor: torch.Tensor = image_resized_to_grid_as_tensor(mask_image, normalize=False)
+            # wherever the blur_tensor is above threshold but less than 1, drop it to threshold
+            blur_tensor = torch.where((blur_tensor > threshold) & (blur_tensor < 1), threshold, blur_tensor)
+
+        # multiply original mask to force actually masked regions to 0
+        blur_tensor = mask_tensor * blur_tensor
 
         mask_name = context.tensors.save(tensor=blur_tensor.unsqueeze(1))
 
-        # compute a [0, 1] mask from the blur_tensor
-        expanded_mask = torch.where((blur_tensor < 1), 0, 1)
-        expanded_mask_image = Image.fromarray((expanded_mask.squeeze(0).numpy() * 255).astype(np.uint8), mode="L")
-        expanded_image_dto = context.images.save(expanded_mask_image)
-
-        return GradientMaskOutput(
-            denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=None, gradient=True),
-            expanded_mask_area=ImageField(image_name=expanded_image_dto.image_name),
+        return DenoiseMaskOutput.build(
+            mask_name=mask_name,
+            masked_latents_name=None,
+            gradient=True,
         )
 
 
@@ -291,11 +270,11 @@ def get_scheduler(
 class DenoiseLatentsInvocation(BaseInvocation):
     """Denoises noisy latents to decodable images"""
 
-    positive_conditioning: Union[ConditioningField, list[ConditioningField]] = InputField(
+    positive_conditioning: ConditioningField = InputField(
         description=FieldDescriptions.positive_cond, input=Input.Connection, ui_order=0
     )
-    negative_conditioning: Union[ConditioningField, list[ConditioningField]] = InputField(
-        description=FieldDescriptions.negative_cond, input=Input.Connection, ui_order=0
+    negative_conditioning: ConditioningField = InputField(
+        description=FieldDescriptions.negative_cond, input=Input.Connection, ui_order=1
     )
     noise: Optional[LatentsField] = InputField(
         default=None,
@@ -372,190 +351,41 @@ class DenoiseLatentsInvocation(BaseInvocation):
                 raise ValueError("cfg_scale must be greater than 1")
         return v
 
-    def _get_text_embeddings_and_masks(
-        self,
-        cond_list: list[ConditioningField],
-        context: InvocationContext,
-        device: torch.device,
-        dtype: torch.dtype,
-    ) -> tuple[Union[list[BasicConditioningInfo], list[SDXLConditioningInfo]], list[Optional[torch.Tensor]]]:
-        """Get the text embeddings and masks from the input conditioning fields."""
-        text_embeddings: Union[list[BasicConditioningInfo], list[SDXLConditioningInfo]] = []
-        text_embeddings_masks: list[Optional[torch.Tensor]] = []
-        for cond in cond_list:
-            cond_data = context.conditioning.load(cond.conditioning_name)
-            text_embeddings.append(cond_data.conditionings[0].to(device=device, dtype=dtype))
-
-            mask = cond.mask
-            if mask is not None:
-                mask = context.tensors.load(mask.mask_name)
-            text_embeddings_masks.append(mask)
-
-        return text_embeddings, text_embeddings_masks
-
-    def _preprocess_regional_prompt_mask(
-        self, mask: Optional[torch.Tensor], target_height: int, target_width: int
-    ) -> torch.Tensor:
-        """Preprocess a regional prompt mask to match the target height and width.
-
-        If mask is None, returns a mask of all ones with the target height and width.
-        If mask is not None, resizes the mask to the target height and width using nearest neighbor interpolation.
-
-        Returns:
-            torch.Tensor: The processed mask. dtype: torch.bool, shape: (1, 1, target_height, target_width).
-        """
-        if mask is None:
-            return torch.ones((1, 1, target_height, target_width), dtype=torch.bool)
-
-        tf = torchvision.transforms.Resize(
-            (target_height, target_width), interpolation=torchvision.transforms.InterpolationMode.NEAREST
-        )
-        mask = mask.unsqueeze(0)  # Shape: (1, h, w) -> (1, 1, h, w)
-        mask = tf(mask)
-
-        return mask
-
-    def concat_regional_text_embeddings(
-        self,
-        text_conditionings: Union[list[BasicConditioningInfo], list[SDXLConditioningInfo]],
-        masks: Optional[list[Optional[torch.Tensor]]],
-        conditioning_fields: list[ConditioningField],
-        latent_height: int,
-        latent_width: int,
-    ) -> tuple[Union[BasicConditioningInfo, SDXLConditioningInfo], Optional[TextConditioningRegions]]:
-        """Concatenate regional text embeddings into a single embedding and track the region masks accordingly."""
-        if masks is None:
-            masks = [None] * len(text_conditionings)
-        assert len(text_conditionings) == len(masks)
-
-        is_sdxl = type(text_conditionings[0]) is SDXLConditioningInfo
-
-        all_masks_are_none = all(mask is None for mask in masks)
-
-        text_embedding = []
-        pooled_embedding = None
-        add_time_ids = None
-        cur_text_embedding_len = 0
-        processed_masks = []
-        embedding_ranges = []
-        extra_conditioning = None
-
-        for prompt_idx, text_embedding_info in enumerate(text_conditionings):
-            mask = masks[prompt_idx]
-            if (
-                text_embedding_info.extra_conditioning is not None
-                and text_embedding_info.extra_conditioning.wants_cross_attention_control
-            ):
-                extra_conditioning = text_embedding_info.extra_conditioning
-
-            if is_sdxl:
-                # We choose a random SDXLConditioningInfo's pooled_embeds and add_time_ids here, with a preference for
-                # prompts without a mask. We prefer prompts without a mask, because they are more likely to contain
-                # global prompt information.  In an ideal case, there should be exactly one global prompt without a
-                # mask, but we don't enforce this.
-
-                # HACK(ryand): The fact that we have to choose a single pooled_embedding and add_time_ids here is a
-                # fundamental interface issue. The SDXL Compel nodes are not designed to be used in the way that we use
-                # them for regional prompting. Ideally, the DenoiseLatents invocation should accept a single
-                # pooled_embeds tensor and a list of standard text embeds with region masks. This change would be a
-                # pretty major breaking change to a popular node, so for now we use this hack.
-                if pooled_embedding is None or mask is None:
-                    pooled_embedding = text_embedding_info.pooled_embeds
-                if add_time_ids is None or mask is None:
-                    add_time_ids = text_embedding_info.add_time_ids
-
-            text_embedding.append(text_embedding_info.embeds)
-            if not all_masks_are_none:
-                # embedding_ranges.append(
-                #     Range(
-                #         start=cur_text_embedding_len, end=cur_text_embedding_len + text_embedding_info.embeds.shape[1]
-                #     )
-                # )
-                # HACK(ryand): Contrary to its name, tokens_count_including_eos_bos does not seem to include eos and bos
-                # in the count.
-                embedding_ranges.append(
-                    Range(
-                        start=cur_text_embedding_len + 1,
-                        end=cur_text_embedding_len
-                        + text_embedding_info.extra_conditioning.tokens_count_including_eos_bos,
-                    )
-                )
-                processed_masks.append(self._preprocess_regional_prompt_mask(mask, latent_height, latent_width))
-
-            cur_text_embedding_len += text_embedding_info.embeds.shape[1]
-
-        text_embedding = torch.cat(text_embedding, dim=1)
-        assert len(text_embedding.shape) == 3  # batch_size, seq_len, token_len
-
-        regions = None
-        if not all_masks_are_none:
-            regions = TextConditioningRegions(
-                masks=torch.cat(processed_masks, dim=1),
-                ranges=embedding_ranges,
-                mask_weights=[x.mask_weight for x in conditioning_fields],
-            )
-
-        if extra_conditioning is not None and len(text_conditionings) > 1:
-            raise ValueError(
-                "Prompt-to-prompt cross-attention control (a.k.a. `swap()`) is not supported when using multiple "
-                "prompts."
-            )
-
-        if is_sdxl:
-            return SDXLConditioningInfo(
-                embeds=text_embedding,
-                extra_conditioning=extra_conditioning,
-                pooled_embeds=pooled_embedding,
-                add_time_ids=add_time_ids,
-            ), regions
-        return BasicConditioningInfo(
-            embeds=text_embedding,
-            extra_conditioning=extra_conditioning,
-        ), regions
-
     def get_conditioning_data(
         self,
         context: InvocationContext,
+        scheduler: Scheduler,
         unet: UNet2DConditionModel,
-        latent_height: int,
-        latent_width: int,
-    ) -> TextConditioningData:
-        # Normalize self.positive_conditioning and self.negative_conditioning to lists.
-        cond_list = self.positive_conditioning
-        if not isinstance(cond_list, list):
-            cond_list = [cond_list]
-        uncond_list = self.negative_conditioning
-        if not isinstance(uncond_list, list):
-            uncond_list = [uncond_list]
+        seed: int,
+    ) -> ConditioningData:
+        positive_cond_data = context.conditioning.load(self.positive_conditioning.conditioning_name)
+        c = positive_cond_data.conditionings[0].to(device=unet.device, dtype=unet.dtype)
+        extra_conditioning_info = c.extra_conditioning
 
-        cond_text_embeddings, cond_text_embedding_masks = self._get_text_embeddings_and_masks(
-            cond_list, context, unet.device, unet.dtype
-        )
+        negative_cond_data = context.conditioning.load(self.negative_conditioning.conditioning_name)
+        uc = negative_cond_data.conditionings[0].to(device=unet.device, dtype=unet.dtype)
 
-        uncond_text_embeddings, uncond_text_embedding_masks = self._get_text_embeddings_and_masks(
-            uncond_list, context, unet.device, unet.dtype
-        )
-        cond_text_embedding, cond_regions = self.concat_regional_text_embeddings(
-            text_conditionings=cond_text_embeddings,
-            masks=cond_text_embedding_masks,
-            conditioning_fields=cond_list,
-            latent_height=latent_height,
-            latent_width=latent_width,
-        )
-        uncond_text_embedding, uncond_regions = self.concat_regional_text_embeddings(
-            text_conditionings=uncond_text_embeddings,
-            masks=uncond_text_embedding_masks,
-            conditioning_fields=uncond_list,
-            latent_height=latent_height,
-            latent_width=latent_width,
-        )
-        conditioning_data = TextConditioningData(
-            uncond_text=uncond_text_embedding,
-            cond_text=cond_text_embedding,
-            uncond_regions=uncond_regions,
-            cond_regions=cond_regions,
+        conditioning_data = ConditioningData(
+            unconditioned_embeddings=uc,
+            text_embeddings=c,
             guidance_scale=self.cfg_scale,
             guidance_rescale_multiplier=self.cfg_rescale_multiplier,
+            extra=extra_conditioning_info,
+            postprocessing_settings=PostprocessingSettings(
+                threshold=0.0,  # threshold,
+                warmup=0.2,  # warmup,
+                h_symmetry_time_pct=None,  # h_symmetry_time_pct,
+                v_symmetry_time_pct=None,  # v_symmetry_time_pct,
+            ),
+        )
+
+        conditioning_data = conditioning_data.add_scheduler_args_if_applicable(  # FIXME
+            scheduler,
+            # for ddim scheduler
+            eta=0.0,  # ddim_eta
+            # for ancestral and sde schedulers
+            # flip all bits to have noise different from initial
+            generator=torch.Generator(device=unet.device).manual_seed(seed ^ 0xFFFFFFFF),
         )
         return conditioning_data
 
@@ -661,6 +491,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
         self,
         context: InvocationContext,
         ip_adapter: Optional[Union[IPAdapterField, list[IPAdapterField]]],
+        conditioning_data: ConditioningData,
         exit_stack: ExitStack,
     ) -> Optional[list[IPAdapterData]]:
         """If IP-Adapter is enabled, then this function loads the requisite models, and adds the image prompt embeddings
@@ -677,6 +508,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
             return None
 
         ip_adapter_data_list = []
+        conditioning_data.ip_adapter_conditioning = []
         for single_ip_adapter in ip_adapter:
             ip_adapter_model: Union[IPAdapter, IPAdapterPlus] = exit_stack.enter_context(
                 context.models.load(key=single_ip_adapter.ip_adapter_model.key)
@@ -699,13 +531,16 @@ class DenoiseLatentsInvocation(BaseInvocation):
                     single_ipa_images, image_encoder_model
                 )
 
+                conditioning_data.ip_adapter_conditioning.append(
+                    IPAdapterConditioningInfo(image_prompt_embeds, uncond_image_prompt_embeds)
+                )
+
             ip_adapter_data_list.append(
                 IPAdapterData(
                     ip_adapter_model=ip_adapter_model,
                     weight=single_ip_adapter.weight,
                     begin_step_percent=single_ip_adapter.begin_step_percent,
                     end_step_percent=single_ip_adapter.end_step_percent,
-                    ip_adapter_conditioning=IPAdapterConditioningInfo(image_prompt_embeds, uncond_image_prompt_embeds),
                 )
             )
 
@@ -795,7 +630,6 @@ class DenoiseLatentsInvocation(BaseInvocation):
         steps: int,
         denoising_start: float,
         denoising_end: float,
-        seed: int,
     ) -> Tuple[int, List[int], int]:
         assert isinstance(scheduler, ConfigMixin)
         if scheduler.config.get("cpu_only", False):
@@ -824,15 +658,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
         timesteps = timesteps[t_start_idx : t_start_idx + t_end_idx]
         num_inference_steps = len(timesteps) // scheduler.order
 
-        scheduler_step_kwargs = {}
-        scheduler_step_signature = inspect.signature(scheduler.step)
-        if "generator" in scheduler_step_signature.parameters:
-            # At some point, someone decided that schedulers that accept a generator should use the original seed with
-            # all bits flipped. I don't know the original rationale for this, but now we must keep it like this for
-            # reproducibility.
-            scheduler_step_kwargs = {"generator": torch.Generator(device=device).manual_seed(seed ^ 0xFFFFFFFF)}
-
-        return num_inference_steps, timesteps, init_timestep, scheduler_step_kwargs
+        return num_inference_steps, timesteps, init_timestep
 
     def prep_inpaint_mask(
         self, context: InvocationContext, latents: torch.Tensor
@@ -925,10 +751,7 @@ class DenoiseLatentsInvocation(BaseInvocation):
                 )
 
                 pipeline = self.create_pipeline(unet, scheduler)
-                _, _, latent_height, latent_width = latents.shape
-                conditioning_data = self.get_conditioning_data(
-                    context=context, unet=unet, latent_height=latent_height, latent_width=latent_width
-                )
+                conditioning_data = self.get_conditioning_data(context, scheduler, unet, seed)
 
                 controlnet_data = self.prep_control_data(
                     context=context,
@@ -942,19 +765,22 @@ class DenoiseLatentsInvocation(BaseInvocation):
                 ip_adapter_data = self.prep_ip_adapter_data(
                     context=context,
                     ip_adapter=self.ip_adapter,
+                    conditioning_data=conditioning_data,
                     exit_stack=exit_stack,
                 )
 
-                num_inference_steps, timesteps, init_timestep, scheduler_step_kwargs = self.init_scheduler(
+                num_inference_steps, timesteps, init_timestep = self.init_scheduler(
                     scheduler,
                     device=unet.device,
                     steps=self.steps,
                     denoising_start=self.denoising_start,
                     denoising_end=self.denoising_end,
-                    seed=seed,
                 )
 
-                result_latents = pipeline.latents_from_embeddings(
+                (
+                    result_latents,
+                    result_attention_map_saver,
+                ) = pipeline.latents_from_embeddings(
                     latents=latents,
                     timesteps=timesteps,
                     init_timestep=init_timestep,
@@ -964,7 +790,6 @@ class DenoiseLatentsInvocation(BaseInvocation):
                     masked_latents=masked_latents,
                     gradient_mask=gradient_mask,
                     num_inference_steps=num_inference_steps,
-                    scheduler_step_kwargs=scheduler_step_kwargs,
                     conditioning_data=conditioning_data,
                     control_data=controlnet_data,
                     ip_adapter_data=ip_adapter_data,

invokeai/app/invocations/primitives.py

@@ -14,7 +14,6 @@ from invokeai.app.invocations.fields import (
     Input,
     InputField,
     LatentsField,
-    MaskField,
     OutputField,
     UIComponent,
 )
@@ -230,18 +229,6 @@ class StringCollectionInvocation(BaseInvocation):
 # region Image
 
 
-@invocation_output("mask_output")
-class MaskOutput(BaseInvocationOutput):
-    """A torch mask tensor.
-    dtype: torch.bool
-    shape: (1, height, width).
-    """
-
-    mask: MaskField = OutputField(description="The mask.")
-    width: int = OutputField(description="The width of the mask in pixels.")
-    height: int = OutputField(description="The height of the mask in pixels.")
-
-
 @invocation_output("image_output")
 class ImageOutput(BaseInvocationOutput):
     """Base class for nodes that output a single image"""
@@ -427,6 +414,10 @@ class ConditioningOutput(BaseInvocationOutput):
 
     conditioning: ConditioningField = OutputField(description=FieldDescriptions.cond)
 
+    @classmethod
+    def build(cls, conditioning_name: str) -> "ConditioningOutput":
+        return cls(conditioning=ConditioningField(conditioning_name=conditioning_name))
+
 
 @invocation_output("conditioning_collection_output")
 class ConditioningCollectionOutput(BaseInvocationOutput):

invokeai/app/services/config/config_default.py

@@ -166,7 +166,6 @@ two configs are kept in separate sections of the config file:
      ...
 
 """
-
 from __future__ import annotations
 
 import os

invokeai/app/services/download/__init__.py

@@ -1,5 +1,4 @@
 """Init file for download queue."""
-
 from .download_base import DownloadJob, DownloadJobStatus, DownloadQueueServiceBase, UnknownJobIDException
 from .download_default import DownloadQueueService, TqdmProgress
 

invokeai/app/services/download/download_default.py

@@ -224,6 +224,7 @@ class DownloadQueueService(DownloadQueueServiceBase):
                 job.job_started = get_iso_timestamp()
                 self._do_download(job)
                 self._signal_job_complete(job)
+
             except (OSError, HTTPError) as excp:
                 job.error_type = excp.__class__.__name__ + f"({str(excp)})"
                 job.error = traceback.format_exc()

invokeai/app/services/model_install/model_install_base.py

@@ -28,7 +28,6 @@ class InstallStatus(str, Enum):
 
     WAITING = "waiting"  # waiting to be dequeued
     DOWNLOADING = "downloading"  # downloading of model files in process
-    DOWNLOADS_DONE = "downloads_done"  # downloading done, waiting to run
     RUNNING = "running"  # being processed
     COMPLETED = "completed"  # finished running
     ERROR = "error"  # terminated with an error message
@@ -230,11 +229,6 @@ class ModelInstallJob(BaseModel):
         """Return true if job is downloading."""
         return self.status == InstallStatus.DOWNLOADING
 
-    @property
-    def downloads_done(self) -> bool:
-        """Return true if job's downloads ae done."""
-        return self.status == InstallStatus.DOWNLOADS_DONE
-
     @property
     def running(self) -> bool:
         """Return true if job is running."""

invokeai/app/services/model_install/model_install_default.py

@@ -7,6 +7,7 @@ import time
 from hashlib import sha256
 from pathlib import Path
 from queue import Empty, Queue
+from random import randbytes
 from shutil import copyfile, copytree, move, rmtree
 from tempfile import mkdtemp
 from typing import Any, Dict, List, Optional, Set, Union
@@ -20,7 +21,6 @@ from invokeai.app.services.download import DownloadJob, DownloadQueueServiceBase
 from invokeai.app.services.events.events_base import EventServiceBase
 from invokeai.app.services.invoker import Invoker
 from invokeai.app.services.model_records import DuplicateModelException, ModelRecordServiceBase
-from invokeai.app.util.misc import uuid_string
 from invokeai.backend.model_manager.config import (
     AnyModelConfig,
     BaseModelType,
@@ -28,6 +28,7 @@ from invokeai.backend.model_manager.config import (
     ModelRepoVariant,
     ModelType,
 )
+from invokeai.backend.model_manager.hash import FastModelHash
 from invokeai.backend.model_manager.metadata import (
     AnyModelRepoMetadata,
     CivitaiMetadataFetch,
@@ -150,9 +151,9 @@ class ModelInstallService(ModelInstallServiceBase):
         config = config or {}
         if not config.get("source"):
             config["source"] = model_path.resolve().as_posix()
-        config["key"] = config.get("key", uuid_string())
 
         info: AnyModelConfig = self._probe_model(Path(model_path), config)
+        old_hash = info.current_hash
 
         if preferred_name := config.get("name"):
             preferred_name = Path(preferred_name).with_suffix(model_path.suffix)
@@ -166,6 +167,8 @@ class ModelInstallService(ModelInstallServiceBase):
             raise DuplicateModelException(
                 f"A model named {model_path.name} is already installed at {dest_path.as_posix()}"
             ) from excp
+        new_hash = FastModelHash.hash(new_path)
+        assert new_hash == old_hash, f"{model_path}: Model hash changed during installation, possibly corrupted."
 
         return self._register(
             new_path,
@@ -178,14 +181,13 @@ class ModelInstallService(ModelInstallServiceBase):
         source: str,
         config: Optional[Dict[str, Any]] = None,
         access_token: Optional[str] = None,
-        inplace: bool = False,
     ) -> ModelInstallJob:
         variants = "|".join(ModelRepoVariant.__members__.values())
         hf_repoid_re = f"^([^/:]+/[^/:]+)(?::({variants})?(?::/?([^:]+))?)?$"
         source_obj: Optional[StringLikeSource] = None
 
         if Path(source).exists():  # A local file or directory
-            source_obj = LocalModelSource(path=Path(source), inplace=inplace)
+            source_obj = LocalModelSource(path=Path(source))
         elif match := re.match(hf_repoid_re, source):
             source_obj = HFModelSource(
                 repo_id=match.group(1),
@@ -280,9 +282,9 @@ class ModelInstallService(ModelInstallServiceBase):
         self._logger.info("Model installer (re)initialized")
 
     def scan_directory(self, scan_dir: Path, install: bool = False) -> List[str]:  # noqa D102
-        self._cached_model_paths = {Path(x.path).absolute() for x in self.record_store.all_models()}
+        self._cached_model_paths = {Path(x.path) for x in self.record_store.all_models()}
         callback = self._scan_install if install else self._scan_register
-        search = ModelSearch(on_model_found=callback, config=self._app_config)
+        search = ModelSearch(on_model_found=callback)
         self._models_installed.clear()
         search.search(scan_dir)
         return list(self._models_installed)
@@ -368,7 +370,7 @@ class ModelInstallService(ModelInstallServiceBase):
                     self._signal_job_errored(job)
 
                 elif (
-                    job.waiting or job.downloads_done
+                    job.waiting or job.downloading
                 ):  # local jobs will be in waiting state, remote jobs will be downloading state
                     job.total_bytes = self._stat_size(job.local_path)
                     job.bytes = job.total_bytes
@@ -446,7 +448,7 @@ class ModelInstallService(ModelInstallServiceBase):
                     installed.update(self.scan_directory(models_dir))
             self._logger.info(f"{len(installed)} new models registered; {len(defunct_models)} unregistered")
 
-    def _sync_model_path(self, key: str) -> AnyModelConfig:
+    def _sync_model_path(self, key: str, ignore_hash_change: bool = False) -> AnyModelConfig:
         """
         Move model into the location indicated by its basetype, type and name.
 
@@ -467,7 +469,14 @@ class ModelInstallService(ModelInstallServiceBase):
         new_path = models_dir / model.base.value / model.type.value / model.name
         self._logger.info(f"Moving {model.name} to {new_path}.")
         new_path = self._move_model(old_path, new_path)
+        new_hash = FastModelHash.hash(new_path)
         model.path = new_path.relative_to(models_dir).as_posix()
+        if model.current_hash != new_hash:
+            assert (
+                ignore_hash_change
+            ), f"{model.name}: Model hash changed during installation, model is possibly corrupted"
+            model.current_hash = new_hash
+            self._logger.info(f"Model has new hash {model.current_hash}, but will continue to be identified by {key}")
         self.record_store.update_model(key, model)
         return model
 
@@ -527,17 +536,16 @@ class ModelInstallService(ModelInstallServiceBase):
                 setattr(info, key, value)
         return info
 
+    def _create_key(self) -> str:
+        return sha256(randbytes(100)).hexdigest()[0:32]
+
     def _register(
         self, model_path: Path, config: Optional[Dict[str, Any]] = None, info: Optional[AnyModelConfig] = None
     ) -> str:
-        # Note that we may be passed a pre-populated AnyModelConfig object,
-        # in which case the key field should have been populated by the caller (e.g. in `install_path`).
-        config["key"] = config.get("key", uuid_string())
+        key = self._create_key()
+        if config and not config.get("key", None):
+            config["key"] = key
         info = info or ModelProbe.probe(model_path, config)
-        override_key: Optional[str] = config.get("key") if config else None
-
-        assert info.original_hash  # always assigned by probe()
-        info.key = override_key or info.original_hash
 
         model_path = model_path.absolute()
         if model_path.is_relative_to(self.app_config.models_path):
@@ -741,8 +749,8 @@ class ModelInstallService(ModelInstallServiceBase):
             self._download_cache.pop(download_job.source, None)
 
             # are there any more active jobs left in this task?
-            if install_job.downloading and all(x.complete for x in install_job.download_parts):
-                install_job.status = InstallStatus.DOWNLOADS_DONE
+            if all(x.complete for x in install_job.download_parts):
+                #  now enqueue job for actual installation into the models directory
                 self._install_queue.put(install_job)
 
             # Let other threads know that the number of downloads has changed

invokeai/app/services/model_metadata/metadata_store_base.py

@@ -7,21 +7,23 @@ from abc import ABC, abstractmethod
 from typing import List, Optional, Set, Tuple
 
 from pydantic import Field
-
 from invokeai.app.util.model_exclude_null import BaseModelExcludeNull
+
 from invokeai.backend.model_manager.metadata import AnyModelRepoMetadata
 from invokeai.backend.model_manager.metadata.metadata_base import ModelDefaultSettings
 
-
 class ModelMetadataChanges(BaseModelExcludeNull, extra="allow"):
     """A set of changes to apply to model metadata.
+
     Only limited changes are valid:
+      - `trigger_phrases`: the list of trigger phrases for this model
       - `default_settings`: the user-configured default settings for this model
     """
 
-    default_settings: Optional[ModelDefaultSettings] = Field(
-        default=None, description="The user-configured default settings for this model"
-    )
+    trigger_phrases: Optional[List[str]] = Field(default=None, description="The model's list of trigger phrases")
+    """The model's list of trigger phrases"""
+
+    default_settings: Optional[ModelDefaultSettings] = Field(default=None, description="The user-configured default settings for this model")
     """The user-configured default settings for this model"""
 
 

invokeai/app/services/model_metadata/metadata_store_sql.py

@@ -115,6 +115,8 @@ class ModelMetadataStoreSQL(ModelMetadataStoreBase):
             except sqlite3.Error as e:
                 self._db.conn.rollback()
                 raise e
+            except Exception as e:
+                raise e
 
         return self.get_metadata(model_key)
 
@@ -182,7 +184,7 @@ class ModelMetadataStoreSQL(ModelMetadataStoreBase):
     def _update_tags(self, model_key: str, tags: Optional[Set[str]]) -> None:
         """Update tags for the model referenced by model_key."""
         if tags:
-            # remove previous tags from this model
+        # remove previous tags from this model
             self._cursor.execute(
                 """--sql
                 DELETE FROM model_tags

invokeai/app/services/model_records/__init__.py

@@ -1,5 +1,4 @@
 """Init file for model record services."""
-
 from .model_records_base import (  # noqa F401
     DuplicateModelException,
     InvalidModelException,

invokeai/app/services/model_records/model_records_sql.py

@@ -39,6 +39,7 @@ Typical usage:
   configs = store.search_by_attr(base_model='sd-2', model_type='main')
 """
 
+
 import json
 import sqlite3
 from math import ceil

invokeai/app/services/session_processor/session_processor_default.py

@@ -200,7 +200,6 @@ class DefaultSessionProcessor(SessionProcessorBase):
                                 self._invoker.services.logger.error(
                                     f"Error while invoking session {self._queue_item.session_id}, invocation {self._invocation.id} ({self._invocation.get_type()}):\n{e}"
                                 )
-                                self._invoker.services.logger.error(error)
 
                                 # Send error event
                                 self._invoker.services.events.emit_invocation_error(

invokeai/app/services/shared/invocation_context.py

@@ -65,86 +65,75 @@ class InvocationContextInterface:
 
 class BoardsInterface(InvocationContextInterface):
     def create(self, board_name: str) -> BoardDTO:
-        """Creates a board.
+        """
+        Creates a board.
 
-        Args:
-            board_name: The name of the board to create.
-
-        Returns:
-            The created board DTO.
+        :param board_name: The name of the board to create.
         """
         return self._services.boards.create(board_name)
 
     def get_dto(self, board_id: str) -> BoardDTO:
-        """Gets a board DTO.
+        """
+        Gets a board DTO.
 
-        Args:
-            board_id: The ID of the board to get.
-
-        Returns:
-            The board DTO.
+        :param board_id: The ID of the board to get.
         """
         return self._services.boards.get_dto(board_id)
 
     def get_all(self) -> list[BoardDTO]:
-        """Gets all boards.
-
-        Returns:
-            A list of all boards.
+        """
+        Gets all boards.
         """
         return self._services.boards.get_all()
 
     def add_image_to_board(self, board_id: str, image_name: str) -> None:
-        """Adds an image to a board.
+        """
+        Adds an image to a board.
 
-        Args:
-            board_id: The ID of the board to add the image to.
-            image_name: The name of the image to add to the board.
+        :param board_id: The ID of the board to add the image to.
+        :param image_name: The name of the image to add to the board.
         """
         return self._services.board_images.add_image_to_board(board_id, image_name)
 
     def get_all_image_names_for_board(self, board_id: str) -> list[str]:
-        """Gets all image names for a board.
+        """
+        Gets all image names for a board.
 
-        Args:
-            board_id: The ID of the board to get the image names for.
-
-        Returns:
-            A list of all image names for the board.
+        :param board_id: The ID of the board to get the image names for.
         """
         return self._services.board_images.get_all_board_image_names_for_board(board_id)
 
 
 class LoggerInterface(InvocationContextInterface):
     def debug(self, message: str) -> None:
-        """Logs a debug message.
+        """
+        Logs a debug message.
 
-        Args:
-            message: The message to log.
+        :param message: The message to log.
         """
         self._services.logger.debug(message)
 
     def info(self, message: str) -> None:
-        """Logs an info message.
+        """
+        Logs an info message.
 
-        Args:
-            message: The message to log.
+        :param message: The message to log.
         """
         self._services.logger.info(message)
 
     def warning(self, message: str) -> None:
-        """Logs a warning message.
+        """
+        Logs a warning message.
 
-        Args:
-            message: The message to log.
+        :param message: The message to log.
         """
         self._services.logger.warning(message)
 
     def error(self, message: str) -> None:
-        """Logs an error message.
+        """
+        Logs an error message.
 
-        Args:
-            message: The message to log.
+        :param message: The message to log.
         """
         self._services.logger.error(message)
 
@@ -157,23 +146,20 @@ class ImagesInterface(InvocationContextInterface):
         image_category: ImageCategory = ImageCategory.GENERAL,
         metadata: Optional[MetadataField] = None,
     ) -> ImageDTO:
-        """Saves an image, returning its DTO.
+        """
+        Saves an image, returning its DTO.
 
         If the current queue item has a workflow or metadata, it is automatically saved with the image.
 
-        Args:
-            image: The image to save, as a PIL image.
-            board_id: The board ID to add the image to, if it should be added. It the invocation \
+        :param image: The image to save, as a PIL image.
+        :param board_id: The board ID to add the image to, if it should be added. It the invocation \
             inherits from `WithBoard`, that board will be used automatically. **Use this only if \
             you want to override or provide a board manually!**
-            image_category: The category of the image. Only the GENERAL category is added \
+        :param image_category: The category of the image. Only the GENERAL category is added \
             to the gallery.
-            metadata: The metadata to save with the image, if it should have any. If the \
+        :param metadata: The metadata to save with the image, if it should have any. If the \
             invocation inherits from `WithMetadata`, that metadata will be used automatically. \
             **Use this only if you want to override or provide metadata manually!**
-
-        Returns:
-            The saved image DTO.
         """
 
         # If `metadata` is provided directly, use that. Else, use the metadata provided by `WithMetadata`, falling back to None.
@@ -203,14 +189,11 @@ class ImagesInterface(InvocationContextInterface):
         )
 
     def get_pil(self, image_name: str, mode: IMAGE_MODES | None = None) -> Image:
-        """Gets an image as a PIL Image object.
+        """
+        Gets an image as a PIL Image object.
 
-        Args:
-            image_name: The name of the image to get.
-            mode: The color mode to convert the image to. If None, the original mode is used.
-
-        Returns:
-            The image as a PIL Image object.
+        :param image_name: The name of the image to get.
+        :param mode: The color mode to convert the image to. If None, the original mode is used.
         """
         image = self._services.images.get_pil_image(image_name)
         if mode and mode != image.mode:
@@ -223,76 +206,58 @@ class ImagesInterface(InvocationContextInterface):
         return image
 
     def get_metadata(self, image_name: str) -> Optional[MetadataField]:
-        """Gets an image's metadata, if it has any.
+        """
+        Gets an image's metadata, if it has any.
 
-        Args:
-            image_name: The name of the image to get the metadata for.
-
-        Returns:
-            The image's metadata, if it has any.
+        :param image_name: The name of the image to get the metadata for.
         """
         return self._services.images.get_metadata(image_name)
 
     def get_dto(self, image_name: str) -> ImageDTO:
-        """Gets an image as an ImageDTO object.
+        """
+        Gets an image as an ImageDTO object.
 
-        Args:
-            image_name: The name of the image to get.
-
-        Returns:
-            The image as an ImageDTO object.
+        :param image_name: The name of the image to get.
         """
         return self._services.images.get_dto(image_name)
 
 
 class TensorsInterface(InvocationContextInterface):
     def save(self, tensor: Tensor) -> str:
-        """Saves a tensor, returning its name.
+        """
+        Saves a tensor, returning its name.
 
-        Args:
-            tensor: The tensor to save.
-
-        Returns:
-            The name of the saved tensor.
+        :param tensor: The tensor to save.
         """
 
         name = self._services.tensors.save(obj=tensor)
         return name
 
     def load(self, name: str) -> Tensor:
-        """Loads a tensor by name.
+        """
+        Loads a tensor by name.
 
-        Args:
-            name: The name of the tensor to load.
-
-        Returns:
-            The loaded tensor.
+        :param name: The name of the tensor to load.
         """
         return self._services.tensors.load(name)
 
 
 class ConditioningInterface(InvocationContextInterface):
     def save(self, conditioning_data: ConditioningFieldData) -> str:
-        """Saves a conditioning data object, returning its name.
+        """
+        Saves a conditioning data object, returning its name.
 
-        Args:
-            conditioning_data: The conditioning data to save.
-
-        Returns:
-            The name of the saved conditioning data.
+        :param conditioning_data: The conditioning data to save.
         """
 
         name = self._services.conditioning.save(obj=conditioning_data)
         return name
 
     def load(self, name: str) -> ConditioningFieldData:
-        """Loads conditioning data by name.
+        """
+        Loads conditioning data by name.
 
-        Args:
-            name: The name of the conditioning data to load.
-
-        Returns:
-            The loaded conditioning data.
+        :param name: The name of the conditioning data to load.
         """
 
         return self._services.conditioning.load(name)
@@ -300,25 +265,20 @@ class ConditioningInterface(InvocationContextInterface):
 
 class ModelsInterface(InvocationContextInterface):
     def exists(self, key: str) -> bool:
-        """Checks if a model exists.
+        """
+        Checks if a model exists.
 
-        Args:
-            key: The key of the model.
-
-        Returns:
-            True if the model exists, False if not.
+        :param key: The key of the model.
         """
         return self._services.model_manager.store.exists(key)
 
     def load(self, key: str, submodel_type: Optional[SubModelType] = None) -> LoadedModel:
-        """Loads a model.
+        """
+        Loads a model.
 
-        Args:
-            key: The key of the model.
-            submodel_type: The submodel of the model to get.
-
-        Returns:
-            An object representing the loaded model.
+        :param key: The key of the model.
+        :param submodel_type: The submodel of the model to get.
+        :returns: An object representing the loaded model.
         """
 
         # The model manager emits events as it loads the model. It needs the context data to build
@@ -329,95 +289,75 @@ class ModelsInterface(InvocationContextInterface):
         )
 
     def load_by_attrs(
-        self, name: str, base: BaseModelType, type: ModelType, submodel_type: Optional[SubModelType] = None
+        self, model_name: str, base_model: BaseModelType, model_type: ModelType, submodel: Optional[SubModelType] = None
     ) -> LoadedModel:
-        """Loads a model by its attributes.
+        """
+        Loads a model by its attributes.
 
-        Args:
-            name: Name of the model.
-            base: The models' base type, e.g. `BaseModelType.StableDiffusion1`, `BaseModelType.StableDiffusionXL`, etc.
-            type: Type of the model, e.g. `ModelType.Main`, `ModelType.Vae`, etc.
-            submodel_type: The type of submodel to load, e.g. `SubModelType.UNet`, `SubModelType.TextEncoder`, etc. Only main
-            models have submodels.
-
-        Returns:
-            An object representing the loaded model.
+        :param model_name: Name of to be fetched.
+        :param base_model: Base model
+        :param model_type: Type of the model
+        :param submodel: For main (pipeline models), the submodel to fetch
         """
         return self._services.model_manager.load_model_by_attr(
-            model_name=name,
-            base_model=base,
-            model_type=type,
-            submodel=submodel_type,
+            model_name=model_name,
+            base_model=base_model,
+            model_type=model_type,
+            submodel=submodel,
             context_data=self._data,
         )
 
     def get_config(self, key: str) -> AnyModelConfig:
-        """Gets a model's config.
+        """
+        Gets a model's info, an dict-like object.
 
-        Args:
-            key: The key of the model.
-
-        Returns:
-            The model's config.
+        :param key: The key of the model.
         """
         return self._services.model_manager.store.get_model(key=key)
 
     def get_metadata(self, key: str) -> Optional[AnyModelRepoMetadata]:
-        """Gets a model's metadata, if it has any.
+        """
+        Gets a model's metadata, if it has any.
 
-        Args:
-            key: The key of the model.
-
-        Returns:
-            The model's metadata, if it has any.
+        :param key: The key of the model.
         """
         return self._services.model_manager.store.get_metadata(key=key)
 
     def search_by_path(self, path: Path) -> list[AnyModelConfig]:
-        """Searches for models by path.
+        """
+        Searches for models by path.
 
-        Args:
-            path: The path to search for.
-
-        Returns:
-            A list of models that match the path.
+        :param path: The path to search for.
         """
         return self._services.model_manager.store.search_by_path(path)
 
     def search_by_attrs(
         self,
-        name: Optional[str] = None,
-        base: Optional[BaseModelType] = None,
-        type: Optional[ModelType] = None,
-        format: Optional[ModelFormat] = None,
+        model_name: Optional[str] = None,
+        base_model: Optional[BaseModelType] = None,
+        model_type: Optional[ModelType] = None,
+        model_format: Optional[ModelFormat] = None,
     ) -> list[AnyModelConfig]:
-        """Searches for models by attributes.
+        """
+        Searches for models by attributes.
 
-        Args:
-            name: The name to search for (exact match).
-            base: The base to search for, e.g. `BaseModelType.StableDiffusion1`, `BaseModelType.StableDiffusionXL`, etc.
-            type: Type type of model to search for, e.g. `ModelType.Main`, `ModelType.Vae`, etc.
-            format: The format of model to search for, e.g. `ModelFormat.Checkpoint`, `ModelFormat.Diffusers`, etc.
-
-        Returns:
-            A list of models that match the attributes.
+        :param model_name: Name of to be fetched.
+        :param base_model: Base model
+        :param model_type: Type of the model
+        :param submodel: For main (pipeline models), the submodel to fetch
         """
 
         return self._services.model_manager.store.search_by_attr(
-            model_name=name,
-            base_model=base,
-            model_type=type,
-            model_format=format,
+            model_name=model_name,
+            base_model=base_model,
+            model_type=model_type,
+            model_format=model_format,
         )
 
 
 class ConfigInterface(InvocationContextInterface):
     def get(self) -> InvokeAIAppConfig:
-        """Gets the app's config.
-
-        Returns:
-            The app's config.
-        """
+        """Gets the app's config."""
 
         return self._services.configuration.get_config()
 
@@ -430,11 +370,7 @@ class UtilInterface(InvocationContextInterface):
         self._cancel_event = cancel_event
 
     def is_canceled(self) -> bool:
-        """Checks if the current session has been canceled.
-
-        Returns:
-            True if the current session has been canceled, False if not.
-        """
+        """Checks if the current invocation has been canceled."""
         return self._cancel_event.is_set()
 
     def sd_step_callback(self, intermediate_state: PipelineIntermediateState, base_model: BaseModelType) -> None:
@@ -444,9 +380,8 @@ class UtilInterface(InvocationContextInterface):
 
         This should be called after each denoising step.
 
-        Args:
-            intermediate_state: The intermediate state of the diffusion pipeline.
-            base_model: The base model for the current denoising step.
+        :param intermediate_state: The intermediate state of the diffusion pipeline.
+        :param base_model: The base model for the current denoising step.
         """
 
         stable_diffusion_step_callback(
@@ -459,17 +394,8 @@ class UtilInterface(InvocationContextInterface):
 
 
 class InvocationContext:
-    """Provides access to various services and data for the current invocation.
-
-    Attributes:
-        images (ImagesInterface): Methods to save, get and update images and their metadata.
-        tensors (TensorsInterface): Methods to save and get tensors, including image, noise, masks, and masked images.
-        conditioning (ConditioningInterface): Methods to save and get conditioning data.
-        models (ModelsInterface): Methods to check if a model exists, get a model, and get a model's info.
-        logger (LoggerInterface): The app logger.
-        config (ConfigInterface): The app config.
-        util (UtilInterface): Utility methods, including a method to check if an invocation was canceled and step callbacks.
-        boards (BoardsInterface): Methods to interact with boards.
+    """
+    The `InvocationContext` provides access to various services and data for the current invocation.
     """
 
     def __init__(
@@ -512,14 +438,11 @@ def build_invocation_context(
     data: InvocationContextData,
     cancel_event: threading.Event,
 ) -> InvocationContext:
-    """Builds the invocation context for a specific invocation execution.
+    """
+    Builds the invocation context for a specific invocation execution.
 
-    Args:
-        services: The invocation services to wrap.
-        data: The invocation context data.
-
-    Returns:
-        The invocation context.
+    :param services: The invocation services to wrap.
+    :param data: The invocation context data.
     """
 
     logger = LoggerInterface(services=services, data=data)

invokeai/app/services/shared/sqlite_migrator/migrations/util/migrate_yaml_config_1.py

@@ -3,6 +3,7 @@
 
 import json
 import sqlite3
+from hashlib import sha1
 from logging import Logger
 from pathlib import Path
 from typing import Optional
@@ -21,7 +22,7 @@ from invokeai.backend.model_manager.config import (
     ModelConfigFactory,
     ModelType,
 )
-from invokeai.backend.model_manager.hash import ModelHash
+from invokeai.backend.model_manager.hash import FastModelHash
 
 ModelsValidator = TypeAdapter(AnyModelConfig)
 
@@ -72,27 +73,19 @@ class MigrateModelYamlToDb1:
 
             base_type, model_type, model_name = str(model_key).split("/")
             try:
-                hash = ModelHash().hash(self.config.models_path / stanza.path)
+                hash = FastModelHash.hash(self.config.models_path / stanza.path)
             except OSError:
                 self.logger.warning(f"The model at {stanza.path} is not a valid file or directory. Skipping migration.")
                 continue
 
+            assert isinstance(model_key, str)
+            new_key = sha1(model_key.encode("utf-8")).hexdigest()
+
             stanza["base"] = BaseModelType(base_type)
             stanza["type"] = ModelType(model_type)
             stanza["name"] = model_name
             stanza["original_hash"] = hash
             stanza["current_hash"] = hash
-            new_key = hash  # deterministic key assignment
-
-            # special case for ip adapters, which need the new `image_encoder_model_id` field
-            if stanza["type"] == ModelType.IPAdapter:
-                try:
-                    stanza["image_encoder_model_id"] = self._get_image_encoder_model_id(
-                        self.config.models_path / stanza.path
-                    )
-                except OSError:
-                    self.logger.warning(f"Could not determine image encoder for {stanza.path}. Skipping.")
-                    continue
 
             new_config: AnyModelConfig = ModelsValidator.validate_python(stanza)  # type: ignore # see https://github.com/pydantic/pydantic/discussions/7094
 
@@ -102,7 +95,7 @@ class MigrateModelYamlToDb1:
                     self.logger.info(f"Updating model {model_name} with information from models.yaml using key {key}")
                     self._update_model(key, new_config)
                 else:
-                    self.logger.info(f"Adding model {model_name} with key {new_key}")
+                    self.logger.info(f"Adding model {model_name} with key {model_key}")
                     self._add_model(new_key, new_config)
             except DuplicateModelException:
                 self.logger.warning(f"Model {model_name} is already in the database")
@@ -156,8 +149,3 @@ class MigrateModelYamlToDb1:
             )
         except sqlite3.IntegrityError as exc:
             raise DuplicateModelException(f"{record.name}: model is already in database") from exc
-
-    def _get_image_encoder_model_id(self, model_path: Path) -> str:
-        with open(model_path / "image_encoder.txt") as f:
-            encoder = f.read()
-        return encoder.strip()

invokeai/app/services/shared/sqlite_migrator/sqlite_migrator_common.py

@@ -17,7 +17,8 @@ class MigrateCallback(Protocol):
     See :class:`Migration` for an example.
     """
 
-    def __call__(self, cursor: sqlite3.Cursor) -> None: ...
+    def __call__(self, cursor: sqlite3.Cursor) -> None:
+        ...
 
 
 class MigrationError(RuntimeError):

invokeai/app/util/ti_utils.py

@@ -1,47 +1,8 @@
 import re
-from typing import List, Tuple
-
-import invokeai.backend.util.logging as logger
-from invokeai.app.services.model_records import UnknownModelException
-from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.backend.model_manager.config import BaseModelType, ModelType
-from invokeai.backend.textual_inversion import TextualInversionModelRaw
 
 
-def extract_ti_triggers_from_prompt(prompt: str) -> List[str]:
-    ti_triggers: List[str] = []
+def extract_ti_triggers_from_prompt(prompt: str) -> list[str]:
+    ti_triggers = []
     for trigger in re.findall(r"<[a-zA-Z0-9., _-]+>", prompt):
-        ti_triggers.append(str(trigger))
+        ti_triggers.append(trigger)
     return ti_triggers
-
-
-def generate_ti_list(
-    prompt: str, base: BaseModelType, context: InvocationContext
-) -> List[Tuple[str, TextualInversionModelRaw]]:
-    ti_list: List[Tuple[str, TextualInversionModelRaw]] = []
-    for trigger in extract_ti_triggers_from_prompt(prompt):
-        name_or_key = trigger[1:-1]
-        try:
-            loaded_model = context.models.load(key=name_or_key)
-            model = loaded_model.model
-            assert isinstance(model, TextualInversionModelRaw)
-            assert loaded_model.config.base == base
-            ti_list.append((name_or_key, model))
-        except UnknownModelException:
-            try:
-                loaded_model = context.models.load_by_attrs(
-                    name=name_or_key, base=base, type=ModelType.TextualInversion
-                )
-                model = loaded_model.model
-                assert isinstance(model, TextualInversionModelRaw)
-                assert loaded_model.config.base == base
-                ti_list.append((name_or_key, model))
-            except UnknownModelException:
-                pass
-        except ValueError:
-            logger.warning(f'trigger: "{trigger}" more than one similarly-named textual inversion models')
-        except AssertionError:
-            logger.warning(f'trigger: "{trigger}" not a valid textual inversion model for this graph')
-        except Exception:
-            logger.warning(f'Failed to load TI model for trigger: "{trigger}"')
-    return ti_list

invokeai/backend/image_util/__init__.py

@@ -1,7 +1,6 @@
 """
 Initialization file for invokeai.backend.image_util methods.
 """
-
 from .patchmatch import PatchMatch  # noqa: F401
 from .pngwriter import PngWriter, PromptFormatter, retrieve_metadata, write_metadata  # noqa: F401
 from .seamless import configure_model_padding  # noqa: F401

invokeai/backend/image_util/invisible_watermark.py

@@ -3,7 +3,6 @@ This module defines a singleton object, "invisible_watermark" that
 wraps the invisible watermark model. It respects the global "invisible_watermark"
 configuration variable, that allows the watermarking to be supressed.
 """
-
 import cv2
 import numpy as np
 from imwatermark import WatermarkEncoder

invokeai/backend/image_util/patchmatch.py

@@ -4,7 +4,6 @@ wraps the actual patchmatch object. It respects the global
 "try_patchmatch" attribute, so that patchmatch loading can
 be suppressed or deferred
 """
-
 import numpy as np
 
 import invokeai.backend.util.logging as logger

invokeai/backend/image_util/pngwriter.py

@@ -6,7 +6,6 @@ PngWriter -- Converts Images generated by T2I into PNGs, finds
 
 Exports function retrieve_metadata(path)
 """
-
 import json
 import os
 import re

invokeai/backend/image_util/safety_checker.py

@@ -3,7 +3,6 @@ This module defines a singleton object, "safety_checker" that
 wraps the safety_checker model. It respects the global "nsfw_checker"
 configuration variable, that allows the checker to be supressed.
 """
-
 import numpy as np
 from PIL import Image
 

invokeai/backend/install/check_root.py

@@ -1,7 +1,6 @@
 """
 Check that the invokeai_root is correctly configured and exit if not.
 """
-
 import sys
 
 from invokeai.app.services.config import InvokeAIAppConfig

invokeai/backend/install/install_helper.py

@@ -1,5 +1,4 @@
 """Utility (backend) functions used by model_install.py"""
-
 from logging import Logger
 from pathlib import Path
 from typing import Any, Dict, List, Optional

invokeai/backend/ip_adapter/attention_processor.py

@@ -0,0 +1,182 @@
+# copied from https://github.com/tencent-ailab/IP-Adapter (Apache License 2.0)
+#   and modified as needed
+
+# tencent-ailab comment:
+# modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from diffusers.models.attention_processor import AttnProcessor2_0 as DiffusersAttnProcessor2_0
+
+from invokeai.backend.ip_adapter.ip_attention_weights import IPAttentionProcessorWeights
+
+
+# Create a version of AttnProcessor2_0 that is a sub-class of nn.Module. This is required for IP-Adapter state_dict
+# loading.
+class AttnProcessor2_0(DiffusersAttnProcessor2_0, nn.Module):
+    def __init__(self):
+        DiffusersAttnProcessor2_0.__init__(self)
+        nn.Module.__init__(self)
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        ip_adapter_image_prompt_embeds=None,
+    ):
+        """Re-definition of DiffusersAttnProcessor2_0.__call__(...) that accepts and ignores the
+        ip_adapter_image_prompt_embeds parameter.
+        """
+        return DiffusersAttnProcessor2_0.__call__(
+            self, attn, hidden_states, encoder_hidden_states, attention_mask, temb
+        )
+
+
+class IPAttnProcessor2_0(torch.nn.Module):
+    r"""
+    Attention processor for IP-Adapater for PyTorch 2.0.
+    Args:
+        hidden_size (`int`):
+            The hidden size of the attention layer.
+        cross_attention_dim (`int`):
+            The number of channels in the `encoder_hidden_states`.
+        scale (`float`, defaults to 1.0):
+            the weight scale of image prompt.
+    """
+
+    def __init__(self, weights: list[IPAttentionProcessorWeights], scales: list[float]):
+        super().__init__()
+
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+        assert len(weights) == len(scales)
+
+        self._weights = weights
+        self._scales = scales
+
+    def __call__(
+        self,
+        attn,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+        temb=None,
+        ip_adapter_image_prompt_embeds=None,
+    ):
+        """Apply IP-Adapter attention.
+
+        Args:
+            ip_adapter_image_prompt_embeds (torch.Tensor): The image prompt embeddings.
+                Shape: (batch_size, num_ip_images, seq_len, ip_embedding_len).
+        """
+        residual = hidden_states
+
+        if attn.spatial_norm is not None:
+            hidden_states = attn.spatial_norm(hidden_states, temb)
+
+        input_ndim = hidden_states.ndim
+
+        if input_ndim == 4:
+            batch_size, channel, height, width = hidden_states.shape
+            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+        batch_size, sequence_length, _ = (
+            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
+        )
+
+        if attention_mask is not None:
+            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+            # scaled_dot_product_attention expects attention_mask shape to be
+            # (batch, heads, source_length, target_length)
+            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
+
+        if attn.group_norm is not None:
+            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+        query = attn.to_q(hidden_states)
+
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.norm_cross:
+            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # the output of sdp = (batch, num_heads, seq_len, head_dim)
+        # TODO: add support for attn.scale when we move to Torch 2.1
+        hidden_states = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+        )
+
+        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        hidden_states = hidden_states.to(query.dtype)
+
+        if encoder_hidden_states is not None:
+            # If encoder_hidden_states is not None, then we are doing cross-attention, not self-attention. In this case,
+            # we will apply IP-Adapter conditioning. We validate the inputs for IP-Adapter conditioning here.
+            assert ip_adapter_image_prompt_embeds is not None
+            assert len(ip_adapter_image_prompt_embeds) == len(self._weights)
+
+            for ipa_embed, ipa_weights, scale in zip(
+                ip_adapter_image_prompt_embeds, self._weights, self._scales, strict=True
+            ):
+                # The batch dimensions should match.
+                assert ipa_embed.shape[0] == encoder_hidden_states.shape[0]
+                # The token_len dimensions should match.
+                assert ipa_embed.shape[-1] == encoder_hidden_states.shape[-1]
+
+                ip_hidden_states = ipa_embed
+
+                # Expected ip_hidden_state shape: (batch_size, num_ip_images, ip_seq_len, ip_image_embedding)
+
+                ip_key = ipa_weights.to_k_ip(ip_hidden_states)
+                ip_value = ipa_weights.to_v_ip(ip_hidden_states)
+
+                # Expected ip_key and ip_value shape: (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads)
+
+                ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+                ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+                # Expected ip_key and ip_value shape: (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim)
+
+                # TODO: add support for attn.scale when we move to Torch 2.1
+                ip_hidden_states = F.scaled_dot_product_attention(
+                    query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
+                )
+
+                # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim)
+
+                ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+                ip_hidden_states = ip_hidden_states.to(query.dtype)
+
+                # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim)
+
+                hidden_states = hidden_states + scale * ip_hidden_states
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        if input_ndim == 4:
+            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+        if attn.residual_connection:
+            hidden_states = hidden_states + residual
+
+        hidden_states = hidden_states / attn.rescale_output_factor
+
+        return hidden_states

invokeai/backend/ip_adapter/unet_patcher.py

@@ -1,55 +1,52 @@
 from contextlib import contextmanager
-from typing import Optional
 
 from diffusers.models import UNet2DConditionModel
 
+from invokeai.backend.ip_adapter.attention_processor import AttnProcessor2_0, IPAttnProcessor2_0
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
-from invokeai.backend.stable_diffusion.diffusion.custom_attention import CustomAttnProcessor2_0
 
 
-class UNetAttentionPatcher:
-    """A class for patching a UNet with CustomAttnProcessor2_0 attention layers."""
+class UNetPatcher:
+    """A class that contains multiple IP-Adapters and can apply them to a UNet."""
 
-    def __init__(self, ip_adapters: Optional[list[IPAdapter]]):
+    def __init__(self, ip_adapters: list[IPAdapter]):
         self._ip_adapters = ip_adapters
-        self._ip_adapter_scales = None
-
-        if self._ip_adapters is not None:
-            self._ip_adapter_scales = [1.0] * len(self._ip_adapters)
+        self._scales = [1.0] * len(self._ip_adapters)
 
     def set_scale(self, idx: int, value: float):
-        self._ip_adapter_scales[idx] = value
+        self._scales[idx] = value
 
     def _prepare_attention_processors(self, unet: UNet2DConditionModel):
         """Prepare a dict of attention processors that can be injected into a unet, and load the IP-Adapter attention
-        weights into them (if IP-Adapters are being applied).
+        weights into them.
 
         Note that the `unet` param is only used to determine attention block dimensions and naming.
         """
         # Construct a dict of attention processors based on the UNet's architecture.
         attn_procs = {}
         for idx, name in enumerate(unet.attn_processors.keys()):
-            if name.endswith("attn1.processor") or self._ip_adapters is None:
-                # "attn1" processors do not use IP-Adapters.
-                attn_procs[name] = CustomAttnProcessor2_0()
+            if name.endswith("attn1.processor"):
+                attn_procs[name] = AttnProcessor2_0()
             else:
                 # Collect the weights from each IP Adapter for the idx'th attention processor.
-                attn_procs[name] = CustomAttnProcessor2_0(
+                attn_procs[name] = IPAttnProcessor2_0(
                     [ip_adapter.attn_weights.get_attention_processor_weights(idx) for ip_adapter in self._ip_adapters],
-                    self._ip_adapter_scales,
+                    self._scales,
                 )
         return attn_procs
 
     @contextmanager
     def apply_ip_adapter_attention(self, unet: UNet2DConditionModel):
-        """A context manager that patches `unet` with CustomAttnProcessor2_0 attention layers."""
+        """A context manager that patches `unet` with IP-Adapter attention processors."""
+
         attn_procs = self._prepare_attention_processors(unet)
+
         orig_attn_processors = unet.attn_processors
 
         try:
-            # Note to future devs: set_attn_processor(...) does something slightly unexpected - it pops elements from
-            # the passed dict. So, if you wanted to keep the dict for future use, you'd have to make a
-            # moderately-shallow copy of it. E.g. `attn_procs_copy = {k: v for k, v in attn_procs.items()}`.
+            # Note to future devs: set_attn_processor(...) does something slightly unexpected - it pops elements from the
+            # passed dict. So, if you wanted to keep the dict for future use, you'd have to make a moderately-shallow copy
+            # of it. E.g. `attn_procs_copy = {k: v for k, v in attn_procs.items()}`.
             unet.set_attn_processor(attn_procs)
             yield None
         finally:

invokeai/backend/model_manager/__init__.py

@@ -1,5 +1,4 @@
 """Re-export frequently-used symbols from the Model Manager backend."""
-
 from .config import (
     AnyModel,
     AnyModelConfig,

invokeai/backend/model_manager/config.py

@@ -19,7 +19,6 @@ Typical usage:
 Validation errors will raise an InvalidModelConfigException error.
 
 """
-
 import time
 from enum import Enum
 from typing import Literal, Optional, Type, Union

invokeai/backend/model_manager/convert_ckpt_to_diffusers.py

@@ -15,7 +15,7 @@
 #
 # Adapted for use in InvokeAI by Lincoln Stein, July 2023
 #
-"""Conversion script for the Stable Diffusion checkpoints."""
+""" Conversion script for the Stable Diffusion checkpoints."""
 
 import re
 from contextlib import nullcontext

invokeai/backend/model_manager/hash.py

@@ -11,175 +11,56 @@ from invokeai.backend.model_managre.model_hash import FastModelHash
 import hashlib
 import os
 from pathlib import Path
-from typing import Callable, Literal, Optional, Union
+from typing import Dict, Union
 
-from blake3 import blake3
-
-MODEL_FILE_EXTENSIONS = (".ckpt", ".safetensors", ".bin", ".pt", ".pth")
-
-ALGORITHM = Literal[
-    "md5",
-    "sha1",
-    "sha224",
-    "sha256",
-    "sha384",
-    "sha512",
-    "blake2b",
-    "blake2s",
-    "sha3_224",
-    "sha3_256",
-    "sha3_384",
-    "sha3_512",
-    "shake_128",
-    "shake_256",
-    "blake3",
-]
+from imohash import hashfile
 
 
-class ModelHash:
-    """
-    Creates a hash of a model using a specified algorithm.
+class FastModelHash(object):
+    """FastModelHash obect provides one public class method, hash()."""
 
-    Args:
-        algorithm: Hashing algorithm to use. Defaults to BLAKE3.
-        file_filter: A function that takes a file name and returns True if the file should be included in the hash.
+    @classmethod
+    def hash(cls, model_location: Union[str, Path]) -> str:
+        """
+        Return hexdigest string for model located at model_location.
 
-    If the model is a single file, it is hashed directly using the provided algorithm.
-
-    If the model is a directory, each model weights file in the directory is hashed using the provided algorithm.
-
-    Only files with the following extensions are hashed: .ckpt, .safetensors, .bin, .pt, .pth
-
-    The final hash is computed by hashing the hashes of all model files in the directory using BLAKE3, ensuring
-    that directory hashes are never weaker than the file hashes.
-
-    Usage:
-        ```py
-        # BLAKE3 hash
-        ModelHash().hash("path/to/some/model.safetensors")
-        # MD5
-        ModelHash("md5").hash("path/to/model/dir/")
-        ```
-    """
-
-    def __init__(self, algorithm: ALGORITHM = "blake3", file_filter: Optional[Callable[[str], bool]] = None) -> None:
-        if algorithm == "blake3":
-            self._hash_file = self._blake3
-        elif algorithm in hashlib.algorithms_available:
-            self._hash_file = self._get_hashlib(algorithm)
+        :param model_location: Path to the model
+        """
+        model_location = Path(model_location)
+        if model_location.is_file():
+            return cls._hash_file(model_location)
+        elif model_location.is_dir():
+            return cls._hash_dir(model_location)
         else:
-            raise ValueError(f"Algorithm {algorithm} not available")
+            raise OSError(f"Not a valid file or directory: {model_location}")
 
-        self._file_filter = file_filter or self._default_file_filter
-
-    def hash(self, model_path: Union[str, Path]) -> str:
+    @classmethod
+    def _hash_file(cls, model_location: Union[str, Path]) -> str:
         """
-        Return hexdigest of hash of model located at model_path using the algorithm provided at class instantiation.
+        Fasthash a single file and return its hexdigest.
 
-        If model_path is a directory, the hash is computed by hashing the hashes of all model files in the
-        directory. The final composite hash is always computed using BLAKE3.
-
-        Args:
-            model_path: Path to the model
-
-        Returns:
-            str: Hexdigest of the hash of the model
+        :param model_location: Path to the model file
         """
+        # we return md5 hash of the filehash to make it shorter
+        # cryptographic security not needed here
+        return hashlib.md5(hashfile(model_location)).hexdigest()
 
-        model_path = Path(model_path)
-        if model_path.is_file():
-            return self._hash_file(model_path)
-        elif model_path.is_dir():
-            return self._hash_dir(model_path)
-        else:
-            raise OSError(f"Not a valid file or directory: {model_path}")
+    @classmethod
+    def _hash_dir(cls, model_location: Union[str, Path]) -> str:
+        components: Dict[str, str] = {}
 
-    def _hash_dir(self, dir: Path) -> str:
-        """Compute the hash for all files in a directory and return a hexdigest.
+        for root, _dirs, files in os.walk(model_location):
+            for file in files:
+                # only tally tensor files because diffusers config files change slightly
+                # depending on how the model was downloaded/converted.
+                if not file.endswith((".ckpt", ".safetensors", ".bin", ".pt", ".pth")):
+                    continue
+                path = (Path(root) / file).as_posix()
+                fast_hash = cls._hash_file(path)
+                components.update({path: fast_hash})
 
-        Args:
-            dir: Path to the directory
-
-        Returns:
-            str: Hexdigest of the hash of the directory
-        """
-        model_component_paths = self._get_file_paths(dir, self._file_filter)
-
-        component_hashes: list[str] = []
-        for component in sorted(model_component_paths):
-            component_hashes.append(self._hash_file(component))
-
-        # BLAKE3 is cryptographically secure. We may as well fall back on a secure algorithm
-        # for the composite hash
-        composite_hasher = blake3()
-        for h in component_hashes:
-            composite_hasher.update(h.encode("utf-8"))
-        return composite_hasher.hexdigest()
-
-    @staticmethod
-    def _get_file_paths(model_path: Path, file_filter: Callable[[str], bool]) -> list[Path]:
-        """Return a list of all model files in the directory.
-
-        Args:
-            model_path: Path to the model
-            file_filter: Function that takes a file name and returns True if the file should be included in the list.
-
-        Returns:
-            List of all model files in the directory
-        """
-
-        files: list[Path] = []
-        for root, _dirs, _files in os.walk(model_path):
-            for file in _files:
-                if file_filter(file):
-                    files.append(Path(root, file))
-        return files
-
-    @staticmethod
-    def _blake3(file_path: Path) -> str:
-        """Hashes a file using BLAKE3
-
-        Args:
-            file_path: Path to the file to hash
-
-        Returns:
-            Hexdigest of the hash of the file
-        """
-        file_hasher = blake3(max_threads=blake3.AUTO)
-        file_hasher.update_mmap(file_path)
-        return file_hasher.hexdigest()
-
-    @staticmethod
-    def _get_hashlib(algorithm: ALGORITHM) -> Callable[[Path], str]:
-        """Factory function that returns a function to hash a file with the given algorithm.
-
-        Args:
-            algorithm: Hashing algorithm to use
-
-        Returns:
-            A function that hashes a file using the given algorithm
-        """
-
-        def hashlib_hasher(file_path: Path) -> str:
-            """Hashes a file using a hashlib algorithm. Uses `memoryview` to avoid reading the entire file into memory."""
-            hasher = hashlib.new(algorithm)
-            buffer = bytearray(128 * 1024)
-            mv = memoryview(buffer)
-            with open(file_path, "rb", buffering=0) as f:
-                while n := f.readinto(mv):
-                    hasher.update(mv[:n])
-            return hasher.hexdigest()
-
-        return hashlib_hasher
-
-    @staticmethod
-    def _default_file_filter(file_path: str) -> bool:
-        """A default file filter that only includes files with the following extensions: .ckpt, .safetensors, .bin, .pt, .pth
-
-        Args:
-            file_path: Path to the file
-
-        Returns:
-            True if the file matches the given extensions, otherwise False
-        """
-        return file_path.endswith(MODEL_FILE_EXTENSIONS)
+        # hash all the model hashes together, using alphabetic file order
+        md5 = hashlib.md5()
+        for _path, fast_hash in sorted(components.items()):
+            md5.update(fast_hash.encode("utf-8"))
+        return md5.hexdigest()

invokeai/backend/model_manager/load/__init__.py

@@ -2,7 +2,6 @@
 """
 Init file for the model loader.
 """
-
 from importlib import import_module
 from pathlib import Path
 

invokeai/backend/model_manager/load/convert_cache/convert_cache_base.py

@@ -1,7 +1,6 @@
 """
 Disk-based converted model cache.
 """
-
 from abc import ABC, abstractmethod
 from pathlib import Path
 

invokeai/backend/model_manager/load/model_loader_registry.py

@@ -14,7 +14,6 @@ Use like this:
     ).load_model(model_config, submodel_type)
 
 """
-
 import hashlib
 from abc import ABC, abstractmethod
 from pathlib import Path

invokeai/backend/model_manager/load/model_loaders/lora.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2024, Lincoln D. Stein and the InvokeAI Development Team
 """Class for LoRA model loading in InvokeAI."""
 
+
 from logging import Logger
 from pathlib import Path
 from typing import Optional, Tuple

invokeai/backend/model_manager/load/model_loaders/stable_diffusion.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2024, Lincoln D. Stein and the InvokeAI Development Team
 """Class for StableDiffusion model loading in InvokeAI."""
 
+
 from pathlib import Path
 from typing import Optional
 

invokeai/backend/model_manager/load/model_loaders/textual_inversion.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2024, Lincoln D. Stein and the InvokeAI Development Team
 """Class for TI model loading in InvokeAI."""
 
+
 from pathlib import Path
 from typing import Optional, Tuple
 

invokeai/backend/model_manager/metadata/__init__.py

@@ -18,7 +18,6 @@ assert isinstance(data, CivitaiMetadata)
 if data.allow_commercial_use:
    print("Commercial use of this model is allowed")
 """
-
 from .fetch import CivitaiMetadataFetch, HuggingFaceMetadataFetch, ModelMetadataFetchBase
 from .metadata_base import (
     AnyModelRepoMetadata,

invokeai/backend/model_manager/metadata/fetch/civitai.py

@@ -160,10 +160,11 @@ class CivitaiMetadataFetch(ModelMetadataFetchBase):
             nsfw=model_json["nsfw"],
             restrictions=LicenseRestrictions(
                 AllowNoCredit=model_json["allowNoCredit"],
-                AllowCommercialUse={CommercialUsage(x) for x in model_json["allowCommercialUse"]},
+                AllowCommercialUse=CommercialUsage(model_json["allowCommercialUse"]),
                 AllowDerivatives=model_json["allowDerivatives"],
                 AllowDifferentLicense=model_json["allowDifferentLicense"],
             ),
+            trigger_phrases=version_json["trainedWords"],
         )
 
     def from_civitai_versionid(self, version_id: int, model_id: Optional[int] = None) -> CivitaiMetadata:

invokeai/backend/model_manager/metadata/metadata_base.py

@@ -24,8 +24,8 @@ from pydantic import BaseModel, Field, TypeAdapter
 from pydantic.networks import AnyHttpUrl
 from requests.sessions import Session
 from typing_extensions import Annotated
-
 from invokeai.app.invocations.constants import SCHEDULER_NAME_VALUES
+
 from invokeai.backend.model_manager import ModelRepoVariant
 
 from ..util import select_hf_files
@@ -55,8 +55,8 @@ class LicenseRestrictions(BaseModel):
     AllowDifferentLicense: bool = Field(
         description="if true, derivatives of this model be redistributed under a different license", default=False
     )
-    AllowCommercialUse: Optional[Set[CommercialUsage] | CommercialUsage] = Field(
-        description="Type of commercial use allowed if no commercial use is allowed.", default=None
+    AllowCommercialUse: Optional[CommercialUsage] = Field(
+        description="Type of commercial use allowed or 'No' if no commercial use is allowed.", default=None
     )
 
 
@@ -77,16 +77,14 @@ class ModelDefaultSettings(BaseModel):
     cfg_scale: float | None
     cfg_rescale_multiplier: float | None
 
-
 class ModelMetadataBase(BaseModel):
     """Base class for model metadata information."""
 
     name: str = Field(description="model's name")
     author: str = Field(description="model's author")
     tags: Optional[Set[str]] = Field(description="tags provided by model source", default=None)
-    default_settings: Optional[ModelDefaultSettings] = Field(
-        description="default settings for this model", default=None
-    )
+    trigger_phrases: Optional[List[str]] = Field(description="trigger phrases for this model", default=None)
+    default_settings: Optional[ModelDefaultSettings] = Field(description="default settings for this model", default=None)
 
 
 class BaseMetadata(ModelMetadataBase):
@@ -155,10 +153,7 @@ class CivitaiMetadata(ModelMetadataWithFiles):
         if self.restrictions.AllowCommercialUse is None:
             return False
         else:
-            # accommodate schema change
-            acu = self.restrictions.AllowCommercialUse
-            commercial_usage = acu if isinstance(acu, set) else {acu}
-            return CommercialUsage.No not in commercial_usage
+            return self.restrictions.AllowCommercialUse != CommercialUsage("None")
 
     @property
     def allow_derivatives(self) -> bool:

invokeai/backend/model_manager/probe.py

@@ -21,7 +21,7 @@ from .config import (
     ModelVariantType,
     SchedulerPredictionType,
 )
-from .hash import ModelHash
+from .hash import FastModelHash
 from .util.model_util import lora_token_vector_length, read_checkpoint_meta
 
 CkptType = Dict[str, Any]
@@ -147,7 +147,7 @@ class ModelProbe(object):
         if not probe_class:
             raise InvalidModelConfigException(f"Unhandled combination of {format_type} and {model_type}")
 
-        hash = ModelHash().hash(model_path)
+        hash = FastModelHash.hash(model_path)
         probe = probe_class(model_path)
 
         fields["path"] = model_path.as_posix()
@@ -188,7 +188,7 @@ class ModelProbe(object):
                 and fields["prediction_type"] == SchedulerPredictionType.VPrediction
             )
 
-        model_info = ModelConfigFactory.make_config(fields)  # , key=fields.get("key", None))
+        model_info = ModelConfigFactory.make_config(fields, key=fields.get("key", None))
         return model_info
 
     @classmethod

invokeai/backend/model_manager/search.py

@@ -28,7 +28,6 @@ from typing import Callable, Optional, Set, Union
 
 from pydantic import BaseModel, Field
 
-from invokeai.app.services.config import InvokeAIAppConfig
 from invokeai.backend.util.logging import InvokeAILogger
 
 default_logger: Logger = InvokeAILogger.get_logger()
@@ -118,10 +117,13 @@ class ModelSearch(ModelSearchBase):
     """
 
     models_found: Set[Path] = Field(default_factory=set)
-    config: InvokeAIAppConfig = InvokeAIAppConfig.get_config()
+    scanned_dirs: Set[Path] = Field(default_factory=set)
+    pruned_paths: Set[Path] = Field(default_factory=set)
 
     def search_started(self) -> None:
         self.models_found = set()
+        self.scanned_dirs = set()
+        self.pruned_paths = set()
         if self.on_search_started:
             self.on_search_started(self._directory)
 
@@ -137,53 +139,53 @@ class ModelSearch(ModelSearchBase):
 
     def search(self, directory: Union[Path, str]) -> Set[Path]:
         self._directory = Path(directory)
-        if not self._directory.is_absolute():
-            self._directory = self.config.models_path / self._directory
         self.stats = SearchStats()  # zero out
         self.search_started()  # This will initialize _models_found to empty
-        self._walk_directory(self._directory)
+        self._walk_directory(directory)
         self.search_completed()
         return self.models_found
 
-    def _walk_directory(self, path: Union[Path, str], max_depth: int = 20) -> None:
-        absolute_path = Path(path)
-        if (
-            len(absolute_path.parts) - len(self._directory.parts) > max_depth
-            or not absolute_path.exists()
-            or absolute_path.parent in self.models_found
-        ):
-            return
-        entries = os.scandir(absolute_path.as_posix())
-        entries = [entry for entry in entries if not entry.name.startswith(".")]
-        dirs = [entry for entry in entries if entry.is_dir()]
-        file_names = [entry.name for entry in entries if entry.is_file()]
-        if any(
-            x in file_names
-            for x in [
-                "config.json",
-                "model_index.json",
-                "learned_embeds.bin",
-                "pytorch_lora_weights.bin",
-                "image_encoder.txt",
-            ]
-        ):
-            try:
-                self.model_found(absolute_path)
-                return
-            except KeyboardInterrupt:
-                raise
-            except Exception as e:
-                self.logger.warning(str(e))
-                return
+    def _walk_directory(self, path: Union[Path, str]) -> None:
+        for root, dirs, files in os.walk(path, followlinks=True):
+            # don't descend into directories that start with a "."
+            # to avoid the Mac .DS_STORE issue.
+            if str(Path(root).name).startswith("."):
+                self.pruned_paths.add(Path(root))
+            if any(Path(root).is_relative_to(x) for x in self.pruned_paths):
+                continue
 
-        for n in file_names:
-            if n.endswith((".ckpt", ".bin", ".pth", ".safetensors", ".pt")):
-                try:
-                    self.model_found(absolute_path / n)
-                except KeyboardInterrupt:
-                    raise
-                except Exception as e:
-                    self.logger.warning(str(e))
+            self.stats.items_scanned += len(dirs) + len(files)
+            for d in dirs:
+                path = Path(root) / d
+                if path.parent in self.scanned_dirs:
+                    self.scanned_dirs.add(path)
+                    continue
+                if any(
+                    (path / x).exists()
+                    for x in [
+                        "config.json",
+                        "model_index.json",
+                        "learned_embeds.bin",
+                        "pytorch_lora_weights.bin",
+                        "image_encoder.txt",
+                    ]
+                ):
+                    self.scanned_dirs.add(path)
+                    try:
+                        self.model_found(path)
+                    except KeyboardInterrupt:
+                        raise
+                    except Exception as e:
+                        self.logger.warning(str(e))
 
-        for d in dirs:
-            self._walk_directory(absolute_path / d)
+            for f in files:
+                path = Path(root) / f
+                if path.parent in self.scanned_dirs:
+                    continue
+                if path.suffix in {".ckpt", ".bin", ".pth", ".safetensors", ".pt"}:
+                    try:
+                        self.model_found(path)
+                    except KeyboardInterrupt:
+                        raise
+                    except Exception as e:
+                        self.logger.warning(str(e))

invokeai/backend/model_patcher.py

@@ -1,16 +1,15 @@
 # Copyright (c) 2024 Ryan Dick, Lincoln D. Stein, and the InvokeAI Development Team
 """These classes implement model patching with LoRAs and Textual Inversions."""
-
 from __future__ import annotations
 
 import pickle
 from contextlib import contextmanager
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 import numpy as np
 import torch
 from diffusers import OnnxRuntimeModel, UNet2DConditionModel
-from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
+from transformers import CLIPTextModel, CLIPTokenizer
 
 from invokeai.app.shared.models import FreeUConfig
 from invokeai.backend.model_manager import AnyModel
@@ -169,7 +168,7 @@ class ModelPatcher:
     def apply_ti(
         cls,
         tokenizer: CLIPTokenizer,
-        text_encoder: Union[CLIPTextModel, CLIPTextModelWithProjection],
+        text_encoder: CLIPTextModel,
         ti_list: List[Tuple[str, TextualInversionModelRaw]],
     ) -> Iterator[Tuple[CLIPTokenizer, TextualInversionManager]]:
         init_tokens_count = None
@@ -266,7 +265,7 @@ class ModelPatcher:
     @contextmanager
     def apply_clip_skip(
         cls,
-        text_encoder: Union[CLIPTextModel, CLIPTextModelWithProjection],
+        text_encoder: CLIPTextModel,
         clip_skip: int,
     ) -> None:
         skipped_layers = []

invokeai/backend/stable_diffusion/__init__.py

@@ -1,14 +1,15 @@
 """
 Initialization file for the invokeai.backend.stable_diffusion package
 """
-
 from .diffusers_pipeline import PipelineIntermediateState, StableDiffusionGeneratorPipeline  # noqa: F401
 from .diffusion import InvokeAIDiffuserComponent  # noqa: F401
+from .diffusion.cross_attention_map_saving import AttentionMapSaver  # noqa: F401
 from .seamless import set_seamless  # noqa: F401
 
 __all__ = [
     "PipelineIntermediateState",
     "StableDiffusionGeneratorPipeline",
     "InvokeAIDiffuserComponent",
+    "AttentionMapSaver",
     "set_seamless",
 ]

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -12,6 +12,7 @@ import torch
 import torchvision.transforms as T
 from diffusers.models import AutoencoderKL, UNet2DConditionModel
 from diffusers.models.controlnet import ControlNetModel
+from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
 from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipeline
 from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
 from diffusers.schedulers import KarrasDiffusionSchedulers
@@ -23,14 +24,11 @@ from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
 
 from invokeai.app.services.config import InvokeAIAppConfig
 from invokeai.backend.ip_adapter.ip_adapter import IPAdapter
-from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
-    IPAdapterConditioningInfo,
-    TextConditioningData,
-)
-from invokeai.backend.stable_diffusion.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent
-from invokeai.backend.stable_diffusion.diffusion.unet_attention_patcher import UNetAttentionPatcher
+from invokeai.backend.ip_adapter.unet_patcher import UNetPatcher
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningData
 
 from ..util import auto_detect_slice_size, normalize_device
+from .diffusion import AttentionMapSaver, InvokeAIDiffuserComponent
 
 
 @dataclass
@@ -41,6 +39,7 @@ class PipelineIntermediateState:
     timestep: int
     latents: torch.Tensor
     predicted_original: Optional[torch.Tensor] = None
+    attention_map_saver: Optional[AttentionMapSaver] = None
 
 
 @dataclass
@@ -173,11 +172,10 @@ class ControlNetData:
 
 @dataclass
 class IPAdapterData:
-    ip_adapter_model: IPAdapter
-    ip_adapter_conditioning: IPAdapterConditioningInfo
-
-    # Either a single weight applied to all steps, or a list of weights for each step.
+    ip_adapter_model: IPAdapter = Field(default=None)
+    # TODO: change to polymorphic so can do different weights per step (once implemented...)
     weight: Union[float, List[float]] = Field(default=1.0)
+    # weight: float = Field(default=1.0)
     begin_step_percent: float = Field(default=0.0)
     end_step_percent: float = Field(default=1.0)
 
@@ -192,6 +190,19 @@ class T2IAdapterData:
     end_step_percent: float = Field(default=1.0)
 
 
+@dataclass
+class InvokeAIStableDiffusionPipelineOutput(StableDiffusionPipelineOutput):
+    r"""
+    Output class for InvokeAI's Stable Diffusion pipeline.
+
+    Args:
+        attention_map_saver (`AttentionMapSaver`): Object containing attention maps that can be displayed to the user
+         after generation completes. Optional.
+    """
+
+    attention_map_saver: Optional[AttentionMapSaver]
+
+
 class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
     r"""
     Pipeline for text-to-image generation using Stable Diffusion.
@@ -318,8 +329,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         self,
         latents: torch.Tensor,
         num_inference_steps: int,
-        scheduler_step_kwargs: dict[str, Any],
-        conditioning_data: TextConditioningData,
+        conditioning_data: ConditioningData,
         *,
         noise: Optional[torch.Tensor],
         timesteps: torch.Tensor,
@@ -333,9 +343,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         masked_latents: Optional[torch.Tensor] = None,
         gradient_mask: Optional[bool] = False,
         seed: Optional[int] = None,
-    ) -> torch.Tensor:
+    ) -> tuple[torch.Tensor, Optional[AttentionMapSaver]]:
         if init_timestep.shape[0] == 0:
-            return latents
+            return latents, None
 
         if additional_guidance is None:
             additional_guidance = []
@@ -375,11 +385,10 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                 additional_guidance.append(AddsMaskGuidance(mask, orig_latents, self.scheduler, noise, gradient_mask))
 
         try:
-            latents = self.generate_latents_from_embeddings(
+            latents, attention_map_saver = self.generate_latents_from_embeddings(
                 latents,
                 timesteps,
                 conditioning_data,
-                scheduler_step_kwargs=scheduler_step_kwargs,
                 additional_guidance=additional_guidance,
                 control_data=control_data,
                 ip_adapter_data=ip_adapter_data,
@@ -393,59 +402,46 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         if mask is not None and not gradient_mask:
             latents = torch.lerp(orig_latents, latents.to(dtype=orig_latents.dtype), mask.to(dtype=orig_latents.dtype))
 
-        return latents
+        return latents, attention_map_saver
 
     def generate_latents_from_embeddings(
         self,
         latents: torch.Tensor,
         timesteps,
-        conditioning_data: TextConditioningData,
-        scheduler_step_kwargs: dict[str, Any],
+        conditioning_data: ConditioningData,
         *,
         additional_guidance: List[Callable] = None,
         control_data: List[ControlNetData] = None,
         ip_adapter_data: Optional[list[IPAdapterData]] = None,
         t2i_adapter_data: Optional[list[T2IAdapterData]] = None,
         callback: Callable[[PipelineIntermediateState], None] = None,
-    ) -> torch.Tensor:
+    ):
         self._adjust_memory_efficient_attention(latents)
         if additional_guidance is None:
             additional_guidance = []
 
         batch_size = latents.shape[0]
+        attention_map_saver: Optional[AttentionMapSaver] = None
 
         if timesteps.shape[0] == 0:
-            return latents
+            return latents, attention_map_saver
 
-        extra_conditioning_info = conditioning_data.cond_text.extra_conditioning
-        use_cross_attention_control = (
-            extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control
-        )
-        use_ip_adapter = ip_adapter_data is not None
-        use_regional_prompting = (
-            conditioning_data.cond_regions is not None or conditioning_data.uncond_regions is not None
-        )
-        if use_cross_attention_control and use_ip_adapter:
-            raise ValueError(
-                "Prompt-to-prompt cross-attention control (`.swap()`) and IP-Adapter cannot be used simultaneously."
-            )
-        if use_cross_attention_control and use_regional_prompting:
-            raise ValueError(
-                "Prompt-to-prompt cross-attention control (`.swap()`) and regional prompting cannot be used simultaneously."
-            )
-
-        unet_attention_patcher = None
-        self.use_ip_adapter = use_ip_adapter
-        attn_ctx = nullcontext()
-        if use_cross_attention_control:
+        ip_adapter_unet_patcher = None
+        if conditioning_data.extra is not None and conditioning_data.extra.wants_cross_attention_control:
             attn_ctx = self.invokeai_diffuser.custom_attention_context(
                 self.invokeai_diffuser.model,
-                extra_conditioning_info=extra_conditioning_info,
+                extra_conditioning_info=conditioning_data.extra,
+                step_count=len(self.scheduler.timesteps),
             )
-        if use_ip_adapter or use_regional_prompting:
-            ip_adapters = [ipa.ip_adapter_model for ipa in ip_adapter_data] if use_ip_adapter else None
-            unet_attention_patcher = UNetAttentionPatcher(ip_adapters)
-            attn_ctx = unet_attention_patcher.apply_ip_adapter_attention(self.invokeai_diffuser.model)
+            self.use_ip_adapter = False
+        elif ip_adapter_data is not None:
+            # TODO(ryand): Should we raise an exception if both custom attention and IP-Adapter attention are active?
+            # As it is now, the IP-Adapter will silently be skipped.
+            ip_adapter_unet_patcher = UNetPatcher([ipa.ip_adapter_model for ipa in ip_adapter_data])
+            attn_ctx = ip_adapter_unet_patcher.apply_ip_adapter_attention(self.invokeai_diffuser.model)
+            self.use_ip_adapter = True
+        else:
+            attn_ctx = nullcontext()
 
         with attn_ctx:
             if callback is not None:
@@ -468,16 +464,31 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                     conditioning_data,
                     step_index=i,
                     total_step_count=len(timesteps),
-                    scheduler_step_kwargs=scheduler_step_kwargs,
                     additional_guidance=additional_guidance,
                     control_data=control_data,
                     ip_adapter_data=ip_adapter_data,
                     t2i_adapter_data=t2i_adapter_data,
-                    unet_attention_patcher=unet_attention_patcher,
+                    ip_adapter_unet_patcher=ip_adapter_unet_patcher,
                 )
                 latents = step_output.prev_sample
+
+                latents = self.invokeai_diffuser.do_latent_postprocessing(
+                    postprocessing_settings=conditioning_data.postprocessing_settings,
+                    latents=latents,
+                    sigma=batched_t,
+                    step_index=i,
+                    total_step_count=len(timesteps),
+                )
+
                 predicted_original = getattr(step_output, "pred_original_sample", None)
 
+                # TODO resuscitate attention map saving
+                # if i == len(timesteps)-1 and extra_conditioning_info is not None:
+                #    eos_token_index = extra_conditioning_info.tokens_count_including_eos_bos - 1
+                #    attention_map_token_ids = range(1, eos_token_index)
+                #    attention_map_saver = AttentionMapSaver(token_ids=attention_map_token_ids, latents_shape=latents.shape[-2:])
+                #    self.invokeai_diffuser.setup_attention_map_saving(attention_map_saver)
+
                 if callback is not None:
                     callback(
                         PipelineIntermediateState(
@@ -487,25 +498,25 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                             timestep=int(t),
                             latents=latents,
                             predicted_original=predicted_original,
+                            attention_map_saver=attention_map_saver,
                         )
                     )
 
-            return latents
+            return latents, attention_map_saver
 
     @torch.inference_mode()
     def step(
         self,
         t: torch.Tensor,
         latents: torch.Tensor,
-        conditioning_data: TextConditioningData,
+        conditioning_data: ConditioningData,
         step_index: int,
         total_step_count: int,
-        scheduler_step_kwargs: dict[str, Any],
         additional_guidance: List[Callable] = None,
         control_data: List[ControlNetData] = None,
         ip_adapter_data: Optional[list[IPAdapterData]] = None,
         t2i_adapter_data: Optional[list[T2IAdapterData]] = None,
-        unet_attention_patcher: Optional[UNetAttentionPatcher] = None,
+        ip_adapter_unet_patcher: Optional[UNetPatcher] = None,
     ):
         # invokeai_diffuser has batched timesteps, but diffusers schedulers expect a single value
         timestep = t[0]
@@ -528,14 +539,20 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                 )
                 if step_index >= first_adapter_step and step_index <= last_adapter_step:
                     # Only apply this IP-Adapter if the current step is within the IP-Adapter's begin/end step range.
-                    unet_attention_patcher.set_scale(i, weight)
+                    ip_adapter_unet_patcher.set_scale(i, weight)
                 else:
                     # Otherwise, set the IP-Adapter's scale to 0, so it has no effect.
-                    unet_attention_patcher.set_scale(i, 0.0)
+                    ip_adapter_unet_patcher.set_scale(i, 0.0)
 
-        # Handle ControlNet(s)
+        # Handle ControlNet(s) and T2I-Adapter(s)
         down_block_additional_residuals = None
         mid_block_additional_residual = None
+        down_intrablock_additional_residuals = None
+        # if control_data is not None and t2i_adapter_data is not None:
+        # TODO(ryand): This is a limitation of the UNet2DConditionModel API, not a fundamental incompatibility
+        # between ControlNets and T2I-Adapters. We will try to fix this upstream in diffusers.
+        #    raise Exception("ControlNet(s) and T2I-Adapter(s) cannot be used simultaneously (yet).")
+        # elif control_data is not None:
         if control_data is not None:
             down_block_additional_residuals, mid_block_additional_residual = self.invokeai_diffuser.do_controlnet_step(
                 control_data=control_data,
@@ -545,9 +562,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                 total_step_count=total_step_count,
                 conditioning_data=conditioning_data,
             )
-
-        # Handle T2I-Adapter(s)
-        down_intrablock_additional_residuals = None
+        # elif t2i_adapter_data is not None:
         if t2i_adapter_data is not None:
             accum_adapter_state = None
             for single_t2i_adapter_data in t2i_adapter_data:
@@ -573,19 +588,16 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
                     for idx, value in enumerate(single_t2i_adapter_data.adapter_state):
                         accum_adapter_state[idx] += value * t2i_adapter_weight
 
+            # down_block_additional_residuals = accum_adapter_state
             down_intrablock_additional_residuals = accum_adapter_state
 
-        ip_adapter_conditioning = None
-        if ip_adapter_data is not None:
-            ip_adapter_conditioning = [ipa.ip_adapter_conditioning for ipa in ip_adapter_data]
-
         uc_noise_pred, c_noise_pred = self.invokeai_diffuser.do_unet_step(
             sample=latent_model_input,
             timestep=t,  # TODO: debug how handled batched and non batched timesteps
             step_index=step_index,
             total_step_count=total_step_count,
             conditioning_data=conditioning_data,
-            ip_adapter_conditioning=ip_adapter_conditioning,
+            # extra:
             down_block_additional_residuals=down_block_additional_residuals,  # for ControlNet
             mid_block_additional_residual=mid_block_additional_residual,  # for ControlNet
             down_intrablock_additional_residuals=down_intrablock_additional_residuals,  # for T2I-Adapter
@@ -605,7 +617,7 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
             )
 
         # compute the previous noisy sample x_t -> x_t-1
-        step_output = self.scheduler.step(noise_pred, timestep, latents, **scheduler_step_kwargs)
+        step_output = self.scheduler.step(noise_pred, timestep, latents, **conditioning_data.scheduler_args)
 
         # TODO: issue to diffusers?
         # undo internal counter increment done by scheduler.step, so timestep can be resolved as before call

invokeai/backend/stable_diffusion/diffusion/__init__.py

@@ -1,5 +1,6 @@
 """
 Initialization file for invokeai.models.diffusion
 """
-
+from .cross_attention_control import InvokeAICrossAttentionMixin  # noqa: F401
+from .cross_attention_map_saving import AttentionMapSaver  # noqa: F401
 from .shared_invokeai_diffusion import InvokeAIDiffuserComponent  # noqa: F401

invokeai/backend/stable_diffusion/diffusion/conditioning_data.py

@@ -1,5 +1,7 @@
-from dataclasses import dataclass
-from typing import List, Optional, Union
+import dataclasses
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, List, Optional, Union
 
 import torch
 
@@ -8,11 +10,6 @@ from .cross_attention_control import Arguments
 
 @dataclass
 class ExtraConditioningInfo:
-    """Extra conditioning information produced by Compel.
-
-    This is used for prompt-to-prompt cross-attention control (a.k.a. `.swap()` in Compel).
-    """
-
     tokens_count_including_eos_bos: int
     cross_attention_control_args: Optional[Arguments] = None
 
@@ -23,10 +20,12 @@ class ExtraConditioningInfo:
 
 @dataclass
 class BasicConditioningInfo:
-    """SD 1/2 text conditioning information produced by Compel."""
-
     embeds: torch.Tensor
+    # TODO(ryand): Right now we awkwardly copy the extra conditioning info from here up to `ConditioningData`. This
+    # should only be stored in one place.
     extra_conditioning: Optional[ExtraConditioningInfo]
+    # weight: float
+    # mode: ConditioningAlgo
 
     def to(self, device, dtype=None):
         self.embeds = self.embeds.to(device=device, dtype=dtype)
@@ -40,8 +39,6 @@ class ConditioningFieldData:
 
 @dataclass
 class SDXLConditioningInfo(BasicConditioningInfo):
-    """SDXL text conditioning information produced by Compel."""
-
     pooled_embeds: torch.Tensor
     add_time_ids: torch.Tensor
 
@@ -51,6 +48,14 @@ class SDXLConditioningInfo(BasicConditioningInfo):
         return super().to(device=device, dtype=dtype)
 
 
+@dataclass(frozen=True)
+class PostprocessingSettings:
+    threshold: float
+    warmup: float
+    h_symmetry_time_pct: Optional[float]
+    v_symmetry_time_pct: Optional[float]
+
+
 @dataclass
 class IPAdapterConditioningInfo:
     cond_image_prompt_embeds: torch.Tensor
@@ -64,55 +69,42 @@ class IPAdapterConditioningInfo:
 
 
 @dataclass
-class Range:
-    start: int
-    end: int
+class ConditioningData:
+    unconditioned_embeddings: BasicConditioningInfo
+    text_embeddings: BasicConditioningInfo
+    """
+    Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+    `guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://arxiv.org/pdf/2205.11487.pdf).
+    Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate
+    images that are closely linked to the text `prompt`, usually at the expense of lower image quality.
+    """
+    guidance_scale: Union[float, List[float]]
+    """ for models trained using zero-terminal SNR ("ztsnr"), it's suggested to use guidance_rescale_multiplier of 0.7 .
+     ref [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf)
+    """
+    guidance_rescale_multiplier: float = 0
+    extra: Optional[ExtraConditioningInfo] = None
+    scheduler_args: dict[str, Any] = field(default_factory=dict)
+    """
+    Additional arguments to pass to invokeai_diffuser.do_latent_postprocessing().
+    """
+    postprocessing_settings: Optional[PostprocessingSettings] = None
 
+    ip_adapter_conditioning: Optional[list[IPAdapterConditioningInfo]] = None
 
-class TextConditioningRegions:
-    def __init__(
-        self,
-        masks: torch.Tensor,
-        ranges: list[Range],
-        mask_weights: list[float],
-    ):
-        # A binary mask indicating the regions of the image that the prompt should be applied to.
-        # Shape: (1, num_prompts, height, width)
-        # Dtype: torch.bool
-        self.masks = masks
+    @property
+    def dtype(self):
+        return self.text_embeddings.dtype
 
-        # A list of ranges indicating the start and end indices of the embeddings that corresponding mask applies to.
-        # ranges[i] contains the embedding range for the i'th prompt / mask.
-        self.ranges = ranges
-
-        self.mask_weights = mask_weights
-
-        assert self.masks.shape[1] == len(self.ranges) == len(self.mask_weights)
-
-
-class TextConditioningData:
-    def __init__(
-        self,
-        uncond_text: Union[BasicConditioningInfo, SDXLConditioningInfo],
-        cond_text: Union[BasicConditioningInfo, SDXLConditioningInfo],
-        uncond_regions: Optional[TextConditioningRegions],
-        cond_regions: Optional[TextConditioningRegions],
-        guidance_scale: Union[float, List[float]],
-        guidance_rescale_multiplier: float = 0,
-    ):
-        self.uncond_text = uncond_text
-        self.cond_text = cond_text
-        self.uncond_regions = uncond_regions
-        self.cond_regions = cond_regions
-        # Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
-        # `guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://arxiv.org/pdf/2205.11487.pdf).
-        # Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate
-        # images that are closely linked to the text `prompt`, usually at the expense of lower image quality.
-        self.guidance_scale = guidance_scale
-        # For models trained using zero-terminal SNR ("ztsnr"), it's suggested to use guidance_rescale_multiplier of 0.7.
-        # See [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
-        self.guidance_rescale_multiplier = guidance_rescale_multiplier
-
-    def is_sdxl(self):
-        assert isinstance(self.uncond_text, SDXLConditioningInfo) == isinstance(self.cond_text, SDXLConditioningInfo)
-        return isinstance(self.cond_text, SDXLConditioningInfo)
+    def add_scheduler_args_if_applicable(self, scheduler, **kwargs):
+        scheduler_args = dict(self.scheduler_args)
+        step_method = inspect.signature(scheduler.step)
+        for name, value in kwargs.items():
+            try:
+                step_method.bind_partial(**{name: value})
+            except TypeError:
+                # FIXME: don't silently discard arguments
+                pass  # debug("%s does not accept argument named %r", scheduler, name)
+            else:
+                scheduler_args[name] = value
+        return dataclasses.replace(self, scheduler_args=scheduler_args)

invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py

@@ -3,13 +3,19 @@
 
 
 import enum
+import math
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Callable, Optional
 
+import diffusers
+import psutil
 import torch
 from compel.cross_attention_control import Arguments
-from diffusers.models.attention_processor import Attention, SlicedAttnProcessor
+from diffusers.models.attention_processor import Attention, AttentionProcessor, AttnProcessor, SlicedAttnProcessor
 from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel
+from torch import nn
+
+import invokeai.backend.util.logging as logger
 
 from ...util import torch_dtype
 
@@ -19,14 +25,72 @@ class CrossAttentionType(enum.Enum):
     TOKENS = 2
 
 
-class CrossAttnControlContext:
-    def __init__(self, arguments: Arguments):
+class Context:
+    cross_attention_mask: Optional[torch.Tensor]
+    cross_attention_index_map: Optional[torch.Tensor]
+
+    class Action(enum.Enum):
+        NONE = 0
+        SAVE = (1,)
+        APPLY = 2
+
+    def __init__(self, arguments: Arguments, step_count: int):
         """
         :param arguments: Arguments for the cross-attention control process
+        :param step_count: The absolute total number of steps of diffusion (for img2img this is likely larger than the number of steps that will actually run)
         """
-        self.cross_attention_mask: Optional[torch.Tensor] = None
-        self.cross_attention_index_map: Optional[torch.Tensor] = None
+        self.cross_attention_mask = None
+        self.cross_attention_index_map = None
+        self.self_cross_attention_action = Context.Action.NONE
+        self.tokens_cross_attention_action = Context.Action.NONE
         self.arguments = arguments
+        self.step_count = step_count
+
+        self.self_cross_attention_module_identifiers = []
+        self.tokens_cross_attention_module_identifiers = []
+
+        self.saved_cross_attention_maps = {}
+
+        self.clear_requests(cleanup=True)
+
+    def register_cross_attention_modules(self, model):
+        for name, _module in get_cross_attention_modules(model, CrossAttentionType.SELF):
+            if name in self.self_cross_attention_module_identifiers:
+                raise AssertionError(f"name {name} cannot appear more than once")
+            self.self_cross_attention_module_identifiers.append(name)
+        for name, _module in get_cross_attention_modules(model, CrossAttentionType.TOKENS):
+            if name in self.tokens_cross_attention_module_identifiers:
+                raise AssertionError(f"name {name} cannot appear more than once")
+            self.tokens_cross_attention_module_identifiers.append(name)
+
+    def request_save_attention_maps(self, cross_attention_type: CrossAttentionType):
+        if cross_attention_type == CrossAttentionType.SELF:
+            self.self_cross_attention_action = Context.Action.SAVE
+        else:
+            self.tokens_cross_attention_action = Context.Action.SAVE
+
+    def request_apply_saved_attention_maps(self, cross_attention_type: CrossAttentionType):
+        if cross_attention_type == CrossAttentionType.SELF:
+            self.self_cross_attention_action = Context.Action.APPLY
+        else:
+            self.tokens_cross_attention_action = Context.Action.APPLY
+
+    def is_tokens_cross_attention(self, module_identifier) -> bool:
+        return module_identifier in self.tokens_cross_attention_module_identifiers
+
+    def get_should_save_maps(self, module_identifier: str) -> bool:
+        if module_identifier in self.self_cross_attention_module_identifiers:
+            return self.self_cross_attention_action == Context.Action.SAVE
+        elif module_identifier in self.tokens_cross_attention_module_identifiers:
+            return self.tokens_cross_attention_action == Context.Action.SAVE
+        return False
+
+    def get_should_apply_saved_maps(self, module_identifier: str) -> bool:
+        if module_identifier in self.self_cross_attention_module_identifiers:
+            return self.self_cross_attention_action == Context.Action.APPLY
+        elif module_identifier in self.tokens_cross_attention_module_identifiers:
+            return self.tokens_cross_attention_action == Context.Action.APPLY
+        return False
 
     def get_active_cross_attention_control_types_for_step(
         self, percent_through: float = None
@@ -47,8 +111,219 @@ class CrossAttnControlContext:
             to_control.append(CrossAttentionType.TOKENS)
         return to_control
 
+    def save_slice(
+        self,
+        identifier: str,
+        slice: torch.Tensor,
+        dim: Optional[int],
+        offset: int,
+        slice_size: Optional[int],
+    ):
+        if identifier not in self.saved_cross_attention_maps:
+            self.saved_cross_attention_maps[identifier] = {
+                "dim": dim,
+                "slice_size": slice_size,
+                "slices": {offset or 0: slice},
+            }
+        else:
+            self.saved_cross_attention_maps[identifier]["slices"][offset or 0] = slice
 
-def setup_cross_attention_control_attention_processors(unet: UNet2DConditionModel, context: CrossAttnControlContext):
+    def get_slice(
+        self,
+        identifier: str,
+        requested_dim: Optional[int],
+        requested_offset: int,
+        slice_size: int,
+    ):
+        saved_attention_dict = self.saved_cross_attention_maps[identifier]
+        if requested_dim is None:
+            if saved_attention_dict["dim"] is not None:
+                raise RuntimeError(f"dim mismatch: expected dim=None, have {saved_attention_dict['dim']}")
+            return saved_attention_dict["slices"][0]
+
+        if saved_attention_dict["dim"] == requested_dim:
+            if slice_size != saved_attention_dict["slice_size"]:
+                raise RuntimeError(
+                    f"slice_size mismatch: expected slice_size={slice_size}, have {saved_attention_dict['slice_size']}"
+                )
+            return saved_attention_dict["slices"][requested_offset]
+
+        if saved_attention_dict["dim"] is None:
+            whole_saved_attention = saved_attention_dict["slices"][0]
+            if requested_dim == 0:
+                return whole_saved_attention[requested_offset : requested_offset + slice_size]
+            elif requested_dim == 1:
+                return whole_saved_attention[:, requested_offset : requested_offset + slice_size]
+
+        raise RuntimeError(f"Cannot convert dim {saved_attention_dict['dim']} to requested dim {requested_dim}")
+
+    def get_slicing_strategy(self, identifier: str) -> tuple[Optional[int], Optional[int]]:
+        saved_attention = self.saved_cross_attention_maps.get(identifier, None)
+        if saved_attention is None:
+            return None, None
+        return saved_attention["dim"], saved_attention["slice_size"]
+
+    def clear_requests(self, cleanup=True):
+        self.tokens_cross_attention_action = Context.Action.NONE
+        self.self_cross_attention_action = Context.Action.NONE
+        if cleanup:
+            self.saved_cross_attention_maps = {}
+
+    def offload_saved_attention_slices_to_cpu(self):
+        for _key, map_dict in self.saved_cross_attention_maps.items():
+            for offset, slice in map_dict["slices"].items():
+                map_dict[offset] = slice.to("cpu")
+
+
+class InvokeAICrossAttentionMixin:
+    """
+    Enable InvokeAI-flavoured Attention calculation, which does aggressive low-memory slicing and calls
+    through both to an attention_slice_wrangler and a slicing_strategy_getter for custom attention map wrangling
+    and dymamic slicing strategy selection.
+    """
+
+    def __init__(self):
+        self.mem_total_gb = psutil.virtual_memory().total // (1 << 30)
+        self.attention_slice_wrangler = None
+        self.slicing_strategy_getter = None
+        self.attention_slice_calculated_callback = None
+
+    def set_attention_slice_wrangler(
+        self,
+        wrangler: Optional[Callable[[nn.Module, torch.Tensor, int, int, int], torch.Tensor]],
+    ):
+        """
+        Set custom attention calculator to be called when attention is calculated
+        :param wrangler: Callback, with args (module, suggested_attention_slice, dim, offset, slice_size),
+        which returns either the suggested_attention_slice or an adjusted equivalent.
+            `module` is the current Attention module for which the callback is being invoked.
+            `suggested_attention_slice` is the default-calculated attention slice
+            `dim` is -1 if the attenion map has not been sliced, or 0 or 1 for dimension-0 or dimension-1 slicing.
+                If `dim` is >= 0, `offset` and `slice_size` specify the slice start and length.
+
+        Pass None to use the default attention calculation.
+        :return:
+        """
+        self.attention_slice_wrangler = wrangler
+
+    def set_slicing_strategy_getter(self, getter: Optional[Callable[[nn.Module], tuple[int, int]]]):
+        self.slicing_strategy_getter = getter
+
+    def set_attention_slice_calculated_callback(self, callback: Optional[Callable[[torch.Tensor], None]]):
+        self.attention_slice_calculated_callback = callback
+
+    def einsum_lowest_level(self, query, key, value, dim, offset, slice_size):
+        # calculate attention scores
+        # attention_scores = torch.einsum('b i d, b j d -> b i j', q, k)
+        attention_scores = torch.baddbmm(
+            torch.empty(
+                query.shape[0],
+                query.shape[1],
+                key.shape[1],
+                dtype=query.dtype,
+                device=query.device,
+            ),
+            query,
+            key.transpose(-1, -2),
+            beta=0,
+            alpha=self.scale,
+        )
+
+        # calculate attention slice by taking the best scores for each latent pixel
+        default_attention_slice = attention_scores.softmax(dim=-1, dtype=attention_scores.dtype)
+        attention_slice_wrangler = self.attention_slice_wrangler
+        if attention_slice_wrangler is not None:
+            attention_slice = attention_slice_wrangler(self, default_attention_slice, dim, offset, slice_size)
+        else:
+            attention_slice = default_attention_slice
+
+        if self.attention_slice_calculated_callback is not None:
+            self.attention_slice_calculated_callback(attention_slice, dim, offset, slice_size)
+
+        hidden_states = torch.bmm(attention_slice, value)
+        return hidden_states
+
+    def einsum_op_slice_dim0(self, q, k, v, slice_size):
+        r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype)
+        for i in range(0, q.shape[0], slice_size):
+            end = i + slice_size
+            r[i:end] = self.einsum_lowest_level(q[i:end], k[i:end], v[i:end], dim=0, offset=i, slice_size=slice_size)
+        return r
+
+    def einsum_op_slice_dim1(self, q, k, v, slice_size):
+        r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype)
+        for i in range(0, q.shape[1], slice_size):
+            end = i + slice_size
+            r[:, i:end] = self.einsum_lowest_level(q[:, i:end], k, v, dim=1, offset=i, slice_size=slice_size)
+        return r
+
+    def einsum_op_mps_v1(self, q, k, v):
+        if q.shape[1] <= 4096:  # (512x512) max q.shape[1]: 4096
+            return self.einsum_lowest_level(q, k, v, None, None, None)
+        else:
+            slice_size = math.floor(2**30 / (q.shape[0] * q.shape[1]))
+            return self.einsum_op_slice_dim1(q, k, v, slice_size)
+
+    def einsum_op_mps_v2(self, q, k, v):
+        if self.mem_total_gb > 8 and q.shape[1] <= 4096:
+            return self.einsum_lowest_level(q, k, v, None, None, None)
+        else:
+            return self.einsum_op_slice_dim0(q, k, v, 1)
+
+    def einsum_op_tensor_mem(self, q, k, v, max_tensor_mb):
+        size_mb = q.shape[0] * q.shape[1] * k.shape[1] * q.element_size() // (1 << 20)
+        if size_mb <= max_tensor_mb:
+            return self.einsum_lowest_level(q, k, v, None, None, None)
+        div = 1 << int((size_mb - 1) / max_tensor_mb).bit_length()
+        if div <= q.shape[0]:
+            return self.einsum_op_slice_dim0(q, k, v, q.shape[0] // div)
+        return self.einsum_op_slice_dim1(q, k, v, max(q.shape[1] // div, 1))
+
+    def einsum_op_cuda(self, q, k, v):
+        # check if we already have a slicing strategy (this should only happen during cross-attention controlled generation)
+        slicing_strategy_getter = self.slicing_strategy_getter
+        if slicing_strategy_getter is not None:
+            (dim, slice_size) = slicing_strategy_getter(self)
+            if dim is not None:
+                # print("using saved slicing strategy with dim", dim, "slice size", slice_size)
+                if dim == 0:
+                    return self.einsum_op_slice_dim0(q, k, v, slice_size)
+                elif dim == 1:
+                    return self.einsum_op_slice_dim1(q, k, v, slice_size)
+
+        # fallback for when there is no saved strategy, or saved strategy does not slice
+        mem_free_total = get_mem_free_total(q.device)
+        # Divide factor of safety as there's copying and fragmentation
+        return self.einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20))
+
+    def get_invokeai_attention_mem_efficient(self, q, k, v):
+        if q.device.type == "cuda":
+            # print("in get_attention_mem_efficient with q shape", q.shape, ", k shape", k.shape, ", free memory is", get_mem_free_total(q.device))
+            return self.einsum_op_cuda(q, k, v)
+
+        if q.device.type == "mps" or q.device.type == "cpu":
+            if self.mem_total_gb >= 32:
+                return self.einsum_op_mps_v1(q, k, v)
+            return self.einsum_op_mps_v2(q, k, v)
+
+        # Smaller slices are faster due to L2/L3/SLC caches.
+        # Tested on i7 with 8MB L3 cache.
+        return self.einsum_op_tensor_mem(q, k, v, 32)
+
+
+def restore_default_cross_attention(
+    model,
+    is_running_diffusers: bool,
+    restore_attention_processor: Optional[AttentionProcessor] = None,
+):
+    if is_running_diffusers:
+        unet = model
+        unet.set_attn_processor(restore_attention_processor or AttnProcessor())
+    else:
+        remove_attention_function(model)
+
+
+def setup_cross_attention_control_attention_processors(unet: UNet2DConditionModel, context: Context):
     """
     Inject attention parameters and functions into the passed in model to enable cross attention editing.
 
@@ -87,6 +362,170 @@ def setup_cross_attention_control_attention_processors(unet: UNet2DConditionMode
         unet.set_attn_processor(SlicedSwapCrossAttnProcesser(slice_size=slice_size))
 
 
+def get_cross_attention_modules(model, which: CrossAttentionType) -> list[tuple[str, InvokeAICrossAttentionMixin]]:
+    cross_attention_class: type = InvokeAIDiffusersCrossAttention
+    which_attn = "attn1" if which is CrossAttentionType.SELF else "attn2"
+    attention_module_tuples = [
+        (name, module)
+        for name, module in model.named_modules()
+        if isinstance(module, cross_attention_class) and which_attn in name
+    ]
+    cross_attention_modules_in_model_count = len(attention_module_tuples)
+    expected_count = 16
+    if cross_attention_modules_in_model_count != expected_count:
+        # non-fatal error but .swap() won't work.
+        logger.error(
+            f"Error! CrossAttentionControl found an unexpected number of {cross_attention_class} modules in the model "
+            f"(expected {expected_count}, found {cross_attention_modules_in_model_count}). Either monkey-patching "
+            "failed or some assumption has changed about the structure of the model itself. Please fix the "
+            f"monkey-patching, and/or update the {expected_count} above to an appropriate number, and/or find and "
+            "inform someone who knows what it means. This error is non-fatal, but it is likely that .swap() and "
+            "attention map display will not work properly until it is fixed."
+        )
+    return attention_module_tuples
+
+
+def inject_attention_function(unet, context: Context):
+    # ORIGINAL SOURCE CODE: https://github.com/huggingface/diffusers/blob/91ddd2a25b848df0fa1262d4f1cd98c7ccb87750/src/diffusers/models/attention.py#L276
+
+    def attention_slice_wrangler(module, suggested_attention_slice: torch.Tensor, dim, offset, slice_size):
+        # memory_usage = suggested_attention_slice.element_size() * suggested_attention_slice.nelement()
+
+        attention_slice = suggested_attention_slice
+
+        if context.get_should_save_maps(module.identifier):
+            # print(module.identifier, "saving suggested_attention_slice of shape",
+            #      suggested_attention_slice.shape, "dim", dim, "offset", offset)
+            slice_to_save = attention_slice.to("cpu") if dim is not None else attention_slice
+            context.save_slice(
+                module.identifier,
+                slice_to_save,
+                dim=dim,
+                offset=offset,
+                slice_size=slice_size,
+            )
+        elif context.get_should_apply_saved_maps(module.identifier):
+            # print(module.identifier, "applying saved attention slice for dim", dim, "offset", offset)
+            saved_attention_slice = context.get_slice(module.identifier, dim, offset, slice_size)
+
+            # slice may have been offloaded to CPU
+            saved_attention_slice = saved_attention_slice.to(suggested_attention_slice.device)
+
+            if context.is_tokens_cross_attention(module.identifier):
+                index_map = context.cross_attention_index_map
+                remapped_saved_attention_slice = torch.index_select(saved_attention_slice, -1, index_map)
+                this_attention_slice = suggested_attention_slice
+
+                mask = context.cross_attention_mask.to(torch_dtype(suggested_attention_slice.device))
+                saved_mask = mask
+                this_mask = 1 - mask
+                attention_slice = remapped_saved_attention_slice * saved_mask + this_attention_slice * this_mask
+            else:
+                # just use everything
+                attention_slice = saved_attention_slice
+
+        return attention_slice
+
+    cross_attention_modules = get_cross_attention_modules(
+        unet, CrossAttentionType.TOKENS
+    ) + get_cross_attention_modules(unet, CrossAttentionType.SELF)
+    for identifier, module in cross_attention_modules:
+        module.identifier = identifier
+        try:
+            module.set_attention_slice_wrangler(attention_slice_wrangler)
+            module.set_slicing_strategy_getter(lambda module: context.get_slicing_strategy(identifier))  # noqa: B023
+        except AttributeError as e:
+            if is_attribute_error_about(e, "set_attention_slice_wrangler"):
+                print(f"TODO: implement set_attention_slice_wrangler for {type(module)}")  # TODO
+            else:
+                raise
+
+
+def remove_attention_function(unet):
+    cross_attention_modules = get_cross_attention_modules(
+        unet, CrossAttentionType.TOKENS
+    ) + get_cross_attention_modules(unet, CrossAttentionType.SELF)
+    for _identifier, module in cross_attention_modules:
+        try:
+            # clear wrangler callback
+            module.set_attention_slice_wrangler(None)
+            module.set_slicing_strategy_getter(None)
+        except AttributeError as e:
+            if is_attribute_error_about(e, "set_attention_slice_wrangler"):
+                print(f"TODO: implement set_attention_slice_wrangler for {type(module)}")
+            else:
+                raise
+
+
+def is_attribute_error_about(error: AttributeError, attribute: str):
+    if hasattr(error, "name"):  # Python 3.10
+        return error.name == attribute
+    else:  # Python 3.9
+        return attribute in str(error)
+
+
+def get_mem_free_total(device):
+    # only on cuda
+    if not torch.cuda.is_available():
+        return None
+    stats = torch.cuda.memory_stats(device)
+    mem_active = stats["active_bytes.all.current"]
+    mem_reserved = stats["reserved_bytes.all.current"]
+    mem_free_cuda, _ = torch.cuda.mem_get_info(device)
+    mem_free_torch = mem_reserved - mem_active
+    mem_free_total = mem_free_cuda + mem_free_torch
+    return mem_free_total
+
+
+class InvokeAIDiffusersCrossAttention(diffusers.models.attention.Attention, InvokeAICrossAttentionMixin):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        InvokeAICrossAttentionMixin.__init__(self)
+
+    def _attention(self, query, key, value, attention_mask=None):
+        # default_result = super()._attention(query,  key, value)
+        if attention_mask is not None:
+            print(f"{type(self).__name__} ignoring passed-in attention_mask")
+        attention_result = self.get_invokeai_attention_mem_efficient(query, key, value)
+
+        hidden_states = self.reshape_batch_dim_to_heads(attention_result)
+        return hidden_states
+
+
+## 🧨diffusers implementation follows
+
+
+"""
+# base implementation
+
+class AttnProcessor:
+    def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None):
+        batch_size, sequence_length, _ = hidden_states.shape
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length)
+
+        query = attn.to_q(hidden_states)
+        query = attn.head_to_batch_dim(query)
+
+        encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        return hidden_states
+
+"""
+
+
 @dataclass
 class SwapCrossAttnContext:
     modified_text_embeddings: torch.Tensor
@@ -94,6 +533,18 @@ class SwapCrossAttnContext:
     mask: torch.Tensor  # in the target space of the index_map
     cross_attention_types_to_do: list[CrossAttentionType] = field(default_factory=list)
 
+    def __int__(
+        self,
+        cac_types_to_do: [CrossAttentionType],
+        modified_text_embeddings: torch.Tensor,
+        index_map: torch.Tensor,
+        mask: torch.Tensor,
+    ):
+        self.cross_attention_types_to_do = cac_types_to_do
+        self.modified_text_embeddings = modified_text_embeddings
+        self.index_map = index_map
+        self.mask = mask
+
     def wants_cross_attention_control(self, attn_type: CrossAttentionType) -> bool:
         return attn_type in self.cross_attention_types_to_do
 

invokeai/backend/stable_diffusion/diffusion/cross_attention_map_saving.py

@@ -0,0 +1,100 @@
+import math
+from typing import Optional
+
+import torch
+from PIL import Image
+from torchvision.transforms.functional import InterpolationMode
+from torchvision.transforms.functional import resize as tv_resize
+
+
+class AttentionMapSaver:
+    def __init__(self, token_ids: range, latents_shape: torch.Size):
+        self.token_ids = token_ids
+        self.latents_shape = latents_shape
+        # self.collated_maps = #torch.zeros([len(token_ids), latents_shape[0], latents_shape[1]])
+        self.collated_maps: dict[str, torch.Tensor] = {}
+
+    def clear_maps(self):
+        self.collated_maps = {}
+
+    def add_attention_maps(self, maps: torch.Tensor, key: str):
+        """
+        Accumulate the given attention maps and store by summing with existing maps at the passed-in key (if any).
+        :param maps: Attention maps to store. Expected shape [A, (H*W), N] where A is attention heads count, H and W are the map size (fixed per-key) and N is the number of tokens (typically 77).
+        :param key: Storage key. If a map already exists for this key it will be summed with the incoming data. In this case the maps sizes (H and W) should match.
+        :return: None
+        """
+        key_and_size = f"{key}_{maps.shape[1]}"
+
+        # extract desired tokens
+        maps = maps[:, :, self.token_ids]
+
+        # merge attention heads to a single map per token
+        maps = torch.sum(maps, 0)
+
+        # store
+        if key_and_size not in self.collated_maps:
+            self.collated_maps[key_and_size] = torch.zeros_like(maps, device="cpu")
+        self.collated_maps[key_and_size] += maps.cpu()
+
+    def write_maps_to_disk(self, path: str):
+        pil_image = self.get_stacked_maps_image()
+        if pil_image is not None:
+            pil_image.save(path, "PNG")
+
+    def get_stacked_maps_image(self) -> Optional[Image.Image]:
+        """
+        Scale all collected attention maps to the same size, blend them together and return as an image.
+        :return: An image containing a vertical stack of blended attention maps, one for each requested token.
+        """
+        num_tokens = len(self.token_ids)
+        if num_tokens == 0:
+            return None
+
+        latents_height = self.latents_shape[0]
+        latents_width = self.latents_shape[1]
+
+        merged = None
+
+        for _key, maps in self.collated_maps.items():
+            # maps has shape [(H*W), N] for N tokens
+            # but we want [N, H, W]
+            this_scale_factor = math.sqrt(maps.shape[0] / (latents_width * latents_height))
+            this_maps_height = int(float(latents_height) * this_scale_factor)
+            this_maps_width = int(float(latents_width) * this_scale_factor)
+            # and we need to do some dimension juggling
+            maps = torch.reshape(
+                torch.swapdims(maps, 0, 1),
+                [num_tokens, this_maps_height, this_maps_width],
+            )
+
+            # scale to output size if necessary
+            if this_scale_factor != 1:
+                maps = tv_resize(maps, [latents_height, latents_width], InterpolationMode.BICUBIC)
+
+            # normalize
+            maps_min = torch.min(maps)
+            maps_range = torch.max(maps) - maps_min
+            # print(f"map {key} size {[this_maps_width, this_maps_height]} range {[maps_min, maps_min + maps_range]}")
+            maps_normalized = (maps - maps_min) / maps_range
+            # expand to (-0.1, 1.1) and clamp
+            maps_normalized_expanded = maps_normalized * 1.1 - 0.05
+            maps_normalized_expanded_clamped = torch.clamp(maps_normalized_expanded, 0, 1)
+
+            # merge together, producing a vertical stack
+            maps_stacked = torch.reshape(
+                maps_normalized_expanded_clamped,
+                [num_tokens * latents_height, latents_width],
+            )
+
+            if merged is None:
+                merged = maps_stacked
+            else:
+                # screen blend
+                merged = 1 - (1 - maps_stacked) * (1 - merged)
+
+        if merged is None:
+            return None
+
+        merged_bytes = merged.mul(0xFF).byte()
+        return Image.fromarray(merged_bytes.numpy(), mode="L")

invokeai/backend/stable_diffusion/diffusion/custom_attention.py

@@ -1,242 +0,0 @@
-import math
-from typing import Optional
-
-import torch
-import torch.nn.functional as F
-from diffusers.models.attention_processor import Attention, AttnProcessor2_0
-from diffusers.utils import USE_PEFT_BACKEND
-
-from invokeai.backend.ip_adapter.ip_attention_weights import IPAttentionProcessorWeights
-from invokeai.backend.stable_diffusion.diffusion.regional_prompt_data import RegionalPromptData
-
-
-class CustomAttnProcessor2_0(AttnProcessor2_0):
-    """A custom implementation of AttnProcessor2_0 that supports additional Invoke features.
-
-    This implementation is based on
-    https://github.com/huggingface/diffusers/blame/fcfa270fbd1dc294e2f3a505bae6bcb791d721c3/src/diffusers/models/attention_processor.py#L1204
-
-    Supported custom features:
-    - IP-Adapter
-    - Regional prompt attention
-    """
-
-    def __init__(
-        self,
-        ip_adapter_weights: Optional[list[IPAttentionProcessorWeights]] = None,
-        ip_adapter_scales: Optional[list[float]] = None,
-    ):
-        """Initialize a CustomAttnProcessor2_0.
-
-        Note: Arguments that are the same for all attention layers are passed to __call__(). Arguments that are
-        layer-specific are passed to __init__().
-
-        Args:
-            ip_adapter_weights: The IP-Adapter attention weights. ip_adapter_weights[i] contains the attention weights
-                for the i'th IP-Adapter.
-            ip_adapter_scales: The IP-Adapter attention scales. ip_adapter_scales[i] contains the attention scale for
-                the i'th IP-Adapter.
-        """
-        super().__init__()
-
-        self._ip_adapter_weights = ip_adapter_weights
-        self._ip_adapter_scales = ip_adapter_scales
-
-        assert (self._ip_adapter_weights is None) == (self._ip_adapter_scales is None)
-        if self._ip_adapter_weights is not None:
-            assert len(ip_adapter_weights) == len(ip_adapter_scales)
-
-    def _is_ip_adapter_enabled(self) -> bool:
-        return self._ip_adapter_weights is not None
-
-    def __call__(
-        self,
-        attn: Attention,
-        hidden_states: torch.FloatTensor,
-        encoder_hidden_states: Optional[torch.FloatTensor] = None,
-        attention_mask: Optional[torch.FloatTensor] = None,
-        temb: Optional[torch.FloatTensor] = None,
-        scale: float = 1.0,
-        # For regional prompting:
-        regional_prompt_data: Optional[RegionalPromptData] = None,
-        percent_through: Optional[float] = None,
-        # For IP-Adapter:
-        ip_adapter_image_prompt_embeds: Optional[list[torch.Tensor]] = None,
-    ) -> torch.FloatTensor:
-        """Apply attention.
-
-        Args:
-            regional_prompt_data: The regional prompt data for the current batch. If not None, this will be used to
-                apply regional prompt masking.
-            ip_adapter_image_prompt_embeds: The IP-Adapter image prompt embeddings for the current batch.
-                ip_adapter_image_prompt_embeds[i] contains the image prompt embeddings for the i'th IP-Adapter. Each
-                tensor has shape (batch_size, num_ip_images, seq_len, ip_embedding_len).
-        """
-        # If true, we are doing cross-attention, if false we are doing self-attention.
-        is_cross_attention = encoder_hidden_states is not None
-
-        # Start unmodified block from AttnProcessor2_0.
-        # vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
-        residual = hidden_states
-        if attn.spatial_norm is not None:
-            hidden_states = attn.spatial_norm(hidden_states, temb)
-
-        input_ndim = hidden_states.ndim
-
-        if input_ndim == 4:
-            batch_size, channel, height, width = hidden_states.shape
-            hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
-
-        batch_size, sequence_length, _ = (
-            hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
-        )
-        # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-        # End unmodified block from AttnProcessor2_0.
-
-        # Handle regional prompt attention masks.
-        if regional_prompt_data is not None:
-            assert percent_through is not None
-            _, query_seq_len, _ = hidden_states.shape
-            if is_cross_attention:
-                prompt_region_attention_mask = regional_prompt_data.get_cross_attn_mask(
-                    query_seq_len=query_seq_len, key_seq_len=sequence_length
-                )
-                # TODO(ryand): Avoid redundant type/device conversion here.
-                prompt_region_attention_mask = prompt_region_attention_mask.to(
-                    dtype=hidden_states.dtype, device=hidden_states.device
-                )
-
-                attn_mask_weight = 1.0 * ((1 - percent_through) ** 5)
-            else:  # self-attention
-                prompt_region_attention_mask = regional_prompt_data.get_self_attn_mask(
-                    query_seq_len=query_seq_len,
-                    percent_through=percent_through,
-                )
-                attn_mask_weight = 0.3 * ((1 - percent_through) ** 5)
-
-        if attn.group_norm is not None:
-            hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
-
-        args = () if USE_PEFT_BACKEND else (scale,)
-        query = attn.to_q(hidden_states, *args)
-
-        if encoder_hidden_states is None:
-            encoder_hidden_states = hidden_states
-        elif attn.norm_cross:
-            encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
-
-        key = attn.to_k(encoder_hidden_states, *args)
-        value = attn.to_v(encoder_hidden_states, *args)
-
-        inner_dim = key.shape[-1]
-        head_dim = inner_dim // attn.heads
-
-        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
-
-        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
-        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
-
-        if attention_mask is not None:
-            attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
-            # scaled_dot_product_attention expects attention_mask shape to be
-            # (batch, heads, source_length, target_length)
-            attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
-
-        if regional_prompt_data is not None and percent_through < 0.3:
-            # Don't apply to uncond????
-
-            prompt_region_attention_mask = attn.prepare_attention_mask(
-                prompt_region_attention_mask, sequence_length, batch_size
-            )
-            # scaled_dot_product_attention expects attention_mask shape to be
-            # (batch, heads, source_length, target_length)
-            prompt_region_attention_mask = prompt_region_attention_mask.view(
-                batch_size, attn.heads, -1, prompt_region_attention_mask.shape[-1]
-            )
-
-            scale_factor = 1 / math.sqrt(query.size(-1))
-            attn_weight = query @ key.transpose(-2, -1) * scale_factor
-            m_pos = attn_weight.max(dim=-1, keepdim=True)[0] - attn_weight
-            m_neg = attn_weight - attn_weight.min(dim=-1, keepdim=True)[0]
-
-            prompt_region_attention_mask = attn_mask_weight * (
-                m_pos * prompt_region_attention_mask - m_neg * (1.0 - prompt_region_attention_mask)
-            )
-
-            if attention_mask is None:
-                attention_mask = prompt_region_attention_mask
-            else:
-                attention_mask = prompt_region_attention_mask + attention_mask
-        else:
-            pass
-
-        # the output of sdp = (batch, num_heads, seq_len, head_dim)
-        # TODO: add support for attn.scale when we move to Torch 2.1
-        hidden_states = F.scaled_dot_product_attention(
-            query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
-        )
-
-        hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
-        hidden_states = hidden_states.to(query.dtype)
-        # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-        # End unmodified block from AttnProcessor2_0.
-
-        # Apply IP-Adapter conditioning.
-        if is_cross_attention and self._is_ip_adapter_enabled():
-            if self._is_ip_adapter_enabled():
-                assert ip_adapter_image_prompt_embeds is not None
-                for ipa_embed, ipa_weights, scale in zip(
-                    ip_adapter_image_prompt_embeds, self._ip_adapter_weights, self._ip_adapter_scales, strict=True
-                ):
-                    # The batch dimensions should match.
-                    assert ipa_embed.shape[0] == encoder_hidden_states.shape[0]
-                    # The token_len dimensions should match.
-                    assert ipa_embed.shape[-1] == encoder_hidden_states.shape[-1]
-
-                    ip_hidden_states = ipa_embed
-
-                    # Expected ip_hidden_state shape: (batch_size, num_ip_images, ip_seq_len, ip_image_embedding)
-
-                    ip_key = ipa_weights.to_k_ip(ip_hidden_states)
-                    ip_value = ipa_weights.to_v_ip(ip_hidden_states)
-
-                    # Expected ip_key and ip_value shape: (batch_size, num_ip_images, ip_seq_len, head_dim * num_heads)
-
-                    ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
-                    ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
-
-                    # Expected ip_key and ip_value shape: (batch_size, num_heads, num_ip_images * ip_seq_len, head_dim)
-
-                    # TODO: add support for attn.scale when we move to Torch 2.1
-                    ip_hidden_states = F.scaled_dot_product_attention(
-                        query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
-                    )
-
-                    # Expected ip_hidden_states shape: (batch_size, num_heads, query_seq_len, head_dim)
-
-                    ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
-                    ip_hidden_states = ip_hidden_states.to(query.dtype)
-
-                    # Expected ip_hidden_states shape: (batch_size, query_seq_len, num_heads * head_dim)
-
-                    hidden_states = hidden_states + scale * ip_hidden_states
-            else:
-                # If IP-Adapter is not enabled, then ip_adapter_image_prompt_embeds should not be passed in.
-                assert ip_adapter_image_prompt_embeds is None
-
-        # Start unmodified block from AttnProcessor2_0.
-        # vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
-        # linear proj
-        hidden_states = attn.to_out[0](hidden_states, *args)
-        # dropout
-        hidden_states = attn.to_out[1](hidden_states)
-
-        if input_ndim == 4:
-            hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
-
-        if attn.residual_connection:
-            hidden_states = hidden_states + residual
-
-        hidden_states = hidden_states / attn.rescale_output_factor
-
-        return hidden_states

invokeai/backend/stable_diffusion/diffusion/regional_prompt_data.py

@@ -1,164 +0,0 @@
-import torch
-import torch.nn.functional as F
-
-from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
-    TextConditioningRegions,
-)
-
-
-class RegionalPromptData:
-    def __init__(
-        self,
-        regions: list[TextConditioningRegions],
-        device: torch.device,
-        dtype: torch.dtype,
-        max_downscale_factor: int = 8,
-    ):
-        """Initialize a `RegionalPromptData` object.
-
-        Args:
-            regions (list[TextConditioningRegions]): regions[i] contains the prompt regions for the i'th sample in the
-                batch.
-            device (torch.device): The device to use for the attention masks.
-            dtype (torch.dtype): The data type to use for the attention masks.
-            max_downscale_factor: Spatial masks will be prepared for downscale factors from 1 to max_downscale_factor
-                in steps of 2x.
-        """
-        self._regions = regions
-        self._device = device
-        self._dtype = dtype
-        # self._spatial_masks_by_seq_len[b][s] contains the spatial masks for the b'th batch sample with a query
-        # sequence length of s.
-        self._spatial_masks_by_seq_len: list[dict[int, torch.Tensor]] = self._prepare_spatial_masks(
-            regions, max_downscale_factor
-        )
-        self._negative_cross_attn_mask_score = 0.0
-        self._size_weight = 1.0
-
-    def _prepare_spatial_masks(
-        self, regions: list[TextConditioningRegions], max_downscale_factor: int = 8
-    ) -> list[dict[int, torch.Tensor]]:
-        """Prepare the spatial masks for all downscaling factors."""
-        # batch_masks_by_seq_len[b][s] contains the spatial masks for the b'th batch sample with a query sequence length
-        # of s.
-        batch_sample_masks_by_seq_len: list[dict[int, torch.Tensor]] = []
-
-        for batch_sample_regions in regions:
-            batch_sample_masks_by_seq_len.append({})
-
-            # Convert the bool masks to float masks so that max pooling can be applied.
-            batch_sample_masks = batch_sample_regions.masks.to(device=self._device, dtype=self._dtype)
-
-            # Downsample the spatial dimensions by factors of 2 until max_downscale_factor is reached.
-            downscale_factor = 1
-            while downscale_factor <= max_downscale_factor:
-                b, _num_prompts, h, w = batch_sample_masks.shape
-                assert b == 1
-                query_seq_len = h * w
-
-                batch_sample_masks_by_seq_len[-1][query_seq_len] = batch_sample_masks
-
-                downscale_factor *= 2
-                if downscale_factor <= max_downscale_factor:
-                    # We use max pooling because we downscale to a pretty low resolution, so we don't want small prompt
-                    # regions to be lost entirely.
-                    # TODO(ryand): In the future, we may want to experiment with other downsampling methods, and could
-                    # potentially use a weighted mask rather than a binary mask.
-                    batch_sample_masks = F.max_pool2d(batch_sample_masks, kernel_size=2, stride=2)
-
-        return batch_sample_masks_by_seq_len
-
-    def get_cross_attn_mask(self, query_seq_len: int, key_seq_len: int) -> torch.Tensor:
-        """Get the cross-attention mask for the given query sequence length.
-
-        Args:
-            query_seq_len: The length of the flattened spatial features at the current downscaling level.
-            key_seq_len (int): The sequence length of the prompt embeddings (which act as the key in the cross-attention
-                layers). This is most likely equal to the max embedding range end, but we pass it explicitly to be sure.
-
-        Returns:
-            torch.Tensor: The masks.
-                shape: (batch_size, query_seq_len, key_seq_len).
-                dtype: float
-                The mask is a binary mask with values of 0.0 and 1.0.
-        """
-        batch_size = len(self._spatial_masks_by_seq_len)
-        batch_spatial_masks = [self._spatial_masks_by_seq_len[b][query_seq_len] for b in range(batch_size)]
-
-        # Create an empty attention mask with the correct shape.
-        attn_mask = torch.zeros((batch_size, query_seq_len, key_seq_len), dtype=self._dtype, device=self._device)
-
-        for batch_idx in range(batch_size):
-            batch_sample_spatial_masks = batch_spatial_masks[batch_idx]
-            batch_sample_regions = self._regions[batch_idx]
-
-            # Flatten the spatial dimensions of the mask by reshaping to (1, num_prompts, query_seq_len, 1).
-            _, num_prompts, _, _ = batch_sample_spatial_masks.shape
-            batch_sample_query_masks = batch_sample_spatial_masks.view((1, num_prompts, query_seq_len, 1))
-
-            for prompt_idx, embedding_range in enumerate(batch_sample_regions.ranges):
-                batch_sample_query_scores = batch_sample_query_masks[0, prompt_idx, :, :]
-                size = batch_sample_query_scores.sum() / batch_sample_query_scores.numel()
-                mask_weight = batch_sample_regions.mask_weights[prompt_idx]
-                # size = size.to(dtype=batch_sample_query_scores.dtype)
-                # batch_sample_query_mask = batch_sample_query_scores > 0.5
-                # batch_sample_query_scores[batch_sample_query_mask] = 1.0 * (1.0 - size)
-                # batch_sample_query_scores[~batch_sample_query_mask] = 0.0
-                attn_mask[batch_idx, :, embedding_range.start : embedding_range.end] = batch_sample_query_scores * (
-                    mask_weight + self._size_weight * (1 - size)
-                )
-
-        return attn_mask
-
-    def get_self_attn_mask(self, query_seq_len: int, percent_through: float) -> torch.Tensor:
-        """Get the self-attention mask for the given query sequence length.
-
-        Args:
-            query_seq_len: The length of the flattened spatial features at the current downscaling level.
-
-        Returns:
-            torch.Tensor: The masks.
-                shape: (batch_size, query_seq_len, query_seq_len).
-                dtype: float
-                The mask is a binary mask with values of 0.0 and 1.0.
-        """
-        batch_size = len(self._spatial_masks_by_seq_len)
-        batch_spatial_masks = [self._spatial_masks_by_seq_len[b][query_seq_len] for b in range(batch_size)]
-
-        # Create an empty attention mask with the correct shape.
-        attn_mask = torch.zeros((batch_size, query_seq_len, query_seq_len), dtype=self._dtype, device=self._device)
-
-        for batch_idx in range(batch_size):
-            batch_sample_spatial_masks = batch_spatial_masks[batch_idx]
-            batch_sample_regions = self._regions[batch_idx]
-
-            # Flatten the spatial dimensions of the mask by reshaping to (1, num_prompts, query_seq_len, 1).
-            _, num_prompts, _, _ = batch_sample_spatial_masks.shape
-            batch_sample_query_masks = batch_sample_spatial_masks.view((1, num_prompts, query_seq_len, 1))
-
-            for prompt_idx in range(num_prompts):
-                prompt_query_mask = batch_sample_query_masks[0, prompt_idx, :, 0]  # Shape: (query_seq_len,)
-                size = prompt_query_mask.sum() / prompt_query_mask.numel()
-                size = size.to(dtype=prompt_query_mask.dtype)
-                mask_weight = batch_sample_regions.mask_weights[prompt_idx]
-                # Multiply a (1, query_seq_len) mask by a (query_seq_len, 1) mask to get a (query_seq_len,
-                # query_seq_len) mask.
-                # TODO(ryand): Is += really the best option here? Maybe elementwise max is better?
-                attn_mask[batch_idx, :, :] = torch.maximum(
-                    attn_mask[batch_idx, :, :],
-                    prompt_query_mask.unsqueeze(0)
-                    * prompt_query_mask.unsqueeze(1)
-                    * (mask_weight + self._size_weight * (1 - size)),
-                )
-
-            # if attn_mask[batch_idx].max() < 0.01:
-            #     attn_mask[batch_idx, ...] = 1.0
-
-        # attn_mask[attn_mask > 0.5] = 1.0
-        # attn_mask[attn_mask <= 0.5] = 0.0
-        # attn_mask_min = attn_mask[batch_idx].min()
-
-        # # Adjust so that the minimum value is 0.0 regardless of whether all pixels are covered or not.
-        # if abs(attn_mask_min) > 0.0001:
-        #     attn_mask[batch_idx] = attn_mask[batch_idx] - attn_mask_min
-        return attn_mask

invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import math
-import time
 from contextlib import contextmanager
 from typing import Any, Callable, Optional, Union
 
@@ -11,20 +10,20 @@ from typing_extensions import TypeAlias
 
 from invokeai.app.services.config import InvokeAIAppConfig
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
+    ConditioningData,
     ExtraConditioningInfo,
-    IPAdapterConditioningInfo,
-    Range,
-    TextConditioningData,
-    TextConditioningRegions,
+    PostprocessingSettings,
+    SDXLConditioningInfo,
 )
-from invokeai.backend.stable_diffusion.diffusion.regional_prompt_data import RegionalPromptData
 
 from .cross_attention_control import (
+    Context,
     CrossAttentionType,
-    CrossAttnControlContext,
     SwapCrossAttnContext,
+    get_cross_attention_modules,
     setup_cross_attention_control_attention_processors,
 )
+from .cross_attention_map_saving import AttentionMapSaver
 
 ModelForwardCallback: TypeAlias = Union[
     # x, t, conditioning, Optional[cross-attention kwargs]
@@ -59,6 +58,7 @@ class InvokeAIDiffuserComponent:
         :param model_forward_callback: a lambda with arguments (x, sigma, conditioning_to_apply). will be called repeatedly. most likely, this should simply call model.forward(x, sigma, conditioning)
         """
         config = InvokeAIAppConfig.get_config()
+        self.conditioning = None
         self.model = model
         self.model_forward_callback = model_forward_callback
         self.cross_attention_control_context = None
@@ -69,12 +69,14 @@ class InvokeAIDiffuserComponent:
         self,
         unet: UNet2DConditionModel,
         extra_conditioning_info: Optional[ExtraConditioningInfo],
+        step_count: int,
     ):
         old_attn_processors = unet.attn_processors
 
         try:
-            self.cross_attention_control_context = CrossAttnControlContext(
+            self.cross_attention_control_context = Context(
                 arguments=extra_conditioning_info.cross_attention_control_args,
+                step_count=step_count,
             )
             setup_cross_attention_control_attention_processors(
                 unet,
@@ -85,6 +87,27 @@ class InvokeAIDiffuserComponent:
         finally:
             self.cross_attention_control_context = None
             unet.set_attn_processor(old_attn_processors)
+            # TODO resuscitate attention map saving
+            # self.remove_attention_map_saving()
+
+    def setup_attention_map_saving(self, saver: AttentionMapSaver):
+        def callback(slice, dim, offset, slice_size, key):
+            if dim is not None:
+                # sliced tokens attention map saving is not implemented
+                return
+            saver.add_attention_maps(slice, key)
+
+        tokens_cross_attention_modules = get_cross_attention_modules(self.model, CrossAttentionType.TOKENS)
+        for identifier, module in tokens_cross_attention_modules:
+            key = "down" if identifier.startswith("down") else "up" if identifier.startswith("up") else "mid"
+            module.set_attention_slice_calculated_callback(
+                lambda slice, dim, offset, slice_size, key=key: callback(slice, dim, offset, slice_size, key)
+            )
+
+    def remove_attention_map_saving(self):
+        tokens_cross_attention_modules = get_cross_attention_modules(self.model, CrossAttentionType.TOKENS)
+        for _, module in tokens_cross_attention_modules:
+            module.set_attention_slice_calculated_callback(None)
 
     def do_controlnet_step(
         self,
@@ -93,7 +116,7 @@ class InvokeAIDiffuserComponent:
         timestep: torch.Tensor,
         step_index: int,
         total_step_count: int,
-        conditioning_data: TextConditioningData,
+        conditioning_data,
     ):
         down_block_res_samples, mid_block_res_sample = None, None
 
@@ -126,30 +149,38 @@ class InvokeAIDiffuserComponent:
                 added_cond_kwargs = None
 
                 if cfg_injection:  # only applying ControlNet to conditional instead of in unconditioned
-                    if conditioning_data.is_sdxl():
+                    if type(conditioning_data.text_embeddings) is SDXLConditioningInfo:
                         added_cond_kwargs = {
-                            "text_embeds": conditioning_data.cond_text.pooled_embeds,
-                            "time_ids": conditioning_data.cond_text.add_time_ids,
+                            "text_embeds": conditioning_data.text_embeddings.pooled_embeds,
+                            "time_ids": conditioning_data.text_embeddings.add_time_ids,
                         }
-                    encoder_hidden_states = conditioning_data.cond_text.embeds
+                    encoder_hidden_states = conditioning_data.text_embeddings.embeds
                     encoder_attention_mask = None
                 else:
-                    if conditioning_data.is_sdxl():
+                    if type(conditioning_data.text_embeddings) is SDXLConditioningInfo:
                         added_cond_kwargs = {
                             "text_embeds": torch.cat(
                                 [
-                                    conditioning_data.uncond_text.pooled_embeds,
-                                    conditioning_data.cond_text.pooled_embeds,
+                                    # TODO: how to pad? just by zeros? or even truncate?
+                                    conditioning_data.unconditioned_embeddings.pooled_embeds,
+                                    conditioning_data.text_embeddings.pooled_embeds,
                                 ],
                                 dim=0,
                             ),
                             "time_ids": torch.cat(
-                                [conditioning_data.uncond_text.add_time_ids, conditioning_data.cond_text.add_time_ids],
+                                [
+                                    conditioning_data.unconditioned_embeddings.add_time_ids,
+                                    conditioning_data.text_embeddings.add_time_ids,
+                                ],
                                 dim=0,
                             ),
                         }
-                    (encoder_hidden_states, encoder_attention_mask) = self._concat_conditionings_for_batch(
-                        conditioning_data.uncond_text.embeds, conditioning_data.cond_text.embeds
+                    (
+                        encoder_hidden_states,
+                        encoder_attention_mask,
+                    ) = self._concat_conditionings_for_batch(
+                        conditioning_data.unconditioned_embeddings.embeds,
+                        conditioning_data.text_embeddings.embeds,
                     )
                 if isinstance(control_datum.weight, list):
                     # if controlnet has multiple weights, use the weight for the current step
@@ -193,56 +224,68 @@ class InvokeAIDiffuserComponent:
         self,
         sample: torch.Tensor,
         timestep: torch.Tensor,
-        conditioning_data: TextConditioningData,
-        ip_adapter_conditioning: Optional[list[IPAdapterConditioningInfo]],
+        conditioning_data,  # TODO: type
         step_index: int,
         total_step_count: int,
-        down_block_additional_residuals: Optional[torch.Tensor] = None,  # for ControlNet
-        mid_block_additional_residual: Optional[torch.Tensor] = None,  # for ControlNet
-        down_intrablock_additional_residuals: Optional[torch.Tensor] = None,  # for T2I-Adapter
+        **kwargs,
     ):
-        percent_through = step_index / total_step_count
         cross_attention_control_types_to_do = []
+        context: Context = self.cross_attention_control_context
         if self.cross_attention_control_context is not None:
-            cross_attention_control_types_to_do = (
-                self.cross_attention_control_context.get_active_cross_attention_control_types_for_step(percent_through)
+            percent_through = step_index / total_step_count
+            cross_attention_control_types_to_do = context.get_active_cross_attention_control_types_for_step(
+                percent_through
             )
+
         wants_cross_attention_control = len(cross_attention_control_types_to_do) > 0
 
-        if wants_cross_attention_control or self.sequential_guidance:
-            # If wants_cross_attention_control is True, we force the sequential mode to be used, because cross-attention
-            # control is currently only supported in sequential mode.
+        if wants_cross_attention_control:
+            (
+                unconditioned_next_x,
+                conditioned_next_x,
+            ) = self._apply_cross_attention_controlled_conditioning(
+                sample,
+                timestep,
+                conditioning_data,
+                cross_attention_control_types_to_do,
+                **kwargs,
+            )
+        elif self.sequential_guidance:
             (
                 unconditioned_next_x,
                 conditioned_next_x,
             ) = self._apply_standard_conditioning_sequentially(
-                x=sample,
-                sigma=timestep,
-                conditioning_data=conditioning_data,
-                ip_adapter_conditioning=ip_adapter_conditioning,
-                percent_through=percent_through,
-                cross_attention_control_types_to_do=cross_attention_control_types_to_do,
-                down_block_additional_residuals=down_block_additional_residuals,
-                mid_block_additional_residual=mid_block_additional_residual,
-                down_intrablock_additional_residuals=down_intrablock_additional_residuals,
+                sample,
+                timestep,
+                conditioning_data,
+                **kwargs,
             )
         else:
             (
                 unconditioned_next_x,
                 conditioned_next_x,
             ) = self._apply_standard_conditioning(
-                x=sample,
-                sigma=timestep,
-                conditioning_data=conditioning_data,
-                percent_through=percent_through,
-                ip_adapter_conditioning=ip_adapter_conditioning,
-                down_block_additional_residuals=down_block_additional_residuals,
-                mid_block_additional_residual=mid_block_additional_residual,
-                down_intrablock_additional_residuals=down_intrablock_additional_residuals,
+                sample,
+                timestep,
+                conditioning_data,
+                **kwargs,
             )
 
         return unconditioned_next_x, conditioned_next_x
 
+    def do_latent_postprocessing(
+        self,
+        postprocessing_settings: PostprocessingSettings,
+        latents: torch.Tensor,
+        sigma,
+        step_index,
+        total_step_count,
+    ) -> torch.Tensor:
+        if postprocessing_settings is not None:
+            percent_through = step_index / total_step_count
+            latents = self.apply_symmetry(postprocessing_settings, latents, percent_through)
+        return latents
+
     def _concat_conditionings_for_batch(self, unconditioning, conditioning):
         def _pad_conditioning(cond, target_len, encoder_attention_mask):
             conditioning_attention_mask = torch.ones(
@@ -290,82 +333,58 @@ class InvokeAIDiffuserComponent:
 
         return torch.cat([unconditioning, conditioning]), encoder_attention_mask
 
-    def _apply_standard_conditioning(
-        self,
-        x,
-        sigma,
-        conditioning_data: TextConditioningData,
-        ip_adapter_conditioning: Optional[list[IPAdapterConditioningInfo]],
-        percent_through: float,
-        down_block_additional_residuals: Optional[torch.Tensor] = None,  # for ControlNet
-        mid_block_additional_residual: Optional[torch.Tensor] = None,  # for ControlNet
-        down_intrablock_additional_residuals: Optional[torch.Tensor] = None,  # for T2I-Adapter
-    ):
+    # methods below are called from do_diffusion_step and should be considered private to this class.
+
+    def _apply_standard_conditioning(self, x, sigma, conditioning_data: ConditioningData, **kwargs):
         """Runs the conditioned and unconditioned UNet forward passes in a single batch for faster inference speed at
         the cost of higher memory usage.
         """
         x_twice = torch.cat([x] * 2)
         sigma_twice = torch.cat([sigma] * 2)
 
-        cross_attention_kwargs = {}
-        if ip_adapter_conditioning is not None:
+        cross_attention_kwargs = None
+        if conditioning_data.ip_adapter_conditioning is not None:
             # Note that we 'stack' to produce tensors of shape (batch_size, num_ip_images, seq_len, token_len).
-            cross_attention_kwargs["ip_adapter_image_prompt_embeds"] = [
-                torch.stack([ipa_conditioning.uncond_image_prompt_embeds, ipa_conditioning.cond_image_prompt_embeds])
-                for ipa_conditioning in ip_adapter_conditioning
-            ]
-
-        uncond_text = conditioning_data.uncond_text
-        cond_text = conditioning_data.cond_text
+            cross_attention_kwargs = {
+                "ip_adapter_image_prompt_embeds": [
+                    torch.stack(
+                        [ipa_conditioning.uncond_image_prompt_embeds, ipa_conditioning.cond_image_prompt_embeds]
+                    )
+                    for ipa_conditioning in conditioning_data.ip_adapter_conditioning
+                ]
+            }
 
         added_cond_kwargs = None
-        if conditioning_data.is_sdxl():
+        if type(conditioning_data.text_embeddings) is SDXLConditioningInfo:
             added_cond_kwargs = {
-                "text_embeds": torch.cat([uncond_text.pooled_embeds, cond_text.pooled_embeds], dim=0),
-                "time_ids": torch.cat([uncond_text.add_time_ids, cond_text.add_time_ids], dim=0),
+                "text_embeds": torch.cat(
+                    [
+                        # TODO: how to pad? just by zeros? or even truncate?
+                        conditioning_data.unconditioned_embeddings.pooled_embeds,
+                        conditioning_data.text_embeddings.pooled_embeds,
+                    ],
+                    dim=0,
+                ),
+                "time_ids": torch.cat(
+                    [
+                        conditioning_data.unconditioned_embeddings.add_time_ids,
+                        conditioning_data.text_embeddings.add_time_ids,
+                    ],
+                    dim=0,
+                ),
             }
 
         both_conditionings, encoder_attention_mask = self._concat_conditionings_for_batch(
-            uncond_text.embeds, cond_text.embeds
+            conditioning_data.unconditioned_embeddings.embeds, conditioning_data.text_embeddings.embeds
         )
-
-        if conditioning_data.cond_regions is not None or conditioning_data.uncond_regions is not None:
-            # TODO(ryand): We currently initialize RegionalPromptData for every denoising step. The text conditionings
-            # and masks are not changing from step-to-step, so this really only needs to be done once. While this seems
-            # painfully inefficient, the time spent is typically negligible compared to the forward inference pass of
-            # the UNet. The main reason that this hasn't been moved up to eliminate redundancy is that it is slightly
-            # awkward to handle both standard conditioning and sequential conditioning further up the stack.
-            regions = []
-            for c, r in [
-                (conditioning_data.uncond_text, conditioning_data.uncond_regions),
-                (conditioning_data.cond_text, conditioning_data.cond_regions),
-            ]:
-                if r is None:
-                    # Create a dummy mask and range for text conditioning that doesn't have region masks.
-                    _, _, h, w = x.shape
-                    r = TextConditioningRegions(
-                        masks=torch.ones((1, 1, h, w), dtype=torch.bool),
-                        ranges=[Range(start=0, end=c.embeds.shape[1])],
-                        mask_weights=[0.0],
-                    )
-                regions.append(r)
-
-            cross_attention_kwargs["regional_prompt_data"] = RegionalPromptData(
-                regions=regions, device=x.device, dtype=x.dtype
-            )
-            cross_attention_kwargs["percent_through"] = percent_through
-            time.sleep(1.0)
-
         both_results = self.model_forward_callback(
             x_twice,
             sigma_twice,
             both_conditionings,
             cross_attention_kwargs=cross_attention_kwargs,
             encoder_attention_mask=encoder_attention_mask,
-            down_block_additional_residuals=down_block_additional_residuals,
-            mid_block_additional_residual=mid_block_additional_residual,
-            down_intrablock_additional_residuals=down_intrablock_additional_residuals,
             added_cond_kwargs=added_cond_kwargs,
+            **kwargs,
         )
         unconditioned_next_x, conditioned_next_x = both_results.chunk(2)
         return unconditioned_next_x, conditioned_next_x
@@ -374,20 +393,15 @@ class InvokeAIDiffuserComponent:
         self,
         x: torch.Tensor,
         sigma,
-        conditioning_data: TextConditioningData,
-        ip_adapter_conditioning: Optional[list[IPAdapterConditioningInfo]],
-        cross_attention_control_types_to_do: list[CrossAttentionType],
-        percent_through: float,
-        down_block_additional_residuals: Optional[torch.Tensor] = None,  # for ControlNet
-        mid_block_additional_residual: Optional[torch.Tensor] = None,  # for ControlNet
-        down_intrablock_additional_residuals: Optional[torch.Tensor] = None,  # for T2I-Adapter
+        conditioning_data: ConditioningData,
+        **kwargs,
     ):
         """Runs the conditioned and unconditioned UNet forward passes sequentially for lower memory usage at the cost of
         slower execution speed.
         """
-        # Since we are running the conditioned and unconditioned passes sequentially, we need to split the ControlNet
-        # and T2I-Adapter residuals into two chunks.
+        # low-memory sequential path
         uncond_down_block, cond_down_block = None, None
+        down_block_additional_residuals = kwargs.pop("down_block_additional_residuals", None)
         if down_block_additional_residuals is not None:
             uncond_down_block, cond_down_block = [], []
             for down_block in down_block_additional_residuals:
@@ -396,6 +410,7 @@ class InvokeAIDiffuserComponent:
                 cond_down_block.append(_cond_down)
 
         uncond_down_intrablock, cond_down_intrablock = None, None
+        down_intrablock_additional_residuals = kwargs.pop("down_intrablock_additional_residuals", None)
         if down_intrablock_additional_residuals is not None:
             uncond_down_intrablock, cond_down_intrablock = [], []
             for down_intrablock in down_intrablock_additional_residuals:
@@ -404,111 +419,151 @@ class InvokeAIDiffuserComponent:
                 cond_down_intrablock.append(_cond_down)
 
         uncond_mid_block, cond_mid_block = None, None
+        mid_block_additional_residual = kwargs.pop("mid_block_additional_residual", None)
         if mid_block_additional_residual is not None:
             uncond_mid_block, cond_mid_block = mid_block_additional_residual.chunk(2)
 
-        # If cross-attention control is enabled, prepare the SwapCrossAttnContext.
-        cross_attn_processor_context = None
-        if self.cross_attention_control_context is not None:
-            # Note that the SwapCrossAttnContext is initialized with an empty list of cross_attention_types_to_do.
-            # This list is empty because cross-attention control is not applied in the unconditioned pass. This field
-            # will be populated before the conditioned pass.
-            cross_attn_processor_context = SwapCrossAttnContext(
-                modified_text_embeddings=self.cross_attention_control_context.arguments.edited_conditioning,
-                index_map=self.cross_attention_control_context.cross_attention_index_map,
-                mask=self.cross_attention_control_context.cross_attention_mask,
-                cross_attention_types_to_do=[],
-            )
-
-        #####################
-        # Unconditioned pass
-        #####################
-
-        cross_attention_kwargs = {}
-
-        # Prepare IP-Adapter cross-attention kwargs for the unconditioned pass.
-        if ip_adapter_conditioning is not None:
+        # Run unconditional UNet denoising.
+        cross_attention_kwargs = None
+        if conditioning_data.ip_adapter_conditioning is not None:
             # Note that we 'unsqueeze' to produce tensors of shape (batch_size=1, num_ip_images, seq_len, token_len).
-            cross_attention_kwargs["ip_adapter_image_prompt_embeds"] = [
-                torch.unsqueeze(ipa_conditioning.uncond_image_prompt_embeds, dim=0)
-                for ipa_conditioning in ip_adapter_conditioning
-            ]
-
-        # Prepare cross-attention control kwargs for the unconditioned pass.
-        if cross_attn_processor_context is not None:
-            cross_attention_kwargs["swap_cross_attn_context"] = cross_attn_processor_context
-
-        # Prepare SDXL conditioning kwargs for the unconditioned pass.
-        added_cond_kwargs = None
-        if conditioning_data.is_sdxl():
-            added_cond_kwargs = {
-                "text_embeds": conditioning_data.uncond_text.pooled_embeds,
-                "time_ids": conditioning_data.uncond_text.add_time_ids,
+            cross_attention_kwargs = {
+                "ip_adapter_image_prompt_embeds": [
+                    torch.unsqueeze(ipa_conditioning.uncond_image_prompt_embeds, dim=0)
+                    for ipa_conditioning in conditioning_data.ip_adapter_conditioning
+                ]
             }
 
-        # Prepare prompt regions for the unconditioned pass.
-        if conditioning_data.uncond_regions is not None:
-            cross_attention_kwargs["regional_prompt_data"] = RegionalPromptData(
-                regions=[conditioning_data.uncond_regions], device=x.device, dtype=x.dtype
-            )
-            cross_attention_kwargs["percent_through"] = percent_through
+        added_cond_kwargs = None
+        is_sdxl = type(conditioning_data.text_embeddings) is SDXLConditioningInfo
+        if is_sdxl:
+            added_cond_kwargs = {
+                "text_embeds": conditioning_data.unconditioned_embeddings.pooled_embeds,
+                "time_ids": conditioning_data.unconditioned_embeddings.add_time_ids,
+            }
 
-        # Run unconditioned UNet denoising (i.e. negative prompt).
         unconditioned_next_x = self.model_forward_callback(
             x,
             sigma,
-            conditioning_data.uncond_text.embeds,
+            conditioning_data.unconditioned_embeddings.embeds,
             cross_attention_kwargs=cross_attention_kwargs,
             down_block_additional_residuals=uncond_down_block,
             mid_block_additional_residual=uncond_mid_block,
             down_intrablock_additional_residuals=uncond_down_intrablock,
             added_cond_kwargs=added_cond_kwargs,
+            **kwargs,
         )
 
-        ###################
-        # Conditioned pass
-        ###################
-
-        cross_attention_kwargs = {}
-
-        # Prepare IP-Adapter cross-attention kwargs for the conditioned pass.
-        if ip_adapter_conditioning is not None:
+        # Run conditional UNet denoising.
+        cross_attention_kwargs = None
+        if conditioning_data.ip_adapter_conditioning is not None:
             # Note that we 'unsqueeze' to produce tensors of shape (batch_size=1, num_ip_images, seq_len, token_len).
-            cross_attention_kwargs["ip_adapter_image_prompt_embeds"] = [
-                torch.unsqueeze(ipa_conditioning.cond_image_prompt_embeds, dim=0)
-                for ipa_conditioning in ip_adapter_conditioning
-            ]
-
-        # Prepare cross-attention control kwargs for the conditioned pass.
-        if cross_attn_processor_context is not None:
-            cross_attn_processor_context.cross_attention_types_to_do = cross_attention_control_types_to_do
-            cross_attention_kwargs["swap_cross_attn_context"] = cross_attn_processor_context
-
-        # Prepare SDXL conditioning kwargs for the conditioned pass.
-        added_cond_kwargs = None
-        if conditioning_data.is_sdxl():
-            added_cond_kwargs = {
-                "text_embeds": conditioning_data.cond_text.pooled_embeds,
-                "time_ids": conditioning_data.cond_text.add_time_ids,
+            cross_attention_kwargs = {
+                "ip_adapter_image_prompt_embeds": [
+                    torch.unsqueeze(ipa_conditioning.cond_image_prompt_embeds, dim=0)
+                    for ipa_conditioning in conditioning_data.ip_adapter_conditioning
+                ]
             }
 
-        # Prepare prompt regions for the conditioned pass.
-        if conditioning_data.cond_regions is not None:
-            cross_attention_kwargs["regional_prompt_data"] = RegionalPromptData(
-                regions=[conditioning_data.cond_regions], device=x.device, dtype=x.dtype
-            )
-            cross_attention_kwargs["percent_through"] = percent_through
+        added_cond_kwargs = None
+        if is_sdxl:
+            added_cond_kwargs = {
+                "text_embeds": conditioning_data.text_embeddings.pooled_embeds,
+                "time_ids": conditioning_data.text_embeddings.add_time_ids,
+            }
 
-        # Run conditioned UNet denoising (i.e. positive prompt).
         conditioned_next_x = self.model_forward_callback(
             x,
             sigma,
-            conditioning_data.cond_text.embeds,
+            conditioning_data.text_embeddings.embeds,
             cross_attention_kwargs=cross_attention_kwargs,
             down_block_additional_residuals=cond_down_block,
             mid_block_additional_residual=cond_mid_block,
             down_intrablock_additional_residuals=cond_down_intrablock,
             added_cond_kwargs=added_cond_kwargs,
+            **kwargs,
+        )
+        return unconditioned_next_x, conditioned_next_x
+
+    def _apply_cross_attention_controlled_conditioning(
+        self,
+        x: torch.Tensor,
+        sigma,
+        conditioning_data,
+        cross_attention_control_types_to_do,
+        **kwargs,
+    ):
+        context: Context = self.cross_attention_control_context
+
+        uncond_down_block, cond_down_block = None, None
+        down_block_additional_residuals = kwargs.pop("down_block_additional_residuals", None)
+        if down_block_additional_residuals is not None:
+            uncond_down_block, cond_down_block = [], []
+            for down_block in down_block_additional_residuals:
+                _uncond_down, _cond_down = down_block.chunk(2)
+                uncond_down_block.append(_uncond_down)
+                cond_down_block.append(_cond_down)
+
+        uncond_down_intrablock, cond_down_intrablock = None, None
+        down_intrablock_additional_residuals = kwargs.pop("down_intrablock_additional_residuals", None)
+        if down_intrablock_additional_residuals is not None:
+            uncond_down_intrablock, cond_down_intrablock = [], []
+            for down_intrablock in down_intrablock_additional_residuals:
+                _uncond_down, _cond_down = down_intrablock.chunk(2)
+                uncond_down_intrablock.append(_uncond_down)
+                cond_down_intrablock.append(_cond_down)
+
+        uncond_mid_block, cond_mid_block = None, None
+        mid_block_additional_residual = kwargs.pop("mid_block_additional_residual", None)
+        if mid_block_additional_residual is not None:
+            uncond_mid_block, cond_mid_block = mid_block_additional_residual.chunk(2)
+
+        cross_attn_processor_context = SwapCrossAttnContext(
+            modified_text_embeddings=context.arguments.edited_conditioning,
+            index_map=context.cross_attention_index_map,
+            mask=context.cross_attention_mask,
+            cross_attention_types_to_do=[],
+        )
+
+        added_cond_kwargs = None
+        is_sdxl = type(conditioning_data.text_embeddings) is SDXLConditioningInfo
+        if is_sdxl:
+            added_cond_kwargs = {
+                "text_embeds": conditioning_data.unconditioned_embeddings.pooled_embeds,
+                "time_ids": conditioning_data.unconditioned_embeddings.add_time_ids,
+            }
+
+        # no cross attention for unconditioning (negative prompt)
+        unconditioned_next_x = self.model_forward_callback(
+            x,
+            sigma,
+            conditioning_data.unconditioned_embeddings.embeds,
+            {"swap_cross_attn_context": cross_attn_processor_context},
+            down_block_additional_residuals=uncond_down_block,
+            mid_block_additional_residual=uncond_mid_block,
+            down_intrablock_additional_residuals=uncond_down_intrablock,
+            added_cond_kwargs=added_cond_kwargs,
+            **kwargs,
+        )
+
+        if is_sdxl:
+            added_cond_kwargs = {
+                "text_embeds": conditioning_data.text_embeddings.pooled_embeds,
+                "time_ids": conditioning_data.text_embeddings.add_time_ids,
+            }
+
+        # do requested cross attention types for conditioning (positive prompt)
+        cross_attn_processor_context.cross_attention_types_to_do = cross_attention_control_types_to_do
+        conditioned_next_x = self.model_forward_callback(
+            x,
+            sigma,
+            conditioning_data.text_embeddings.embeds,
+            {"swap_cross_attn_context": cross_attn_processor_context},
+            down_block_additional_residuals=cond_down_block,
+            mid_block_additional_residual=cond_mid_block,
+            down_intrablock_additional_residuals=cond_down_intrablock,
+            added_cond_kwargs=added_cond_kwargs,
+            **kwargs,
         )
         return unconditioned_next_x, conditioned_next_x
 
@@ -517,3 +572,115 @@ class InvokeAIDiffuserComponent:
         scaled_delta = (conditioned_next_x - unconditioned_next_x) * guidance_scale
         combined_next_x = unconditioned_next_x + scaled_delta
         return combined_next_x
+
+    def apply_symmetry(
+        self,
+        postprocessing_settings: PostprocessingSettings,
+        latents: torch.Tensor,
+        percent_through: float,
+    ) -> torch.Tensor:
+        # Reset our last percent through if this is our first step.
+        if percent_through == 0.0:
+            self.last_percent_through = 0.0
+
+        if postprocessing_settings is None:
+            return latents
+
+        # Check for out of bounds
+        h_symmetry_time_pct = postprocessing_settings.h_symmetry_time_pct
+        if h_symmetry_time_pct is not None and (h_symmetry_time_pct <= 0.0 or h_symmetry_time_pct > 1.0):
+            h_symmetry_time_pct = None
+
+        v_symmetry_time_pct = postprocessing_settings.v_symmetry_time_pct
+        if v_symmetry_time_pct is not None and (v_symmetry_time_pct <= 0.0 or v_symmetry_time_pct > 1.0):
+            v_symmetry_time_pct = None
+
+        dev = latents.device.type
+
+        latents.to(device="cpu")
+
+        if (
+            h_symmetry_time_pct is not None
+            and self.last_percent_through < h_symmetry_time_pct
+            and percent_through >= h_symmetry_time_pct
+        ):
+            # Horizontal symmetry occurs on the 3rd dimension of the latent
+            width = latents.shape[3]
+            x_flipped = torch.flip(latents, dims=[3])
+            latents = torch.cat(
+                [
+                    latents[:, :, :, 0 : int(width / 2)],
+                    x_flipped[:, :, :, int(width / 2) : int(width)],
+                ],
+                dim=3,
+            )
+
+        if (
+            v_symmetry_time_pct is not None
+            and self.last_percent_through < v_symmetry_time_pct
+            and percent_through >= v_symmetry_time_pct
+        ):
+            # Vertical symmetry occurs on the 2nd dimension of the latent
+            height = latents.shape[2]
+            y_flipped = torch.flip(latents, dims=[2])
+            latents = torch.cat(
+                [
+                    latents[:, :, 0 : int(height / 2)],
+                    y_flipped[:, :, int(height / 2) : int(height)],
+                ],
+                dim=2,
+            )
+
+        self.last_percent_through = percent_through
+        return latents.to(device=dev)
+
+    # todo: make this work
+    @classmethod
+    def apply_conjunction(cls, x, t, forward_func, uc, c_or_weighted_c_list, global_guidance_scale):
+        x_in = torch.cat([x] * 2)
+        t_in = torch.cat([t] * 2)  # aka sigmas
+
+        deltas = None
+        uncond_latents = None
+        weighted_cond_list = (
+            c_or_weighted_c_list if isinstance(c_or_weighted_c_list, list) else [(c_or_weighted_c_list, 1)]
+        )
+
+        # below is fugly omg
+        conditionings = [uc] + [c for c, weight in weighted_cond_list]
+        weights = [1] + [weight for c, weight in weighted_cond_list]
+        chunk_count = math.ceil(len(conditionings) / 2)
+        deltas = None
+        for chunk_index in range(chunk_count):
+            offset = chunk_index * 2
+            chunk_size = min(2, len(conditionings) - offset)
+
+            if chunk_size == 1:
+                c_in = conditionings[offset]
+                latents_a = forward_func(x_in[:-1], t_in[:-1], c_in)
+                latents_b = None
+            else:
+                c_in = torch.cat(conditionings[offset : offset + 2])
+                latents_a, latents_b = forward_func(x_in, t_in, c_in).chunk(2)
+
+            # first chunk is guaranteed to be 2 entries: uncond_latents + first conditioining
+            if chunk_index == 0:
+                uncond_latents = latents_a
+                deltas = latents_b - uncond_latents
+            else:
+                deltas = torch.cat((deltas, latents_a - uncond_latents))
+                if latents_b is not None:
+                    deltas = torch.cat((deltas, latents_b - uncond_latents))
+
+        # merge the weighted deltas together into a single merged delta
+        per_delta_weights = torch.tensor(weights[1:], dtype=deltas.dtype, device=deltas.device)
+        normalize = False
+        if normalize:
+            per_delta_weights /= torch.sum(per_delta_weights)
+        reshaped_weights = per_delta_weights.reshape(per_delta_weights.shape + (1, 1, 1))
+        deltas_merged = torch.sum(deltas * reshaped_weights, dim=0, keepdim=True)
+
+        # old_return_value = super().forward(x, sigma, uncond, cond, cond_scale)
+        # assert(0 == len(torch.nonzero(old_return_value - (uncond_latents + deltas_merged * cond_scale))))
+
+        return uncond_latents + deltas_merged * global_guidance_scale

invokeai/backend/training/__init__.py

@@ -1,5 +1,4 @@
 """
 Initialization file for invokeai.backend.training
 """
-
 from .textual_inversion_training import do_textual_inversion_training, parse_args  # noqa: F401

invokeai/backend/training/textual_inversion_training.py

@@ -858,9 +858,9 @@ def do_textual_inversion_training(
                 # Let's make sure we don't update any embedding weights besides the newly added token
                 index_no_updates = torch.arange(len(tokenizer)) != placeholder_token_id
                 with torch.no_grad():
-                    accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[index_no_updates] = (
-                        orig_embeds_params[index_no_updates]
-                    )
+                    accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[
+                        index_no_updates
+                    ] = orig_embeds_params[index_no_updates]
 
             # Checks if the accelerator has performed an optimization step behind the scenes
             if accelerator.sync_gradients:

invokeai/backend/util/__init__.py

@@ -1,7 +1,6 @@
 """
 Initialization file for invokeai.backend.util
 """
-
 from .attention import auto_detect_slice_size  # noqa: F401
 from .devices import (  # noqa: F401
     CPU_DEVICE,

invokeai/backend/util/attention.py

@@ -3,7 +3,6 @@
 Utility routine used for autodetection of optimal slice size
 for attention mechanism.
 """
-
 import psutil
 import torch
 

invokeai/backend/util/silence_warnings.py

@@ -1,5 +1,4 @@
 """Context class to silence transformers and diffusers warnings."""
-
 import warnings
 from typing import Any
 

invokeai/backend/util/util.py

@@ -340,17 +340,14 @@ def download_with_resume(url: str, dest: Path, access_token: str = None) -> Path
             logger.error(f"ERROR DOWNLOADING {url}: {resp.text}")
             return None
 
-        with (
-            open(dest, open_mode) as file,
-            tqdm(
-                desc=str(dest),
-                initial=exist_size,
-                total=content_length,
-                unit="iB",
-                unit_scale=True,
-                unit_divisor=1000,
-            ) as bar,
-        ):
+        with open(dest, open_mode) as file, tqdm(
+            desc=str(dest),
+            initial=exist_size,
+            total=content_length,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=1000,
+        ) as bar:
             for data in resp.iter_content(chunk_size=1024):
                 size = file.write(data)
                 bar.update(size)

invokeai/frontend/CLI/__init__.py

@@ -1,5 +1,4 @@
 """
 Initialization file for invokeai.frontend.CLI
 """
-
 from .CLI import main as invokeai_command_line_interface  # noqa: F401

invokeai/frontend/install/invokeai_configure.py

@@ -1,7 +1,6 @@
 """
 Wrapper for invokeai.backend.configure.invokeai_configure
 """
-
 from ...backend.install.invokeai_configure import main as invokeai_configure  # noqa: F401
 
 __all__ = ["invokeai_configure"]

invokeai/frontend/install/invokeai_update.py

@@ -2,7 +2,6 @@
 Minimalist updater script. Prompts user for the tag or branch to update to and runs
 pip install <path_to_git_source>.
 """
-
 import os
 import platform
 from distutils.version import LooseVersion

invokeai/frontend/install/widgets.py

@@ -1,7 +1,6 @@
 """
 Widget class definitions used by model_select.py, merge_diffusers.py and textual_inversion.py
 """
-
 import curses
 import math
 import os

invokeai/frontend/merge/__init__.py

@@ -1,5 +1,4 @@
 """
 Initialization file for invokeai.frontend.merge
 """
-
 from .merge_diffusers import main as invokeai_merge_diffusers  # noqa: F401

invokeai/frontend/merge/merge_diffusers.py

@@ -4,7 +4,6 @@ used to merge 2-3 models together and create a new InvokeAI-registered diffusion
 
 Copyright (c) 2023 Lincoln Stein and the InvokeAI Development Team
 """
-
 import argparse
 import curses
 import re

invokeai/frontend/training/__init__.py

@@ -1,5 +1,4 @@
 """
 Initialization file for invokeai.frontend.training
 """
-
 from .textual_inversion import main as invokeai_textual_inversion  # noqa: F401

invokeai/frontend/training/textual_inversion.py

@@ -6,6 +6,7 @@ This is the frontend to "textual_inversion_training.py".
 Copyright (c) 2023-24 Lincoln Stein and the InvokeAI Development Team
 """
 
+
 import os
 import re
 import shutil

invokeai/frontend/web/public/locales/de.json

@@ -81,7 +81,7 @@
         "outputs": "Ausgabe",
         "data": "Daten",
         "safetensors": "Safe-Tensors",
-        "outpaint": "Outpaint (Außen ausmalen)",
+        "outpaint": "Ausmalen",
         "details": "Details",
         "format": "Format",
         "unknown": "Unbekannt",
@@ -110,18 +110,17 @@
         "nextPage": "Nächste Seite",
         "unknownError": "Unbekannter Fehler",
         "unsaved": "Nicht gespeichert",
-        "aboutDesc": "Verwenden Sie Invoke für die Arbeit? Siehe hier:",
+        "aboutDesc": "Verwenden Sie Invoke für die Arbeit? Dann siehe hier:",
         "localSystem": "Lokales System",
         "orderBy": "Ordnen nach",
-        "saveAs": "Speichern als",
+        "saveAs": "Speicher als",
         "updated": "Aktualisiert",
         "copy": "Kopieren",
-        "aboutHeading": "Nutzen Sie Ihre kreative Energie",
-        "toResolve": "Lösen"
+        "aboutHeading": "Nutzen Sie Ihre kreative Energie"
     },
     "gallery": {
         "generations": "Erzeugungen",
-        "showGenerations": "Zeige Ergebnisse",
+        "showGenerations": "Zeige Erzeugnisse",
         "uploads": "Uploads",
         "showUploads": "Zeige Uploads",
         "galleryImageSize": "Bildgröße",
@@ -134,6 +133,8 @@
         "loadMore": "Mehr laden",
         "noImagesInGallery": "Keine Bilder in der Galerie",
         "loading": "Lade",
+        "preparingDownload": "bereite Download vor",
+        "preparingDownloadFailed": "Problem beim Download vorbereiten",
         "deleteImage": "Lösche Bild",
         "copy": "Kopieren",
         "download": "Runterladen",
@@ -149,9 +150,9 @@
         "problemDeletingImagesDesc": "Ein oder mehrere Bilder konnten nicht gelöscht werden",
         "starImage": "Bild markieren",
         "assets": "Ressourcen",
-        "unstarImage": "Markierung entfernen",
+        "unstarImage": "Markierung Entfernen",
         "image": "Bild",
-        "deleteSelection": "Lösche Auswahl",
+        "deleteSelection": "Lösche markierte",
         "dropToUpload": "$t(gallery.drop) zum hochladen",
         "dropOrUpload": "$t(gallery.drop) oder hochladen",
         "drop": "Ablegen",
@@ -589,21 +590,10 @@
         "general": "Allgemein",
         "hiresStrength": "High Res Stärke",
         "hidePreview": "Verstecke Vorschau",
-        "showPreview": "Zeige Vorschau",
-        "aspect": "Seitenverhältnis",
-        "aspectRatio": "Seitenverhältnis",
-        "scheduler": "Planer",
-        "aspectRatioFree": "Frei",
-        "setToOptimalSizeTooLarge": "$t(parameters.setToOptimalSize) (kann zu groß sein)",
-        "lockAspectRatio": "Seitenverhältnis sperren",
-        "swapDimensions": "Seitenverhältnis umkehren",
-        "setToOptimalSize": "Optimiere Größe für Modell",
-        "useSize": "Maße übernehmen",
-        "remixImage": "Remix des Bilds erstellen",
-        "imageActions": "Weitere Bildaktionen"
+        "showPreview": "Zeige Vorschau"
     },
     "settings": {
-        "displayInProgress": "Zwischenbilder anzeigen",
+        "displayInProgress": "Bilder in Bearbeitung anzeigen",
         "saveSteps": "Speichern der Bilder alle n Schritte",
         "confirmOnDelete": "Bestätigen beim Löschen",
         "displayHelpIcons": "Hilfesymbole anzeigen",
@@ -616,34 +606,7 @@
         "useSlidersForAll": "Schieberegler für alle Optionen verwenden",
         "showAdvancedOptions": "Erweiterte Optionen anzeigen",
         "alternateCanvasLayout": "Alternatives Leinwand-Layout",
-        "clearIntermediatesDesc1": "Das Löschen der Zwischenbilder setzt Leinwand und ControlNet zurück.",
-        "favoriteSchedulers": "Lieblings-Planer",
-        "favoriteSchedulersPlaceholder": "Keine Planer favorisiert",
-        "generation": "Erzeugung",
-        "enableInformationalPopovers": "Info-Popouts anzeigen",
-        "shouldLogToConsole": "Konsole loggen",
-        "showProgressInViewer": "Zwischenbilder im Viewer anzeigen",
-        "clearIntermediatesDesc3": "Ihre Bilder werden nicht gelöscht.",
-        "clearIntermediatesWithCount_one": "Lösche {{count}} Zwischenbilder",
-        "clearIntermediatesWithCount_other": "Lösche {{count}} Zwischenbilder",
-        "reloadingIn": "Neuladen in",
-        "enableNodesEditor": "Nodes Editor aktivieren",
-        "autoChangeDimensions": "Breite/Höhe auf Modellstandard setzen",
-        "experimental": "Experimentell",
-        "intermediatesCleared_one": "{{count}} Zwischenbilder gelöscht",
-        "intermediatesCleared_other": "{{count}} Zwischenbilder gelöscht",
-        "enableInvisibleWatermark": "Unsichtbares Wasserzeichen aktivieren",
-        "general": "Allgemein",
-        "consoleLogLevel": "Protokollierungsstufe",
-        "clearIntermediatesDisabled": "Warteschlange muss leer sein, um Zwischenbilder zu löschen",
-        "developer": "Entwickler",
-        "antialiasProgressImages": "Zwischenbilder mit Anti-Alias",
-        "beta": "Beta",
-        "ui": "Benutzeroberfläche",
-        "clearIntermediatesDesc2": "Zwischenbilder sind Nebenprodukte der Erstellung. Sie zu löschen macht Festplattenspeicher frei.",
-        "clearIntermediates": "Zwischenbilder löschen",
-        "intermediatesClearedFailed": "Problem beim Löschen der Zwischenbilder",
-        "enableNSFWChecker": "Auf unangemessene Inhalte prüfen"
+        "clearIntermediatesDesc1": "Das Löschen der Zwischenprodukte setzt Leinwand und ControlNet zurück."
     },
     "toast": {
         "tempFoldersEmptied": "Temp-Ordner geleert",
@@ -688,9 +651,7 @@
         "problemCopyingCanvas": "Problem beim Kopieren der Leinwand",
         "problemCopyingCanvasDesc": "Kann Basis-Layer nicht exportieren",
         "problemDownloadingCanvas": "Problem beim Herunterladen der Leinwand",
-        "setAsCanvasInitialImage": "Als Ausgangsbild gesetzt",
-        "addedToBoard": "Dem Board hinzugefügt",
-        "loadedWithWarnings": "Workflow mit Warnungen geladen"
+        "setAsCanvasInitialImage": "Als Ausgangsbild gesetzt"
     },
     "tooltip": {
         "feature": {
@@ -772,23 +733,23 @@
     "accessibility": {
         "modelSelect": "Modell-Auswahl",
         "uploadImage": "Bild hochladen",
-        "previousImage": "Vorheriges Bild",
+        "previousImage": "Voriges Bild",
         "useThisParameter": "Benutze diesen Parameter",
-        "copyMetadataJson": "Kopiere JSON-Metadaten",
+        "copyMetadataJson": "Kopiere Metadaten JSON",
         "zoomIn": "Vergrößern",
         "rotateClockwise": "Im Uhrzeigersinn drehen",
         "flipHorizontally": "Horizontal drehen",
         "flipVertically": "Vertikal drehen",
         "modifyConfig": "Optionen einstellen",
         "toggleAutoscroll": "Auroscroll ein/ausschalten",
-        "toggleLogViewer": "Log-Betrachter ein/ausschalten",
+        "toggleLogViewer": "Log Betrachter ein/ausschalten",
         "showOptionsPanel": "Seitenpanel anzeigen",
         "reset": "Zurücksetzten",
         "nextImage": "Nächstes Bild",
         "zoomOut": "Verkleinern",
         "rotateCounterClockwise": "Gegen den Uhrzeigersinn drehen",
-        "showGalleryPanel": "Galerie-Panel anzeigen",
-        "exitViewer": "Betrachter beenden",
+        "showGalleryPanel": "Galeriefenster anzeigen",
+        "exitViewer": "Betrachten beenden",
         "menu": "Menü",
         "loadMore": "Mehr laden",
         "invokeProgressBar": "Invoke Fortschrittsanzeige",
@@ -798,7 +759,7 @@
         "about": "Über"
     },
     "boards": {
-        "autoAddBoard": "Automatisches Hinzufügen zum Board",
+        "autoAddBoard": "Automatisches Hinzufügen zum Ordner",
         "topMessage": "Dieser Ordner enthält Bilder die in den folgenden Funktionen verwendet werden:",
         "move": "Bewegen",
         "menuItemAutoAdd": "Auto-Hinzufügen zu diesem Ordner",
@@ -807,13 +768,13 @@
         "noMatching": "Keine passenden Ordner",
         "selectBoard": "Ordner aussuchen",
         "cancel": "Abbrechen",
-        "addBoard": "Board hinzufügen",
+        "addBoard": "Ordner hinzufügen",
         "uncategorized": "Ohne Kategorie",
         "downloadBoard": "Ordner runterladen",
         "changeBoard": "Ordner wechseln",
         "loading": "Laden...",
         "clearSearch": "Suche leeren",
-        "bottomMessage": "Löschen des Boards und seiner Bilder setzt alle Funktionen zurück, die sie gerade verwenden.",
+        "bottomMessage": "Durch das Löschen dieses Ordners und seiner Bilder werden alle Funktionen zurückgesetzt, die sie derzeit verwenden.",
         "deleteBoardOnly": "Nur Ordner löschen",
         "deleteBoard": "Löschen Ordner",
         "deleteBoardAndImages": "Löschen Ordner und Bilder",
@@ -859,7 +820,7 @@
         "colorMap": "Farbe",
         "lowThreshold": "Niedrige Schwelle",
         "highThreshold": "Hohe Schwelle",
-        "toggleControlNet": "Dieses ControlNet ein- oder ausschalten",
+        "toggleControlNet": "Schalten ControlNet um",
         "delete": "Löschen",
         "controlAdapter_one": "Control Adapter",
         "controlAdapter_other": "Control Adapter",
@@ -904,23 +865,18 @@
         "maxFaces": "Maximale Anzahl Gesichter",
         "resizeSimple": "Größe ändern (einfach)",
         "large": "Groß",
-        "modelSize": "Modellgröße",
+        "modelSize": "Modell Größe",
         "small": "Klein",
         "base": "Basis",
-        "depthAnything": "Depth Anything",
-        "depthAnythingDescription": "Erstellung einer Tiefenkarte mit der Depth-Anything-Technik",
-        "face": "Gesicht",
-        "body": "Körper",
-        "hands": "Hände",
-        "dwOpenpose": "DW Openpose",
-        "dwOpenposeDescription": "Posenschätzung mit DW Openpose"
+        "depthAnything": "Depth Anything / \"Tiefe irgendwas\"",
+        "depthAnythingDescription": "Erstellung einer Tiefenkarte mit der Depth Anything-Technik"
     },
     "queue": {
         "status": "Status",
         "cancelTooltip": "Aktuellen Aufgabe abbrechen",
         "queueEmpty": "Warteschlange leer",
         "in_progress": "In Arbeit",
-        "queueFront": "Am Anfang der Warteschlange einreihen",
+        "queueFront": "An den Anfang der Warteschlange tun",
         "completed": "Fertig",
         "queueBack": "In die Warteschlange",
         "clearFailed": "Probleme beim leeren der Warteschlange",
@@ -948,7 +904,7 @@
         "batchValues": "Stapel Werte",
         "queueCountPrediction": "{{promptsCount}} Prompts × {{iterations}} Iterationen -> {{count}} Generationen",
         "queuedCount": "{{pending}} wartenden Elemente",
-        "clearQueueAlertDialog": "\"Die Warteschlange leeren\" stoppt den aktuellen Prozess und leert die Warteschlange komplett.",
+        "clearQueueAlertDialog": "Die Warteschlange leeren, stoppt den aktuellen Prozess und leert die Warteschlange komplett.",
         "completedIn": "Fertig in",
         "cancelBatchSucceeded": "Stapel abgebrochen",
         "cancelBatch": "Stapel stoppen",
@@ -957,20 +913,20 @@
         "cancelBatchFailed": "Problem beim Abbruch vom Stapel",
         "clearQueueAlertDialog2": "Warteschlange wirklich leeren?",
         "pruneSucceeded": "{{item_count}} abgeschlossene Elemente aus der Warteschlange entfernt",
-        "pauseSucceeded": "Prozess angehalten",
+        "pauseSucceeded": "Prozessor angehalten",
         "cancelFailed": "Problem beim Stornieren des Auftrags",
-        "pauseFailed": "Problem beim Anhalten des Prozesses",
+        "pauseFailed": "Problem beim Anhalten des Prozessors",
         "front": "Vorne",
         "pruneTooltip": "Bereinigen Sie {{item_count}} abgeschlossene Aufträge",
-        "resumeFailed": "Problem beim Fortsetzen des Prozesses",
+        "resumeFailed": "Problem beim wieder aufnehmen von Prozessor",
         "pruneFailed": "Problem beim leeren der Warteschlange",
-        "pauseTooltip": "Prozess anhalten",
-        "back": "Ende",
-        "resumeSucceeded": "Prozess wird fortgesetzt",
-        "resumeTooltip": "Prozess wieder aufnehmen",
+        "pauseTooltip": "Pause von Prozessor",
+        "back": "Hinten",
+        "resumeSucceeded": "Prozessor wieder aufgenommen",
+        "resumeTooltip": "Prozessor wieder aufnehmen",
         "time": "Zeit",
-        "batchQueuedDesc_one": "{{count}} Eintrag an {{direction}} der Wartschlange hinzugefügt",
-        "batchQueuedDesc_other": "{{count}} Einträge an {{direction}} der Wartschlange hinzugefügt",
+        "batchQueuedDesc_one": "{{count}} Eintrag ans {{direction}} der Wartschlange hinzugefügt",
+        "batchQueuedDesc_other": "{{count}} Einträge ans {{direction}} der Wartschlange hinzugefügt",
         "openQueue": "Warteschlange öffnen",
         "batchFailedToQueue": "Fehler beim Einreihen in die Stapelverarbeitung",
         "batchFieldValues": "Stapelverarbeitungswerte",
@@ -1005,12 +961,11 @@
         "workflow": "Workflow",
         "scheduler": "Planer",
         "noRecallParameters": "Es wurden keine Parameter zum Abrufen gefunden",
-        "recallParameters": "Parameter wiederherstellen",
-        "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)"
+        "recallParameters": "Parameter wiederherstellen"
     },
     "popovers": {
         "noiseUseCPU": {
-            "heading": "Nutze CPU-Rauschen",
+            "heading": "Nutze Prozessor rauschen",
             "paragraphs": [
                 "Entscheidet, ob auf der CPU oder GPU Rauschen erzeugt wird.",
                 "Mit aktiviertem CPU-Rauschen wird ein bestimmter Seedwert das gleiche Bild auf jeder Maschine erzeugen.",
@@ -1020,7 +975,8 @@
         "paramModel": {
             "heading": "Modell",
             "paragraphs": [
-                "Modell für die Entrauschungsschritte."
+                "Modell für die Entrauschungsschritte.",
+                "Verschiedene Modelle werden in der Regel so trainiert, dass sie sich auf die Erzeugung bestimmter Ästhetik und/oder Inhalte spezialisiert."
             ]
         },
         "paramIterations": {
@@ -1128,23 +1084,12 @@
                 "Wie stark wird das ControlNet das generierte Bild beeinflussen wird."
             ],
             "heading": "Einfluss"
-        },
-        "paramScheduler": {
-            "paragraphs": [
-                "\"Planer\" definiert, wie iterativ Rauschen zu einem Bild hinzugefügt wird, oder wie ein Sample bei der Ausgabe eines Modells aktualisiert wird."
-            ],
-            "heading": "Planer"
-        },
-        "imageFit": {
-            "paragraphs": [
-                "Reduziert das Ausgangsbild auf die Breite und Höhe des Ausgangsbildes. Empfohlen zu aktivieren."
-            ]
         }
     },
     "ui": {
         "lockRatio": "Verhältnis sperren",
-        "hideProgressImages": "Fortschrittsbilder verbergen",
-        "showProgressImages": "Fortschrittsbilder anzeigen",
+        "hideProgressImages": "Verstecke Prozess Bild",
+        "showProgressImages": "Zeige Prozess Bild",
         "swapSizes": "Tausche Größen"
     },
     "invocationCache": {
@@ -1342,19 +1287,7 @@
         "vaeFieldDescription": "VAE Submodell.",
         "unknownInput": "Unbekannte Eingabe: {{name}}",
         "unknownNodeType": "Unbekannter Knotentyp",
-        "float": "Kommazahlen",
-        "latentsPolymorphic": "Latents Polymorph",
-        "integerPolymorphicDescription": "Eine Sammlung von ganzen Zahlen.",
-        "integerPolymorphic": "Ganze Zahl Polymorph",
-        "ipAdapterPolymorphic": "IP-Adapter Polymorph",
-        "floatPolymorphic": "Fließkommazahl Polymorph",
-        "enumDescription": "Aufzählungen sind Werte, die eine von mehreren Optionen sein können.",
-        "floatCollection": "Fließkommazahl Sammlung",
-        "enum": "Aufzählung",
-        "floatPolymorphicDescription": "Eine Sammlung von Fließkommazahlen",
-        "fullyContainNodes": "Vollständig ausgewählte Nodes auswählen",
-        "editMode": "Im Workflow-Editor bearbeiten",
-        "floatCollectionDescription": "Eine Sammlung von Fließkommazahlen"
+        "float": "Kommazahlen"
     },
     "hrf": {
         "enableHrf": "Korrektur für hohe Auflösungen",
@@ -1403,12 +1336,12 @@
         },
         "control": {
             "title": "Kontrolle",
-            "controlAdaptersTab": "Kontroll-Adapter",
-            "ipTab": "Bild-Prompts"
+            "controlAdaptersTab": "Kontroll Adapter",
+            "ipTab": "Bild Beschreibung"
         },
         "compositing": {
             "coherenceTab": "Kohärenzpass",
-            "infillTab": "Füllung / Infill",
+            "infillTab": "Füllung",
             "title": "Compositing"
         }
     },
@@ -1446,15 +1379,5 @@
     },
     "app": {
         "storeNotInitialized": "App-Store ist nicht initialisiert"
-    },
-    "sdxl": {
-        "concatPromptStyle": "Verknüpfen von Prompt & Stil",
-        "scheduler": "Planer",
-        "steps": "Schritte",
-        "useRefiner": "Refiner verwenden",
-        "selectAModel": "Modell auswählen"
-    },
-    "dynamicPrompts": {
-        "showDynamicPrompts": "Dynamische Prompts anzeigen"
     }
 }

invokeai/frontend/web/public/locales/en.json

@@ -304,6 +304,12 @@
             "method": "High Resolution Fix Method"
         }
     },
+    "prompt": {
+        "addPromptTrigger": "Add Prompt Trigger",
+        "compatibleEmbeddings": "Compatible Embeddings",
+        "noPromptTriggers": "No triggers available",
+        "noMatchingTriggers": "No matching triggers"
+    },
     "embedding": {
         "addEmbedding": "Add Embedding",
         "incompatibleModel": "Incompatible base model:",
@@ -867,7 +873,6 @@
     "models": {
         "addLora": "Add LoRA",
         "allLoRAsAdded": "All LoRAs added",
-        "concepts": "Concepts",
         "loraAlreadyAdded": "LoRA already added",
         "esrganModel": "ESRGAN Model",
         "loading": "loading",
@@ -1464,8 +1469,9 @@
         "clipSkip": {
             "heading": "CLIP Skip",
             "paragraphs": [
-                "How many layers of the CLIP model to skip.",
-                "Certain models are better suited to be used with CLIP Skip."
+                "Choose how many layers of the CLIP model to skip.",
+                "Some models work better with certain CLIP Skip settings.",
+                "A higher value typically results in a less detailed image."
             ]
         },
         "paramNegativeConditioning": {
@@ -1485,8 +1491,7 @@
         "paramScheduler": {
             "heading": "Scheduler",
             "paragraphs": [
-                "Scheduler used during the generation process.",
-                "Each scheduler defines how to iteratively add noise to an image or how to update a sample based on a model's output."
+                "Scheduler defines how to iteratively add noise to an image or how to update a sample based on a model's output."
             ]
         },
         "compositingMaskBlur": {
@@ -1503,7 +1508,7 @@
         },
         "compositingCoherenceMode": {
             "heading": "Mode",
-            "paragraphs": ["Method used to create a coherent image with the newly generated masked area."]
+            "paragraphs": ["The mode of the Coherence Pass."]
         },
         "compositingCoherenceEdgeSize": {
             "heading": "Edge Size",
@@ -1520,38 +1525,30 @@
             "heading": "Mask Adjustments",
             "paragraphs": ["Adjust the mask."]
         },
+        "controlNetBeginEnd": {
+            "heading": "Begin / End Step Percentage",
+            "paragraphs": [
+                "Which steps of the denoising process will have the ControlNet applied.",
+                "ControlNets applied at the beginning of the process guide composition, and ControlNets applied at the end guide details."
+            ]
+        },
+        "controlNetControlMode": {
+            "heading": "Control Mode",
+            "paragraphs": ["Lends more weight to either the prompt or ControlNet."]
+        },
+        "controlNetResizeMode": {
+            "heading": "Resize Mode",
+            "paragraphs": ["How the ControlNet image will be fit to the image output size."]
+        },
         "controlNet": {
             "heading": "ControlNet",
             "paragraphs": [
                 "ControlNets provide guidance to the generation process, helping create images with controlled composition, structure, or style, depending on the model selected."
             ]
         },
-        "controlNetBeginEnd": {
-            "heading": "Begin / End Step Percentage",
-            "paragraphs": [
-                "The part of the of the denoising process that will have the Control Adapter applied.",
-                "Generally, Control Adapters applied at the start of the process guide composition, and Control Adapters applied at the end guide details."
-            ]
-        },
-        "controlNetControlMode": {
-            "heading": "Control Mode",
-            "paragraphs": ["Lend more weight to either the prompt or ControlNet."]
-        },
-        "controlNetProcessor": {
-            "heading": "Processor",
-            "paragraphs": [
-                "Method of processing the input image to guide the generation process. Different processors will providedifferent effects or styles in your generated images."
-            ]
-        },
-        "controlNetResizeMode": {
-            "heading": "Resize Mode",
-            "paragraphs": ["Method to fit Control Adapter's input image size to the output generation size."]
-        },
         "controlNetWeight": {
             "heading": "Weight",
-            "paragraphs": [
-                "Weight of the Control Adapter. Higher weight will lead to larger impacts on the final image."
-            ]
+            "paragraphs": ["How strongly the ControlNet will impact the generated image."]
         },
         "dynamicPrompts": {
             "heading": "Dynamic Prompts",
@@ -1574,23 +1571,13 @@
                 "Per Image will use a unique seed for each image. This provides more variation."
             ]
         },
-        "imageFit": {
-            "heading": "Fit Initial Image to Output Size",
-            "paragraphs": [
-                "Resizes the initial image to the width and height of the output image. Recommended to enable."
-            ]
-        },
         "infillMethod": {
             "heading": "Infill Method",
-            "paragraphs": ["Method of infilling during the Outpainting or Inpainting process."]
+            "paragraphs": ["Method to infill the selected area."]
         },
         "lora": {
-            "heading": "LoRA",
-            "paragraphs": ["Lightweight models that are used in conjunction with base models."]
-        },
-        "loraWeight": {
-            "heading": "Weight",
-            "paragraphs": ["Weight of the LoRA. Higher weight will lead to larger impacts on the final image."]
+            "heading": "LoRA Weight",
+            "paragraphs": ["Higher LoRA weight will lead to larger impacts on the final image."]
         },
         "noiseUseCPU": {
             "heading": "Use CPU Noise",
@@ -1600,25 +1587,14 @@
                 "There is no performance impact to enabling CPU Noise."
             ]
         },
-        "paramAspect": {
-            "heading": "Aspect",
-            "paragraphs": [
-                "Aspect ratio of the generated image. Changing the ratio will update the Width and Height accordingly.",
-                "“Optimize” will set the Width and Height to optimal dimensions for the chosen model."
-            ]
-        },
         "paramCFGScale": {
             "heading": "CFG Scale",
-            "paragraphs": [
-                "Controls how much the prompt influences the generation process.",
-                "High CFG Scale values can result in over-saturation and distorted generation results. "
-            ]
+            "paragraphs": ["Controls how much your prompt influences the generation process."]
         },
         "paramCFGRescaleMultiplier": {
             "heading": "CFG Rescale Multiplier",
             "paragraphs": [
-                "Rescale multiplier for CFG guidance, used for models trained using zero-terminal SNR (ztsnr).",
-                "Suggested value of 0.7 for these models."
+                "Rescale multiplier for CFG guidance, used for models trained using zero-terminal SNR (ztsnr). Suggested value 0.7."
             ]
         },
         "paramDenoisingStrength": {
@@ -1628,16 +1604,6 @@
                 "0 will result in an identical image, while 1 will result in a completely new image."
             ]
         },
-        "paramHeight": {
-            "heading": "Height",
-            "paragraphs": ["Height of the generated image. Must be a multiple of 8."]
-        },
-        "paramHrf": {
-            "heading": "Enable High Resolution Fix",
-            "paragraphs": [
-                "Generate high quality images at a larger resolution than optimal for the model. Generally used to prevent duplication in the generated image."
-            ]
-        },
         "paramIterations": {
             "heading": "Iterations",
             "paragraphs": [
@@ -1648,7 +1614,8 @@
         "paramModel": {
             "heading": "Model",
             "paragraphs": [
-                "Model used for generation. Different models are trained to specialize in producing different aesthetic results and content."
+                "Model used for the denoising steps.",
+                "Different models are typically trained to specialize in producing particular aesthetic results and content."
             ]
         },
         "paramRatio": {
@@ -1662,7 +1629,7 @@
             "heading": "Seed",
             "paragraphs": [
                 "Controls the starting noise used for generation.",
-                "Disable the “Random” option to produce identical results with the same generation settings."
+                "Disable “Random Seed” to produce identical results with the same generation settings."
             ]
         },
         "paramSteps": {
@@ -1672,10 +1639,6 @@
                 "Higher step counts will typically create better images but will require more generation time."
             ]
         },
-        "paramUpscaleMethod": {
-            "heading": "Upscale Method",
-            "paragraphs": ["Method used to upscale the image for High Resolution Fix."]
-        },
         "paramVAE": {
             "heading": "VAE",
             "paragraphs": ["Model used for translating AI output into the final image."]
@@ -1683,82 +1646,14 @@
         "paramVAEPrecision": {
             "heading": "VAE Precision",
             "paragraphs": [
-                "The precision used during VAE encoding and decoding.",
-                "Fp16/Half precision is more efficient, at the expense of minor image variations."
-            ]
-        },
-        "paramWidth": {
-            "heading": "Width",
-            "paragraphs": ["Width of the generated image. Must be a multiple of 8."]
-        },
-        "patchmatchDownScaleSize": {
-            "heading": "Downscale",
-            "paragraphs": [
-                "How much downscaling occurs before infilling.",
-                "Higher downscaling will improve performance and reduce quality."
-            ]
-        },
-        "refinerModel": {
-            "heading": "Refiner Model",
-            "paragraphs": [
-                "Model used during the refiner portion of the generation process.",
-                "Similar to the Generation Model."
-            ]
-        },
-        "refinerPositiveAestheticScore": {
-            "heading": "Positive Aesthetic Score",
-            "paragraphs": [
-                "Weight generations to be more similar to images with a high aesthetic score, based on the training data."
-            ]
-        },
-        "refinerNegativeAestheticScore": {
-            "heading": "Negative Aesthetic Score",
-            "paragraphs": [
-                "Weight generations to be more similar to images with a low aesthetic score, based on the training data."
-            ]
-        },
-        "refinerScheduler": {
-            "heading": "Scheduler",
-            "paragraphs": [
-                "Scheduler used during the refiner portion of the generation process.",
-                "Similar to the Generation Scheduler."
-            ]
-        },
-        "refinerStart": {
-            "heading": "Refiner Start",
-            "paragraphs": [
-                "Where in the generation process the refiner will start to be used.",
-                "0 means the refiner will be used for the entire generation process, 0.8 means the refiner will be used for the last 20% of the generation process."
-            ]
-        },
-        "refinerSteps": {
-            "heading": "Steps",
-            "paragraphs": [
-                "Number of steps that will be performed during the refiner portion of the generation process.",
-                "Similar to the Generation Steps."
-            ]
-        },
-        "refinerCfgScale": {
-            "heading": "CFG Scale",
-            "paragraphs": [
-                "Controls how much the prompt influences the generation process.",
-                "Similar to the Generation CFG Scale."
+                "The precision used during VAE encoding and decoding. FP16/half precision is more efficient, at the expense of minor image variations."
             ]
         },
         "scaleBeforeProcessing": {
             "heading": "Scale Before Processing",
             "paragraphs": [
-                "“Auto” scales the selected area to the size best suited for the model before the image generation process.",
-                "“Manual” allows you to choose the width and height the selected area will be scaled to before the image generation process."
+                "Scales the selected area to the size best suited for the model before the image generation process."
             ]
-        },
-        "seamlessTilingXAxis": {
-            "heading": "Seamless Tiling X Axis",
-            "paragraphs": ["Seamlessly tile an image along the horizontal axis."]
-        },
-        "seamlessTilingYAxis": {
-            "heading": "Seamless Tiling Y Axis",
-            "paragraphs": ["Seamlessly tile an image along the vertical axis."]
         }
     },
     "ui": {

invokeai/frontend/web/public/locales/es.json

@@ -505,6 +505,8 @@
         "seamLowThreshold": "Bajo",
         "coherencePassHeader": "Parámetros de la coherencia",
         "compositingSettingsHeader": "Ajustes de la composición",
+        "coherenceSteps": "Pasos",
+        "coherenceStrength": "Fuerza",
         "patchmatchDownScaleSize": "Reducir a escala",
         "coherenceMode": "Modo"
     },

invokeai/frontend/web/public/locales/it.json

@@ -47,7 +47,7 @@
         "statusModelConverted": "Modello Convertito",
         "statusConvertingModel": "Conversione Modello",
         "loading": "Caricamento in corso",
-        "loadingInvokeAI": "Caricamento di Invoke AI",
+        "loadingInvokeAI": "Caricamento Invoke AI",
         "postprocessing": "Post Elaborazione",
         "txt2img": "Testo a Immagine",
         "accept": "Accetta",
@@ -61,7 +61,7 @@
         "imagePrompt": "Prompt Immagine",
         "darkMode": "Modalità scura",
         "batch": "Gestione Lotto",
-        "modelManager": "Gestore Modelli",
+        "modelManager": "Gestore modello",
         "communityLabel": "Comunità",
         "nodeEditor": "Editor dei nodi",
         "statusProcessing": "Elaborazione in corso",
@@ -81,7 +81,7 @@
         "error": "Errore",
         "installed": "Installato",
         "template": "Schema",
-        "outputs": "Risultati",
+        "outputs": "Uscite",
         "data": "Dati",
         "somethingWentWrong": "Qualcosa è andato storto",
         "copyError": "$t(gallery.copy) Errore",
@@ -93,7 +93,7 @@
         "created": "Creato",
         "prevPage": "Pagina precedente",
         "delete": "Elimina",
-        "orderBy": "Ordina per",
+        "orderBy": "Ordinato per",
         "nextPage": "Pagina successiva",
         "saveAs": "Salva come",
         "unsaved": "Non salvato",
@@ -109,13 +109,7 @@
         "green": "Verde",
         "blue": "Blu",
         "alpha": "Alfa",
-        "copy": "Copia",
-        "on": "Attivato",
-        "checkpoint": "Checkpoint",
-        "safetensors": "Safetensors",
-        "ai": "ia",
-        "file": "File",
-        "toResolve": "Da risolvere"
+        "copy": "Copia"
     },
     "gallery": {
         "generations": "Generazioni",
@@ -143,6 +137,8 @@
         "copy": "Copia",
         "download": "Scarica",
         "setCurrentImage": "Imposta come immagine corrente",
+        "preparingDownload": "Preparazione del download",
+        "preparingDownloadFailed": "Problema durante la preparazione del download",
         "downloadSelection": "Scarica gli elementi selezionati",
         "noImageSelected": "Nessuna immagine selezionata",
         "deleteSelection": "Elimina la selezione",
@@ -608,6 +604,8 @@
         "seamLowThreshold": "Basso",
         "seamHighThreshold": "Alto",
         "coherencePassHeader": "Passaggio di coerenza",
+        "coherenceSteps": "Passi",
+        "coherenceStrength": "Forza",
         "compositingSettingsHeader": "Impostazioni di composizione",
         "patchmatchDownScaleSize": "Ridimensiona",
         "coherenceMode": "Modalità",
@@ -936,7 +934,7 @@
         "executionStateCompleted": "Completato",
         "boardFieldDescription": "Una bacheca della galleria",
         "addNodeToolTip": "Aggiungi nodo (Shift+A, Space)",
-        "sDXLRefinerModelField": "Modello Affinatore",
+        "sDXLRefinerModelField": "Modello Refiner",
         "problemReadingMetadata": "Problema durante la lettura dei metadati dall'immagine",
         "colorCodeEdgesHelp": "Bordi con codice colore in base ai campi collegati",
         "animatedEdges": "Bordi animati",
@@ -1140,11 +1138,7 @@
         "unsupportedAnyOfLength": "unione di troppi elementi ({{count}})",
         "clearWorkflowDesc": "Cancellare questo flusso di lavoro e avviarne uno nuovo?",
         "clearWorkflow": "Cancella il flusso di lavoro",
-        "clearWorkflowDesc2": "Il tuo flusso di lavoro attuale presenta modifiche non salvate.",
-        "viewMode": "Utilizzare nella vista lineare",
-        "reorderLinearView": "Riordina la vista lineare",
-        "editMode": "Modifica nell'editor del flusso di lavoro",
-        "resetToDefaultValue": "Ripristina il valore predefinito"
+        "clearWorkflowDesc2": "Il tuo flusso di lavoro attuale presenta modifiche non salvate."
     },
     "boards": {
         "autoAddBoard": "Aggiungi automaticamente bacheca",
@@ -1247,16 +1241,7 @@
         "large": "Grande",
         "small": "Piccolo",
         "depthAnythingDescription": "Generazione di mappe di profondità utilizzando la tecnica Depth Anything",
-        "modelSize": "Dimensioni del modello",
-        "dwOpenposeDescription": "Stima della posa umana utilizzando DW Openpose",
-        "face": "Viso",
-        "body": "Corpo",
-        "hands": "Mani",
-        "lineartAnime": "Linea Anime",
-        "base": "Base",
-        "lineart": "Linea",
-        "controlnet": "$t(controlnet.controlAdapter_one) #{{number}} ($t(common.controlNet))",
-        "mediapipeFace": "Mediapipe Volto"
+        "modelSize": "Dimensioni del modello"
     },
     "queue": {
         "queueFront": "Aggiungi all'inizio della coda",
@@ -1336,7 +1321,7 @@
         "noModelsAvailable": "Nessun modello disponibile",
         "selectModel": "Seleziona un modello",
         "selectLoRA": "Seleziona un LoRA",
-        "noRefinerModelsInstalled": "Nessun modello affinatore SDXL installato",
+        "noRefinerModelsInstalled": "Nessun modello SDXL Refiner installato",
         "noLoRAsInstalled": "Nessun LoRA installato",
         "esrganModel": "Modello ESRGAN",
         "addLora": "Aggiungi LoRA",
@@ -1386,8 +1371,7 @@
     "popovers": {
         "paramScheduler": {
             "paragraphs": [
-                "Il campionatore utilizzato durante il processo di generazione.",
-                "Ciascun campionatore definisce come aggiungere in modo iterativo il rumore a un'immagine o come aggiornare un campione in base all'output di un modello."
+                "Il campionatore definisce come aggiungere in modo iterativo il rumore a un'immagine o come aggiornare un campione in base all'output di un modello."
             ],
             "heading": "Campionatore"
         },
@@ -1397,16 +1381,30 @@
                 "Regola la maschera."
             ]
         },
+        "compositingCoherenceSteps": {
+            "heading": "Passi",
+            "paragraphs": [
+                "Numero di passi di riduzione del rumore utilizzati nel Passaggio di Coerenza.",
+                "Uguale al parametro principale Passi."
+            ]
+        },
+        "compositingBlur": {
+            "heading": "Sfocatura",
+            "paragraphs": [
+                "Il raggio di sfocatura della maschera."
+            ]
+        },
         "compositingCoherenceMode": {
             "heading": "Modalità",
             "paragraphs": [
-                "Metodo utilizzato per creare un'immagine coerente con l'area mascherata appena generata."
+                "La modalità del Passaggio di Coerenza."
             ]
         },
         "clipSkip": {
             "paragraphs": [
                 "Scegli quanti livelli del modello CLIP saltare.",
-                "Alcuni modelli funzionano meglio con determinate impostazioni di CLIP Skip."
+                "Alcuni modelli funzionano meglio con determinate impostazioni di CLIP Skip.",
+                "Un valore più alto in genere produce un'immagine meno dettagliata."
             ]
         },
         "compositingCoherencePass": {
@@ -1415,6 +1413,13 @@
                 "Un secondo ciclo di riduzione del rumore aiuta a comporre l'immagine Inpaint/Outpaint."
             ]
         },
+        "compositingStrength": {
+            "heading": "Forza",
+            "paragraphs": [
+                "Intensità di riduzione del rumore per il passaggio di coerenza.",
+                "Uguale al parametro intensità di riduzione del rumore da immagine a immagine."
+            ]
+        },
         "paramNegativeConditioning": {
             "paragraphs": [
                 "Il processo di generazione evita i concetti nel prompt negativo. Utilizzatelo per escludere qualità o oggetti dall'output.",
@@ -1438,8 +1443,8 @@
         "controlNetBeginEnd": {
             "heading": "Percentuale passi Inizio / Fine",
             "paragraphs": [
-                "La parte del processo di rimozione del rumore in cui verrà applicato l'adattatore di controllo.",
-                "In genere, gli adattatori di controllo applicati all'inizio del processo guidano la composizione, mentre quelli applicati alla fine guidano i dettagli."
+                "A quali passi del processo di rimozione del rumore verrà applicato ControlNet.",
+                "I ControlNet applicati all'inizio del processo guidano la composizione, mentre i ControlNet applicati alla fine guidano i dettagli."
             ]
         },
         "noiseUseCPU": {
@@ -1452,8 +1457,7 @@
         },
         "scaleBeforeProcessing": {
             "paragraphs": [
-                "\"Auto\" ridimensiona l'area selezionata alla dimensione più adatta al modello prima del processo di generazione dell'immagine.",
-                "\"Manuale\" consente di scegliere la larghezza e l'altezza a cui verrà ridimensionata l'area selezionata prima del processo di generazione dell'immagine."
+                "Ridimensiona l'area selezionata alla dimensione più adatta al modello prima del processo di generazione dell'immagine."
             ],
             "heading": "Scala prima dell'elaborazione"
         },
@@ -1488,21 +1492,20 @@
         "paramVAEPrecision": {
             "heading": "Precisione VAE",
             "paragraphs": [
-                "La precisione utilizzata durante la codifica e decodifica VAE.",
-                "Fp16/Mezza precisione è più efficiente, a scapito di minori variazioni dell'immagine."
+                "La precisione utilizzata durante la codifica e decodifica VAE. FP16/mezza precisione è più efficiente, a scapito di minori variazioni dell'immagine."
             ]
         },
         "paramSeed": {
             "paragraphs": [
                 "Controlla il rumore iniziale utilizzato per la generazione.",
-                "Disabilita l'opzione \"Casuale\" per produrre risultati identici con le stesse impostazioni di generazione."
+                "Disabilita seme \"Casuale\" per produrre risultati identici con le stesse impostazioni di generazione."
             ],
             "heading": "Seme"
         },
         "controlNetResizeMode": {
             "heading": "Modalità ridimensionamento",
             "paragraphs": [
-                "Metodo per adattare le dimensioni dell'immagine in ingresso dell'adattatore di controllo alle dimensioni della generazione di output."
+                "Come l'immagine ControlNet verrà adattata alle dimensioni di output dell'immagine."
             ]
         },
         "dynamicPromptsSeedBehaviour": {
@@ -1517,7 +1520,8 @@
         "paramModel": {
             "heading": "Modello",
             "paragraphs": [
-                "Modello utilizzato per la generazione. Diversi modelli vengono addestrati per specializzarsi nella produzione di risultati e contenuti estetici diversi."
+                "Modello utilizzato per i passaggi di riduzione del rumore.",
+                "Diversi modelli sono generalmente addestrati per specializzarsi nella produzione di particolari risultati e contenuti estetici."
             ]
         },
         "paramDenoisingStrength": {
@@ -1535,26 +1539,25 @@
         },
         "infillMethod": {
             "paragraphs": [
-                "Metodo di riempimento durante il processo di Outpainting o Inpainting."
+                "Metodo per riempire l'area selezionata."
             ],
             "heading": "Metodo di riempimento"
         },
         "controlNetWeight": {
             "heading": "Peso",
             "paragraphs": [
-                "Peso dell'adattatore di controllo. Un peso maggiore porterà a impatti maggiori sull'immagine finale."
+                "Quanto forte sarà l'impatto di ControlNet sull'immagine generata."
             ]
         },
         "paramCFGScale": {
             "heading": "Scala CFG",
             "paragraphs": [
-                "Controlla quanto il prompt influenza il processo di generazione.",
-                "Valori elevati della scala CFG possono provocare una saturazione eccessiva e distorsioni nei risultati della generazione. "
+                "Controlla quanto il tuo prompt influenza il processo di generazione."
             ]
         },
         "controlNetControlMode": {
             "paragraphs": [
-                "Attribuisce più peso al prompt oppure a ControlNet."
+                "Attribuisce più peso al prompt o a ControlNet."
             ],
             "heading": "Modalità di controllo"
         },
@@ -1566,9 +1569,9 @@
             ]
         },
         "lora": {
-            "heading": "LoRA",
+            "heading": "Peso LoRA",
             "paragraphs": [
-                "Modelli leggeri utilizzati insieme ai modelli base."
+                "Un peso LoRA più elevato porterà a impatti maggiori sull'immagine finale."
             ]
         },
         "controlNet": {
@@ -1580,123 +1583,7 @@
         "paramCFGRescaleMultiplier": {
             "heading": "Moltiplicatore di riscala CFG",
             "paragraphs": [
-                "Moltiplicatore di riscala per la guida CFG, utilizzato per modelli addestrati utilizzando SNR a terminale zero (ztsnr).",
-                "Valore suggerito di 0.7 per questi modelli."
-            ]
-        },
-        "controlNetProcessor": {
-            "heading": "Processore",
-            "paragraphs": [
-                "Metodo di elaborazione dell'immagine di input per guidare il processo di generazione. Processori diversi forniranno effetti o stili diversi nelle immagini generate."
-            ]
-        },
-        "imageFit": {
-            "heading": "Adatta l'immagine iniziale alle dimensioni di output",
-            "paragraphs": [
-                "Ridimensiona l'immagine iniziale in base alla larghezza e all'altezza dell'immagine di output. Si consiglia di abilitarlo."
-            ]
-        },
-        "loraWeight": {
-            "heading": "Peso",
-            "paragraphs": [
-                "Peso del LoRA. Un peso maggiore comporterà un impatto maggiore sull'immagine finale."
-            ]
-        },
-        "paramAspect": {
-            "heading": "Aspetto",
-            "paragraphs": [
-                "Proporzioni dell'immagine generata. La modifica del rapporto aggiornerà di conseguenza la larghezza e l'altezza.",
-                "\"Ottimizza\" imposterà la larghezza e l'altezza alle dimensioni ottimali per il modello scelto."
-            ]
-        },
-        "paramHeight": {
-            "heading": "Altezza",
-            "paragraphs": [
-                "Altezza dell'immagine generata. Deve essere un multiplo di 8."
-            ]
-        },
-        "paramHrf": {
-            "heading": "Abilita correzione alta risoluzione",
-            "paragraphs": [
-                "Genera immagini di alta qualità con una risoluzione maggiore di quella ottimale per il modello. Generalmente utilizzato per impedire la duplicazione nell'immagine generata."
-            ]
-        },
-        "paramUpscaleMethod": {
-            "heading": "Metodo di ampliamento",
-            "paragraphs": [
-                "Metodo utilizzato per eseguire l'ampliamento dell'immagine per la correzione ad alta risoluzione."
-            ]
-        },
-        "patchmatchDownScaleSize": {
-            "heading": "Ridimensiona",
-            "paragraphs": [
-                "Quanto ridimensionamento avviene prima del riempimento.",
-                "Un ridimensionamento più elevato migliorerà le prestazioni e ridurrà la qualità."
-            ]
-        },
-        "paramWidth": {
-            "paragraphs": [
-                "Larghezza dell'immagine generata. Deve essere un multiplo di 8."
-            ],
-            "heading": "Larghezza"
-        },
-        "refinerModel": {
-            "heading": "Modello Affinatore",
-            "paragraphs": [
-                "Modello utilizzato durante la parte di affinamento del processo di generazione.",
-                "Simile al modello di generazione."
-            ]
-        },
-        "refinerNegativeAestheticScore": {
-            "paragraphs": [
-                "Valuta le generazioni in modo che siano più simili alle immagini con un punteggio estetico basso, in base ai dati di addestramento."
-            ],
-            "heading": "Punteggio estetico negativo"
-        },
-        "refinerScheduler": {
-            "paragraphs": [
-                "Campionatore utilizzato durante la parte di affinamento del processo di generazione.",
-                "Simile al campionatore di generazione."
-            ],
-            "heading": "Campionatore"
-        },
-        "refinerStart": {
-            "heading": "Inizio affinamento",
-            "paragraphs": [
-                "A che punto nel processo di generazione inizierà ad essere utilizzato l'affinatore.",
-                "0 significa che l'affinatore verrà utilizzato per l'intero processo di generazione, 0.8 significa che l'affinatore verrà utilizzato per l'ultimo 20% del processo di generazione."
-            ]
-        },
-        "refinerSteps": {
-            "heading": "Passi",
-            "paragraphs": [
-                "Numero di passi che verranno eseguiti durante la parte di affinamento del processo di generazione.",
-                "Simile ai passi di generazione."
-            ]
-        },
-        "refinerCfgScale": {
-            "heading": "Scala CFG",
-            "paragraphs": [
-                "Controlla quanto il prompt influenza il processo di generazione.",
-                "Simile alla scala CFG di generazione."
-            ]
-        },
-        "seamlessTilingXAxis": {
-            "heading": "Asse X di piastrellatura senza cuciture",
-            "paragraphs": [
-                "Affianca senza soluzione di continuità un'immagine lungo l'asse orizzontale."
-            ]
-        },
-        "seamlessTilingYAxis": {
-            "heading": "Asse Y di piastrellatura senza cuciture",
-            "paragraphs": [
-                "Affianca senza soluzione di continuità un'immagine lungo l'asse verticale."
-            ]
-        },
-        "refinerPositiveAestheticScore": {
-            "heading": "Punteggio estetico positivo",
-            "paragraphs": [
-                "Valuta le generazioni in modo che siano più simili alle immagini con un punteggio estetico elevato, in base ai dati di addestramento."
+                "Moltiplicatore di riscala per la guida CFG, utilizzato per modelli addestrati utilizzando SNR a terminale zero (ztsnr). Valore suggerito 0.7."
             ]
         }
     },
@@ -1745,8 +1632,7 @@
         "steps": "Passi",
         "scheduler": "Campionatore",
         "recallParameters": "Richiama i parametri",
-        "noRecallParameters": "Nessun parametro da richiamare trovato",
-        "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)"
+        "noRecallParameters": "Nessun parametro da richiamare trovato"
     },
     "hrf": {
         "enableHrf": "Abilita Correzione Alta Risoluzione",