From e03a3fcf68184694271019715cab7f6439fc7e82 Mon Sep 17 00:00:00 2001 From: Carson Katri Date: Sun, 16 Oct 2022 22:45:18 -0400 Subject: [PATCH 1/4] Add seamless_axes options --- ldm/generate.py | 26 +++++++++++++++++++++++++- ldm/invoke/args.py | 6 ++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ldm/generate.py b/ldm/generate.py index fb7b0dc26f..b0d64cd737 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -173,6 +173,7 @@ class Generate: self.precision = precision self.strength = 0.75 self.seamless = False + self.seamless_axes = {'x','y'} self.hires_fix = False self.embedding_path = embedding_path self.model = None # empty for now @@ -258,6 +259,7 @@ class Generate: height = None, sampler_name = None, seamless = False, + seamless_axes = {'x','y'}, log_tokenization = False, with_variations = None, variation_amount = 0.0, @@ -330,6 +332,7 @@ class Generate: width = width or self.width height = height or self.height seamless = seamless or self.seamless + seamless_axes = seamless_axes or self.seamless_axes hires_fix = hires_fix or self.hires_fix cfg_scale = cfg_scale or self.cfg_scale ddim_eta = ddim_eta or self.ddim_eta @@ -348,9 +351,30 @@ class Generate: width = width or self.width height = height or self.height + def _conv_forward_asymmetric(self, input, weight, bias): + """ + Patch for Conv2d._conv_forward that supports asymmetric padding + """ + working = nn.functional.pad(input, self.asymmetric_padding['x'], mode=self.asymmetric_padding_mode['x']) + working = nn.functional.pad(working, self.asymmetric_padding['y'], mode=self.asymmetric_padding_mode['y']) + return nn.functional.conv2d(working, weight, bias, self.stride, nn.modules.utils._pair(0), self.dilation, self.groups) + for m in model.modules(): if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): - m.padding_mode = 'circular' if seamless else m._orig_padding_mode + if seamless: + m.asymmetric_padding_mode = {} + m.asymmetric_padding = {} + m.asymmetric_padding_mode['x'] = 'circular' if ('x' in seamless_axes) else 'constant' + m.asymmetric_padding['x'] = (m._reversed_padding_repeated_twice[0], m._reversed_padding_repeated_twice[1], 0, 0) + m.asymmetric_padding_mode['y'] = 'circular' if ('y' in seamless_axes) else 'constant' + m.asymmetric_padding['y'] = (0, 0, m._reversed_padding_repeated_twice[2], m._reversed_padding_repeated_twice[3]) + m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) + else: + m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) + if hasattr(m, 'asymmetric_padding_mode'): + del m.asymmetric_padding_mode + if hasattr(m, 'asymmetric_padding'): + del m.asymmetric_padding assert cfg_scale > 1.0, 'CFG_Scale (-C) must be >1.0' assert threshold >= 0.0, '--threshold must be >=0.0' diff --git a/ldm/invoke/args.py b/ldm/invoke/args.py index 11f81a9359..78ead1bfb4 100644 --- a/ldm/invoke/args.py +++ b/ldm/invoke/args.py @@ -761,6 +761,12 @@ class Args(object): action='store_true', help='Change the model to seamless tiling (circular) mode', ) + special_effects_group.add_argument( + '--seamless_axes', + default=['x', 'y'], + type=list[str], + help='Specify which axes to use circular convolution on.', + ) variation_group.add_argument( '-v', '--variation_amount', From 99581dbbf780dcfea096d8238eadb26ee0910787 Mon Sep 17 00:00:00 2001 From: Carson Katri Date: Mon, 17 Oct 2022 19:31:20 -0400 Subject: [PATCH 2/4] Split seamless config into separate file --- ldm/generate.py | 26 ++------------------------ ldm/invoke/seamless.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 ldm/invoke/seamless.py diff --git a/ldm/generate.py b/ldm/generate.py index b0d64cd737..3c43e40db3 100644 --- a/ldm/generate.py +++ b/ldm/generate.py @@ -34,6 +34,7 @@ from ldm.invoke.image_util import InitImageResizer from ldm.invoke.devices import choose_torch_device, choose_precision from ldm.invoke.conditioning import get_uc_and_c from ldm.invoke.model_cache import ModelCache +from ldm.invoke.seamless import configure_model_padding def fix_func(orig): if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): @@ -350,31 +351,8 @@ class Generate: # to the width and height of the image training set width = width or self.width height = height or self.height - - def _conv_forward_asymmetric(self, input, weight, bias): - """ - Patch for Conv2d._conv_forward that supports asymmetric padding - """ - working = nn.functional.pad(input, self.asymmetric_padding['x'], mode=self.asymmetric_padding_mode['x']) - working = nn.functional.pad(working, self.asymmetric_padding['y'], mode=self.asymmetric_padding_mode['y']) - return nn.functional.conv2d(working, weight, bias, self.stride, nn.modules.utils._pair(0), self.dilation, self.groups) - for m in model.modules(): - if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): - if seamless: - m.asymmetric_padding_mode = {} - m.asymmetric_padding = {} - m.asymmetric_padding_mode['x'] = 'circular' if ('x' in seamless_axes) else 'constant' - m.asymmetric_padding['x'] = (m._reversed_padding_repeated_twice[0], m._reversed_padding_repeated_twice[1], 0, 0) - m.asymmetric_padding_mode['y'] = 'circular' if ('y' in seamless_axes) else 'constant' - m.asymmetric_padding['y'] = (0, 0, m._reversed_padding_repeated_twice[2], m._reversed_padding_repeated_twice[3]) - m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) - else: - m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) - if hasattr(m, 'asymmetric_padding_mode'): - del m.asymmetric_padding_mode - if hasattr(m, 'asymmetric_padding'): - del m.asymmetric_padding + configure_model_padding(model, seamless, seamless_axes) assert cfg_scale > 1.0, 'CFG_Scale (-C) must be >1.0' assert threshold >= 0.0, '--threshold must be >=0.0' diff --git a/ldm/invoke/seamless.py b/ldm/invoke/seamless.py new file mode 100644 index 0000000000..2dbe241921 --- /dev/null +++ b/ldm/invoke/seamless.py @@ -0,0 +1,30 @@ +import torch.nn as nn + +def _conv_forward_asymmetric(self, input, weight, bias): + """ + Patch for Conv2d._conv_forward that supports asymmetric padding + """ + working = nn.functional.pad(input, self.asymmetric_padding['x'], mode=self.asymmetric_padding_mode['x']) + working = nn.functional.pad(working, self.asymmetric_padding['y'], mode=self.asymmetric_padding_mode['y']) + return nn.functional.conv2d(working, weight, bias, self.stride, nn.modules.utils._pair(0), self.dilation, self.groups) + +def configure_model_padding(model, seamless, seamless_axes): + """ + Modifies the 2D convolution layers to use a circular padding mode based on the `seamless` and `seamless_axes` options. + """ + for m in model.modules(): + if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): + if seamless: + m.asymmetric_padding_mode = {} + m.asymmetric_padding = {} + m.asymmetric_padding_mode['x'] = 'circular' if ('x' in seamless_axes) else 'constant' + m.asymmetric_padding['x'] = (m._reversed_padding_repeated_twice[0], m._reversed_padding_repeated_twice[1], 0, 0) + m.asymmetric_padding_mode['y'] = 'circular' if ('y' in seamless_axes) else 'constant' + m.asymmetric_padding['y'] = (0, 0, m._reversed_padding_repeated_twice[2], m._reversed_padding_repeated_twice[3]) + m._conv_forward = _conv_forward_asymmetric.__get__(m, nn.Conv2d) + else: + m._conv_forward = nn.Conv2d._conv_forward.__get__(m, nn.Conv2d) + if hasattr(m, 'asymmetric_padding_mode'): + del m.asymmetric_padding_mode + if hasattr(m, 'asymmetric_padding'): + del m.asymmetric_padding \ No newline at end of file From 4079333e2992cc26bedd4f1b725fb0a91d9d44ff Mon Sep 17 00:00:00 2001 From: Carson Katri Date: Mon, 17 Oct 2022 19:33:17 -0400 Subject: [PATCH 3/4] Document the seamless_axes argument --- docs/features/OTHER.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/features/OTHER.md b/docs/features/OTHER.md index b05e3ea36e..b3d8be6077 100644 --- a/docs/features/OTHER.md +++ b/docs/features/OTHER.md @@ -26,6 +26,12 @@ for each `invoke>` prompt as shown here: invoke> "pond garden with lotus by claude monet" --seamless -s100 -n4 ``` +By default this will tile on both the X and Y axes. However, you can also specify specific axes to tile on with `--seamless_axes`. +Possible values are `x`, `y`, and `x,y`: +```python +invoke> "pond garden with lotus by claude monet" --seamless --seamless_axes=x -s100 -n4 +``` + --- ## **Shortcuts: Reusing Seeds** From d6195522aa6b81184aaece417435108269be81b8 Mon Sep 17 00:00:00 2001 From: Carson Katri Date: Mon, 17 Oct 2022 20:17:34 -0400 Subject: [PATCH 4/4] Add seamless_axes docs to CLI.md --- docs/features/CLI.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/features/CLI.md b/docs/features/CLI.md index f68379891d..b22c5fbeeb 100644 --- a/docs/features/CLI.md +++ b/docs/features/CLI.md @@ -157,6 +157,7 @@ Here are the invoke> command that apply to txt2img: | --individual | -i | True | Turn off grid mode (deprecated; leave off --grid instead) | | --outdir | -o | outputs/img_samples | Temporarily change the location of these images | | --seamless | | False | Activate seamless tiling for interesting effects | +| --seamless_axes | | x,y | Specify which axes to use circular convolution on. | | --log_tokenization | -t | False | Display a color-coded list of the parsed tokens derived from the prompt | | --skip_normalization| -x | False | Weighted subprompts will not be normalized. See [Weighted Prompts](./OTHER.md#weighted-prompts) | | --upscale | -U | -U 1 0.75| Upscale image by magnification factor (2, 4), and set strength of upscaling (0.0-1.0). If strength not set, will default to 0.75. |