Revert "Update diffusers (#1094)" (#1096)

This reverts commit 0064cc2a6e.

apps/stable_diffusion/scripts/img2img.py

@@ -35,7 +35,7 @@ schedulers = None
 def img2img_inf(
     prompt: str,
     negative_prompt: str,
-    init_image: str,
+    init_image: Image,
     height: int,
     width: int,
     steps: int,
@@ -64,8 +64,11 @@ def img2img_inf(
     args.steps = steps
     args.strength = strength
     args.scheduler = scheduler
-    args.img_path = init_image
-    image = Image.open(args.img_path).convert("RGB")
+    args.img_path = "not none"
+
+    if init_image is None:
+        return None, "An Initial Image is required"
+    image = init_image.convert("RGB")
 
     # set ckpt_loc and hf_model_id.
     types = (
@@ -86,9 +89,6 @@ def img2img_inf(
     else:
         args.hf_model_id = custom_model
 
-    if image is None:
-        return None, "An Initial Image is required"
-
     args.save_metadata_to_json = save_metadata_to_json
     args.write_metadata_to_png = save_metadata_to_png
 

apps/stable_diffusion/scripts/inpaint.py

@@ -35,7 +35,7 @@ schedulers = None
 def inpaint_inf(
     prompt: str,
     negative_prompt: str,
-    image_loc,
+    image_dict,
     height: int,
     width: int,
     steps: int,
@@ -61,8 +61,8 @@ def inpaint_inf(
     args.guidance_scale = guidance_scale
     args.steps = steps
     args.scheduler = scheduler
-    args.img_path = image_loc["image"]
-    args.mask_path = image_loc["mask"]
+    args.img_path = "not none"
+    args.mask_path = "not none"
 
     # set ckpt_loc and hf_model_id.
     types = (
@@ -139,8 +139,8 @@ def inpaint_inf(
     generated_imgs = []
     seeds = []
     img_seed = utils.sanitize_seed(seed)
-    image = Image.open(args.img_path)
-    mask_image = Image.open(args.mask_path)
+    image = image_dict["image"]
+    mask_image = image_dict["mask"]
     for i in range(batch_count):
         if i > 0:
             img_seed = utils.sanitize_seed(-1)

apps/stable_diffusion/scripts/outpaint.py

@@ -157,8 +157,8 @@ def outpaint_inf(
             prompt,
             negative_prompt,
             image,
-            args.pixels,
-            args.mask_blur,
+            pixels,
+            mask_blur,
             left,
             right,
             top,

apps/stable_diffusion/src/models/model_wrappers.py

@@ -226,6 +226,11 @@ class SharkifyStableDiffusionModel:
                 )
                 self.in_channels = self.unet.in_channels
                 self.train(False)
+                if(args.attention_slicing is not None and args.attention_slicing != "none"):
+                    if(args.attention_slicing.isdigit()):
+                        self.unet.set_attention_slice(int(args.attention_slicing))
+                    else:
+                        self.unet.set_attention_slice(args.attention_slicing)
 
             def forward(
                 self, latent, timestep, text_embedding, guidance_scale

apps/stable_diffusion/src/pipelines/pipeline_shark_stable_diffusion_outpaint.py

@@ -69,11 +69,13 @@ class OutpaintPipeline(StableDiffusionPipeline):
         latents = latents * self.scheduler.init_noise_sigma
         return latents
 
-    def prepare_mask_and_masked_image(self, image, mask, mask_blur):
+    def prepare_mask_and_masked_image(
+        self, image, mask, mask_blur, width, height
+    ):
         if mask_blur > 0:
             mask = mask.filter(ImageFilter.GaussianBlur(mask_blur))
-        image = image.resize((512, 512))
-        mask = mask.resize((512, 512))
+        image = image.resize((width, height))
+        mask = mask.resize((width, height))
 
         # preprocess image
         if isinstance(image, (Image.Image, np.ndarray)):
@@ -478,7 +480,7 @@ class OutpaintPipeline(StableDiffusionPipeline):
 
             # Preprocess mask and image
             mask, masked_image = self.prepare_mask_and_masked_image(
-                image_to_process, mask_to_process, mask_blur
+                image_to_process, mask_to_process, mask_blur, width, height
             )
 
             # Prepare mask latent variables

apps/stable_diffusion/src/utils/stable_args.py

@@ -61,6 +61,7 @@ p.add_argument(
     "--height",
     type=int,
     default=512,
+    choices=range(384, 768, 8),
     help="the height of the output image.",
 )
 
@@ -68,6 +69,7 @@ p.add_argument(
     "--width",
     type=int,
     default=512,
+    choices=range(384, 768, 8),
     help="the width of the output image.",
 )
 
@@ -263,6 +265,13 @@ p.add_argument(
     help="Use the accelerate package to reduce cpu memory consumption",
 )
 
+p.add_argument(
+    "--attention_slicing",
+    type=str,
+    default="none",
+    help="Amount of attention slicing to use (one of 'max', 'auto', 'none', or an integer)",
+)
+
 ##############################################################################
 ### IREE - Vulkan supported flags
 ##############################################################################

apps/stable_diffusion/web/ui/img2img_ui.py

@@ -75,7 +75,7 @@ with gr.Blocks(title="Image-to-Image") as img2img_web:
                         elem_id="negative_prompt_box",
                     )
 
-                init_image = gr.Image(label="Input Image", type="filepath")
+                init_image = gr.Image(label="Input Image", type="pil")
 
                 with gr.Accordion(label="Advanced Options", open=False):
                     with gr.Row():
@@ -133,7 +133,7 @@ with gr.Blocks(title="Image-to-Image") as img2img_web:
                             0,
                             1,
                             value=args.strength,
-                            step=0.1,
+                            step=0.01,
                             label="Strength",
                         )
                     with gr.Row():

apps/stable_diffusion/web/ui/inpaint_ui.py

@@ -75,7 +75,7 @@ with gr.Blocks(title="Inpainting") as inpaint_web:
                     label="Masked Image",
                     source="upload",
                     tool="sketch",
-                    type="filepath",
+                    type="pil",
                 )
 
                 with gr.Accordion(label="Advanced Options", open=False):

build_tools/stable_diffusion_testing.py

@@ -20,6 +20,33 @@ model_config_dicts = get_json_file(
 )
 
 
+def parse_sd_out(filename, command, device, use_tune, model_name, import_mlir):
+    with open(filename, "r+") as f:
+        lines = f.readlines()
+    metrics = {}
+    vals_to_read = [
+        "Clip Inference time",
+        "Average step",
+        "VAE Inference time",
+        "Total image generation",
+    ]
+    for line in lines:
+        for val in vals_to_read:
+            if val in line:
+                metrics[val] = line.split(" ")[-1].strip("\n")
+
+    metrics["Average step"] = metrics["Average step"].strip("ms/it")
+    metrics["Total image generation"] = metrics[
+        "Total image generation"
+    ].strip("sec")
+    metrics["device"] = device
+    metrics["use_tune"] = use_tune
+    metrics["model_name"] = model_name
+    metrics["import_mlir"] = import_mlir
+    metrics["command"] = command
+    return metrics
+
+
 def get_inpaint_inputs():
     os.mkdir("./test_images/inputs")
     img_url = (
@@ -39,6 +66,7 @@ def get_inpaint_inputs():
 def test_loop(device="vulkan", beta=False, extra_flags=[]):
     # Get golden values from tank
     shutil.rmtree("./test_images", ignore_errors=True)
+    model_metrics = []
     os.mkdir("./test_images")
     os.mkdir("./test_images/golden")
     get_inpaint_inputs()
@@ -52,9 +80,16 @@ def test_loop(device="vulkan", beta=False, extra_flags=[]):
         inpaint_prompt_text = '--prompt="Face of a yellow cat, high resolution, sitting on a park bench"'
     if beta:
         extra_flags.append("--beta_models=True")
+    extra_flags.append("--no-progress_bar")
+    to_skip = [
+        "Linaqruf/anything-v3.0",
+        "prompthero/openjourney",
+        "wavymulder/Analog-Diffusion",
+        "dreamlike-art/dreamlike-diffusion-1.0",
+    ]
     for import_opt in import_options:
         for model_name in hf_model_names:
-            if model_name == "Linaqruf/anything-v3.0":
+            if model_name in to_skip:
                 continue
             for use_tune in tuned_options:
                 command = (
@@ -73,7 +108,7 @@ def test_loop(device="vulkan", beta=False, extra_flags=[]):
                     ]
                     if "inpainting" not in model_name
                     else [
-                        "python",
+                        executable,
                         "apps/stable_diffusion/scripts/inpaint.py",
                         "--device=" + device,
                         inpaint_prompt_text,
@@ -91,12 +126,27 @@ def test_loop(device="vulkan", beta=False, extra_flags=[]):
                 command += extra_flags
                 if os.name == "nt":
                     command = " ".join(command)
-                generated_image = not subprocess.call(
-                    command, stdout=subprocess.DEVNULL
-                )
+                dumpfile_name = "_".join(model_name.split("/")) + ".txt"
+                dumpfile_name = os.path.join(os.getcwd(), dumpfile_name)
+                with open(dumpfile_name, "w+") as f:
+                    generated_image = not subprocess.call(
+                        command,
+                        stdout=f,
+                        stderr=f,
+                    )
                 if os.name != "nt":
                     command = " ".join(command)
                 if generated_image:
+                    model_metrics.append(
+                        parse_sd_out(
+                            dumpfile_name,
+                            command,
+                            device,
+                            use_tune,
+                            model_name,
+                            import_opt,
+                        )
+                    )
                     print(command)
                     print("Successfully generated image")
                     os.makedirs(
@@ -127,6 +177,22 @@ def test_loop(device="vulkan", beta=False, extra_flags=[]):
                     if "2_1_base" in model_name:
                         print("failed a known successful model.")
                         exit(1)
+    with open(os.path.join(os.getcwd(), "sd_testing_metrics.csv"), "w+") as f:
+        header = "model_name;device;use_tune;import_opt;Clip Inference time(ms);Average Step (ms/it);VAE Inference time(ms);total image generation(s);command\n"
+        f.write(header)
+        for metric in model_metrics:
+            output = [
+                metric["model_name"],
+                metric["device"],
+                metric["use_tune"],
+                metric["import_mlir"],
+                metric["Clip Inference time"],
+                metric["Average step"],
+                metric["VAE Inference time"],
+                metric["Total image generation"],
+                metric["command"],
+            ]
+            f.write(";".join(output) + "\n")
 
 
 parser = argparse.ArgumentParser()