Merge pull request #1787 from easydiffusion/beta

Controlnet Strength and SDXL Controlnet support for img2img and inpainting

CHANGES.md

@@ -17,6 +17,8 @@
 - **Major rewrite of the code** - We've switched to using diffusers under-the-hood, which allows us to release new features faster, and focus on making the UI and installer even easier to use.
 
 ### Detailed changelog
+* 3.0.9 - 28 May 2024 - Slider for controlling the strength of controlnets.
+* 3.0.8 - 27 May 2024 - SDXL ControlNets for Img2Img and Inpainting.
 * 3.0.7 - 11 Dec 2023 - Setting to enable/disable VAE tiling (in the Image Settings panel). Sometimes VAE tiling reduces the quality of the image, so this setting will help control that.
 * 3.0.6 - 18 Sep 2023 - Add thumbnails to embeddings from the UI, using the new `Upload Thumbnail` button in the Embeddings popup. Thanks @JeLuf.
 * 3.0.6 - 15 Sep 2023 - Fix broken embeddings dialog when LoRA information couldn't be fetched.

scripts/check_modules.py

@@ -22,7 +22,8 @@ modules_to_check = {
     "torch": ("1.11.0", "1.13.1", "2.0.0", "2.0.1"),
     "torchvision": ("0.12.0", "0.14.1", "0.15.1", "0.15.2"),
     "setuptools": "69.5.1",
-    "sdkit": "2.0.15",
+    "sdkit": "2.0.15.6",
+    "diffusers": "0.21.4",
     "stable-diffusion-sdkit": "2.1.5",
     "rich": "12.6.0",
     "uvicorn": "0.19.0",

ui/easydiffusion/utils/save_utils.py

@@ -31,6 +31,7 @@ TASK_TEXT_MAPPING = {
     "clip_skip": "Clip Skip",
     "use_controlnet_model": "ControlNet model",
     "control_filter_to_apply": "ControlNet Filter",
+    "control_alpha": "ControlNet Strength",
     "use_vae_model": "VAE model",
     "sampler_name": "Sampler",
     "width": "Width",

ui/index.html

@@ -35,7 +35,7 @@
             <h1>
                 <img id="logo_img" src="/media/images/icon-512x512.png" >
                 Easy Diffusion
-                <small><span id="version">v3.0.7</span> <span id="updateBranchLabel"></span></small>
+                <small><span id="version">v3.0.9</span> <span id="updateBranchLabel"></span></small>
             </h1>
         </div>
         <div id="server-status">
@@ -235,6 +235,8 @@
                                 <label for="controlnet_model"><small>Model:</small></label> <input id="controlnet_model" type="text" spellcheck="false" autocomplete="off" class="model-filter" data-path="" />
                                 <br/>
                                 <label><small>Will download the necessary models, the first time.</small></label>
+                                <br/>
+                                <label for="controlnet_alpha_slider"><small>Strength:</small></label> <input id="controlnet_alpha_slider" name="controlnet_alpha_slider" class="editor-slider" value="10" type="range" min="0" max="10"> <input id="controlnet_alpha" name="controlnet_alpha" size="4" pattern="^[0-9\.]+$" onkeypress="preventNonNumericalInput(event)" inputmode="decimal">
                             </div>
                         </td>
                     </tr>

ui/media/js/auto-save.js

@@ -57,6 +57,7 @@ const SETTINGS_IDS_LIST = [
     "embedding-card-size-selector",
     "lora_model",
     "enable_vae_tiling",
+    "controlnet_alpha",
 ]
 
 const IGNORE_BY_DEFAULT = ["prompt"]

ui/media/js/dnd.js

@@ -309,10 +309,21 @@ const TASK_MAPPING = {
         readUI: () => controlImageFilterField.value,
         parse: (val) => val,
     },
+    control_alpha: {
+        name: "ControlNet Strength",
+        setUI: (control_alpha) => {
+            control_alpha = control_alpha || 1.0
+            controlAlphaField.value = control_alpha
+            updateControlAlphaSlider()
+        },
+        readUI: () => parseFloat(controlAlphaField.value),
+        parse: (val) => val === null ? 1.0 : parseFloat(val),
+    },
     use_lora_model: {
         name: "LoRA model",
         setUI: (use_lora_model) => {
             let modelPaths = []
+            use_lora_model = use_lora_model === null ? "" : use_lora_model
             use_lora_model = Array.isArray(use_lora_model) ? use_lora_model : [use_lora_model]
             use_lora_model.forEach((m) => {
                 if (m.includes("models\\lora\\")) {
@@ -529,6 +540,11 @@ function restoreTaskToUI(task, fieldsToSkip) {
         // listen for inpainter loading event, which happens AFTER the main image loads (which reloads the inpai
         controlImagePreview.src = task.reqBody.control_image
     }
+
+    if ("use_controlnet_model" in task.reqBody && task.reqBody.use_controlnet_model && !("control_alpha" in task.reqBody)) {
+        controlAlphaField.value = 1.0
+        updateControlAlphaSlider()
+    }
 }
 function readUI() {
     const reqBody = {}
@@ -587,6 +603,7 @@ const TASK_TEXT_MAPPING = {
     lora_alpha: "LoRA Strength",
     use_controlnet_model: "ControlNet model",
     control_filter_to_apply: "ControlNet Filter",
+    control_alpha: "ControlNet Strength",
     tiling: "Seamless Tiling",
 }
 function parseTaskFromText(str) {

ui/media/js/main.js

@@ -51,6 +51,10 @@ const taskConfigSetup = {
         preserve_init_image_color_profile: "Preserve Color Profile",
         strict_mask_border: "Strict Mask Border",
         use_controlnet_model: "ControlNet Model",
+        control_alpha: {
+            label: "ControlNet Strength",
+            visible: ({ reqBody }) => !!reqBody?.use_controlnet_model,
+        },
     },
     pluginTaskConfig: {},
     getCSSKey: (key) =>
@@ -99,6 +103,8 @@ let controlImagePreview = document.querySelector("#control_image_preview")
 let controlImageClearBtn = document.querySelector(".control_image_clear")
 let controlImageContainer = document.querySelector("#control_image_wrapper")
 let controlImageFilterField = document.querySelector("#control_image_filter")
+let controlAlphaSlider = document.querySelector("#controlnet_alpha_slider")
+let controlAlphaField = document.querySelector("#controlnet_alpha")
 let applyColorCorrectionField = document.querySelector("#apply_color_correction")
 let strictMaskBorderField = document.querySelector("#strict_mask_border")
 let colorCorrectionSetting = document.querySelector("#apply_color_correction_setting")
@@ -1395,6 +1401,7 @@ function getCurrentUserRequest() {
     if (controlnetModelField.value !== "" && IMAGE_REGEX.test(controlImagePreview.src)) {
         newTask.reqBody.use_controlnet_model = controlnetModelField.value
         newTask.reqBody.control_image = controlImagePreview.src
+        newTask.reqBody.control_alpha = parseFloat(controlAlphaField.value)
         if (controlImageFilterField.value !== "") {
             newTask.reqBody.control_filter_to_apply = controlImageFilterField.value
         }
@@ -2015,6 +2022,27 @@ function updateHypernetworkStrengthContainer() {
 hypernetworkModelField.addEventListener("change", updateHypernetworkStrengthContainer)
 updateHypernetworkStrengthContainer()
 
+/********************* Controlnet Alpha **************************/
+function updateControlAlpha() {
+    controlAlphaField.value = controlAlphaSlider.value / 10
+    controlAlphaField.dispatchEvent(new Event("change"))
+}
+
+function updateControlAlphaSlider() {
+    if (controlAlphaField.value < 0) {
+        controlAlphaField.value = 0
+    } else if (controlAlphaField.value > 10) {
+        controlAlphaField.value = 10
+    }
+
+    controlAlphaSlider.value = controlAlphaField.value * 10
+    controlAlphaSlider.dispatchEvent(new Event("change"))
+}
+
+controlAlphaSlider.addEventListener("input", updateControlAlpha)
+controlAlphaField.addEventListener("input", updateControlAlphaSlider)
+updateControlAlpha()
+
 /********************* JPEG/WEBP Quality **********************/
 function updateOutputQuality() {
     outputQualityField.value = 0 | outputQualitySlider.value