From d39e1da183703f220c507725d1e22c7bc5ab616b Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 11:49:30 +0530 Subject: [PATCH 1/9] Fixes for TensorRT --- ui/easydiffusion/tasks/render_images.py | 24 +++++++++++++++++++++++- ui/index.html | 2 +- ui/media/js/main.js | 12 +++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/ui/easydiffusion/tasks/render_images.py b/ui/easydiffusion/tasks/render_images.py index bbc36aa5..f14d478d 100644 --- a/ui/easydiffusion/tasks/render_images.py +++ b/ui/easydiffusion/tasks/render_images.py @@ -63,7 +63,7 @@ class RenderTask(Task): if ( runtime.set_vram_optimizations(context) or self.has_param_changed(context, "clip_skip") - or self.has_param_changed(context, "convert_to_tensorrt") + or self.trt_needs_reload(context) ): models_to_force_reload.append("stable-diffusion") @@ -92,6 +92,17 @@ class RenderTask(Task): new_val = self.models_data.model_params.get("stable-diffusion", {}).get(param_name, False) return model["params"].get(param_name) != new_val + def trt_needs_reload(self, context): + if not self.has_param_changed(context, "convert_to_tensorrt"): + return False + + model = context.models["stable-diffusion"] + pipe = model["default"] + if hasattr(pipe.unet, "_allocate_trt_buffers"): # TRT already loaded + return False + + return True + def make_images( context, @@ -148,6 +159,7 @@ def make_images_internal( context, req, task_data, + models_data, data_queue, task_temp_images, step_callback, @@ -174,6 +186,7 @@ def generate_images_internal( context, req: GenerateImageRequest, task_data: TaskData, + models_data: ModelsData, data_queue: queue.Queue, task_temp_images: list, step_callback, @@ -197,6 +210,15 @@ def generate_images_internal( if req.init_image is not None and not context.test_diffusers: req.sampler_name = "ddim" + if context.test_diffusers: + pipe = context.models["stable-diffusion"]["default"] + if hasattr(pipe.unet, "_allocate_trt_buffers"): + convert_to_trt = models_data.model_params["stable-diffusion"].get("convert_to_tensorrt", False) + pipe.unet.forward = pipe.unet._trt_forward if convert_to_trt else pipe.unet._non_trt_forward + # pipe.vae.decoder.forward = ( + # pipe.vae.decoder._trt_forward if convert_to_trt else pipe.vae.decoder._non_trt_forward + # ) + images = generate_images(context, callback=callback, **req.dict()) user_stopped = False except UserInitiatedStop: diff --git a/ui/index.html b/ui/index.html index 0fcb807d..a16db050 100644 --- a/ui/index.html +++ b/ui/index.html @@ -148,7 +148,7 @@ Click to learn more about custom models - + Click to learn more about TensorRT diff --git a/ui/media/js/main.js b/ui/media/js/main.js index 1c76543a..23be437c 100644 --- a/ui/media/js/main.js +++ b/ui/media/js/main.js @@ -1346,9 +1346,19 @@ function createTask(task) { function getCurrentUserRequest() { const numOutputsTotal = parseInt(numOutputsTotalField.value) - const numOutputsParallel = parseInt(numOutputsParallelField.value) + let numOutputsParallel = parseInt(numOutputsParallelField.value) const seed = randomSeedField.checked ? Math.floor(Math.random() * (2 ** 32 - 1)) : parseInt(seedField.value) + if ( + testDiffusers.checked && + document.getElementById("toggle-tensorrt-install").innerHTML == "Uninstall" && + document.querySelector("#convert_to_tensorrt").checked + ) { + // TRT enabled + + numOutputsParallel = 1 // force 1 parallel + } + const newTask = { batchesDone: 0, numOutputsTotal: numOutputsTotal, From 4cee1be99c608e3cfe1bbb849464ed403643166b Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 12:43:23 +0530 Subject: [PATCH 2/9] Default settings for TensorRT demo; Don't show splash screen for diffusers --- ui/index.html | 2 +- ui/media/js/main.js | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ui/index.html b/ui/index.html index a16db050..e98ede61 100644 --- a/ui/index.html +++ b/ui/index.html @@ -706,7 +706,7 @@ async function init() { ping: onPing } }) - splashScreen() + // splashScreen() // load models again, but scan for malicious this time await getModels(true) diff --git a/ui/media/js/main.js b/ui/media/js/main.js index 23be437c..157e6e79 100644 --- a/ui/media/js/main.js +++ b/ui/media/js/main.js @@ -2271,6 +2271,8 @@ function tunnelUpdate(event) { } } +let trtSettingsForced = false + function packagesUpdate(event) { let trtBtn = document.getElementById("toggle-tensorrt-install") let trtInstalled = "packages_installed" in event && "tensorrt" in event["packages_installed"] @@ -2285,6 +2287,19 @@ function packagesUpdate(event) { if (document.getElementById("toggle-tensorrt-install").innerHTML == "Uninstall") { document.querySelector("#enable_trt_config").classList.remove("displayNone") + + if (!trtSettingsForced) { + // settings for demo + promptField.value = "Dragons fighting with a knight, castle, war scene, fantasy, cartoon, flames, HD" + seedField.value = 3187947173 + widthField.value = 1024 + heightField.value = 768 + randomSeedField.checked = false + seedField.disabled = false + stableDiffusionModelField.value = "sd-v1-4" + + trtSettingsForced = true + } } } From 9690fd1fa868849f81b22f30dddcd78758a9b247 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 12:45:56 +0530 Subject: [PATCH 3/9] sdkit 1.0.154 - restrict tensorrt from 768x768 to 1024x1024, and Unet-only, to avoid going out of memory --- scripts/check_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_modules.py b/scripts/check_modules.py index d5886a9a..6bf3b1b3 100644 --- a/scripts/check_modules.py +++ b/scripts/check_modules.py @@ -18,7 +18,7 @@ os_name = platform.system() modules_to_check = { "torch": ("1.11.0", "1.13.1", "2.0.0"), "torchvision": ("0.12.0", "0.14.1", "0.15.1"), - "sdkit": "1.0.153", + "sdkit": "1.0.154", "stable-diffusion-sdkit": "2.1.4", "rich": "12.6.0", "uvicorn": "0.19.0", From 05ed110519ab80bb0e907c116d2d55e3c9251d56 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 13:02:53 +0530 Subject: [PATCH 4/9] Don't show parallel field for tensorrt demo --- ui/index.html | 2 +- ui/media/js/main.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/index.html b/ui/index.html index e98ede61..d2a6194c 100644 --- a/ui/index.html +++ b/ui/index.html @@ -141,7 +141,7 @@
Image Settings - + @@ -162,6 +161,58 @@ Click to learn more about Clip Skip + @@ -211,6 +211,8 @@
+
+
diff --git a/ui/media/js/main.js b/ui/media/js/main.js index 157e6e79..27724ee1 100644 --- a/ui/media/js/main.js +++ b/ui/media/js/main.js @@ -2298,6 +2298,9 @@ function packagesUpdate(event) { seedField.disabled = false stableDiffusionModelField.value = "sd-v1-4" + numOutputsParallelField.classList.add("displayNone") + document.querySelector("#num_outputs_parallel_label").classList.add("displayNone") + trtSettingsForced = true } } From ee6db857681a7a86befbe6e24cea04f095c3bd3d Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 15:39:15 +0530 Subject: [PATCH 5/9] Initial support for Controlnet --- ui/easydiffusion/model_manager.py | 9 +++- ui/easydiffusion/tasks/render_images.py | 3 ++ ui/easydiffusion/types.py | 5 ++ ui/index.html | 59 ++++++++++++++++++++-- ui/media/css/main.css | 16 ++++-- ui/media/js/main.js | 65 +++++++++++++++++++++++++ ui/media/js/searchable-models.js | 5 +- 7 files changed, 153 insertions(+), 9 deletions(-) diff --git a/ui/easydiffusion/model_manager.py b/ui/easydiffusion/model_manager.py index 1ee5ce9d..2e7fef67 100644 --- a/ui/easydiffusion/model_manager.py +++ b/ui/easydiffusion/model_manager.py @@ -9,6 +9,7 @@ from easydiffusion.types import ModelsData from easydiffusion.utils import log from sdkit import Context from sdkit.models import load_model, scan_model, unload_model, download_model, get_model_info_from_db +from sdkit.models.model_loader.controlnet_filters import filters as cn_filters from sdkit.utils import hash_file_quick KNOWN_MODEL_TYPES = [ @@ -19,6 +20,8 @@ KNOWN_MODEL_TYPES = [ "realesrgan", "lora", "codeformer", + "embeddings", + "controlnet", ] MODEL_EXTENSIONS = { "stable-diffusion": [".ckpt", ".safetensors"], @@ -29,6 +32,7 @@ MODEL_EXTENSIONS = { "lora": [".ckpt", ".safetensors"], "codeformer": [".pth"], "embeddings": [".pt", ".bin", ".safetensors"], + "controlnet": [".pth", ".safetensors"], } DEFAULT_MODELS = { "stable-diffusion": [ @@ -177,7 +181,8 @@ def reload_models_if_necessary(context: Context, models_data: ModelsData, models def resolve_model_paths(models_data: ModelsData): model_paths = models_data.model_paths for model_type in model_paths: - if model_type in ("latent_upscaler", "nsfw_checker"): # doesn't use model paths + skip_models = cn_filters + ["latent_upscaler", "nsfw_checker"] + if model_type in skip_models: # doesn't use model paths continue if model_type == "codeformer": download_if_necessary("codeformer", "codeformer.pth", "codeformer-0.1.0") @@ -291,6 +296,7 @@ def getModels(scan_for_malicious: bool = True): "lora": [], "codeformer": ["codeformer"], "embeddings": [], + "controlnet": [], }, } @@ -350,6 +356,7 @@ def getModels(scan_for_malicious: bool = True): listModels(model_type="gfpgan") listModels(model_type="lora") listModels(model_type="embeddings") + listModels(model_type="controlnet") if scan_for_malicious and models_scanned > 0: log.info(f"[green]Scanned {models_scanned} models. Nothing infected[/]") diff --git a/ui/easydiffusion/tasks/render_images.py b/ui/easydiffusion/tasks/render_images.py index f14d478d..8df208b6 100644 --- a/ui/easydiffusion/tasks/render_images.py +++ b/ui/easydiffusion/tasks/render_images.py @@ -210,6 +210,9 @@ def generate_images_internal( if req.init_image is not None and not context.test_diffusers: req.sampler_name = "ddim" + if req.control_image and task_data.control_filter_to_apply: + req.control_image = filter_images(context, req.control_image, task_data.control_filter_to_apply)[0] + if context.test_diffusers: pipe = context.models["stable-diffusion"]["default"] if hasattr(pipe.unet, "_allocate_trt_buffers"): diff --git a/ui/easydiffusion/types.py b/ui/easydiffusion/types.py index 894867b8..181a9505 100644 --- a/ui/easydiffusion/types.py +++ b/ui/easydiffusion/types.py @@ -75,6 +75,7 @@ class TaskData(BaseModel): use_controlnet_model: Union[str, List[str]] = None filters: List[str] = [] filter_params: Dict[str, Dict[str, Any]] = {} + control_filter_to_apply: Union[str, List[str]] = None show_only_filtered_image: bool = False block_nsfw: bool = False @@ -135,6 +136,7 @@ class GenerateImageResponse: def json(self): del self.render_request.init_image del self.render_request.init_image_mask + del self.render_request.control_image task_data = self.task_data.dict() task_data.update(self.output_format.dict()) @@ -212,6 +214,9 @@ def convert_legacy_render_req_to_new(old_req: dict): model_paths["latent_upscaler"] = ( model_paths["latent_upscaler"] if "latent_upscaler" in model_paths["latent_upscaler"].lower() else None ) + if "control_filter_to_apply" in old_req: + filter_model = old_req["control_filter_to_apply"] + model_paths[filter_model] = filter_model if old_req.get("block_nsfw"): model_paths["nsfw_checker"] = "nsfw_checker" diff --git a/ui/index.html b/ui/index.html index d2a6194c..68616ed4 100644 --- a/ui/index.html +++ b/ui/index.html @@ -83,8 +83,8 @@
-
- +
+
@@ -151,7 +151,6 @@
- Click to learn more about TensorRT
+
+ + + +
+ + Click to learn more about ControlNets +
+ + +
+ +
+
Click to learn more about VAEs @@ -239,7 +290,7 @@
- Recent sizes + Advanced sizes
Custom size:
diff --git a/ui/media/css/main.css b/ui/media/css/main.css index 5e1cee43..3e07448e 100644 --- a/ui/media/css/main.css +++ b/ui/media/css/main.css @@ -794,7 +794,7 @@ div.img-preview img { margin-bottom: 8px; } -#init_image_preview_container:not(.has-image) #init_image_wrapper, +#init_image_preview_container:not(.has-image) .preview_image_wrapper, #init_image_preview_container:not(.has-image) #inpaint_button_container { display: none; } @@ -831,14 +831,14 @@ div.img-preview img { gap: 8px; } -#init_image_wrapper { +.preview_image_wrapper { grid-row: span 3; position: relative; width: fit-content; max-height: 150px; } -#init_image_preview { +.image_preview { max-height: 150px; height: 100%; width: 100%; @@ -1817,3 +1817,13 @@ div#enlarge-buttons { .imgContainer .spinnerStatus { font-size: 10pt; } + +#controlnet_model_container small { + color: var(--text-color) +} +#control_image { + width: 130pt; +} +#controlnet_model { + width: 77%; +} \ No newline at end of file diff --git a/ui/media/js/main.js b/ui/media/js/main.js index 27724ee1..97ce807a 100644 --- a/ui/media/js/main.js +++ b/ui/media/js/main.js @@ -93,6 +93,11 @@ let initImagePreview = document.querySelector("#init_image_preview") let initImageSizeBox = document.querySelector("#init_image_size_box") let maskImageSelector = document.querySelector("#mask") let maskImagePreview = document.querySelector("#mask_preview") +let controlImageSelector = document.querySelector("#control_image") +let controlImagePreview = document.querySelector("#control_image_preview") +let controlImageClearBtn = document.querySelector(".control_image_clear") +let controlImageContainer = document.querySelector("#control_image_wrapper") +let controlImageFilterField = document.querySelector("#control_image_filter") let applyColorCorrectionField = document.querySelector("#apply_color_correction") let strictMaskBorderField = document.querySelector("#strict_mask_border") let colorCorrectionSetting = document.querySelector("#apply_color_correction_setting") @@ -114,6 +119,7 @@ let codeformerFidelityField = document.querySelector("#codeformer_fidelity") let stableDiffusionModelField = new ModelDropdown(document.querySelector("#stable_diffusion_model"), "stable-diffusion") let clipSkipField = document.querySelector("#clip_skip") let tilingField = document.querySelector("#tiling") +let controlnetModelField = new ModelDropdown(document.querySelector("#controlnet_model"), "controlnet", "None") let vaeModelField = new ModelDropdown(document.querySelector("#vae_model"), "vae", "None") let hypernetworkModelField = new ModelDropdown(document.querySelector("#hypernetwork_model"), "hypernetwork", "None") let hypernetworkStrengthSlider = document.querySelector("#hypernetwork_strength_slider") @@ -1447,6 +1453,13 @@ function getCurrentUserRequest() { // TRT is installed newTask.reqBody.convert_to_tensorrt = document.querySelector("#convert_to_tensorrt").checked } + if (controlnetModelField.value !== "" && IMAGE_REGEX.test(controlImagePreview.src)) { + newTask.reqBody.use_controlnet_model = controlnetModelField.value + newTask.reqBody.control_image = controlImagePreview.src + if (controlImageFilterField.value !== "") { + newTask.reqBody.control_filter_to_apply = controlImageFilterField.value + } + } return newTask } @@ -1853,6 +1866,20 @@ function onFixFaceModelChange() { gfpganModelField.addEventListener("change", onFixFaceModelChange) onFixFaceModelChange() +function onControlnetModelChange() { + let configBox = document.querySelector("#controlnet_config") + if (IMAGE_REGEX.test(controlImagePreview.src)) { + configBox.classList.remove("displayNone") + controlImageContainer.classList.remove("displayNone") + } else { + configBox.classList.add("displayNone") + controlImageContainer.classList.add("displayNone") + } +} +controlImagePreview.addEventListener("load", onControlnetModelChange) +controlImagePreview.addEventListener("unload", onControlnetModelChange) +onControlnetModelChange() + upscaleModelField.disabled = !useUpscalingField.checked upscaleAmountField.disabled = !useUpscalingField.checked useUpscalingField.addEventListener("change", function(e) { @@ -2143,6 +2170,44 @@ promptsFromFileBtn.addEventListener("click", function() { promptsFromFileSelector.click() }) +function loadControlnetImageFromFile() { + if (controlImageSelector.files.length === 0) { + return + } + + let reader = new FileReader() + let file = controlImageSelector.files[0] + + reader.addEventListener("load", function(event) { + controlImagePreview.src = reader.result + }) + + if (file) { + reader.readAsDataURL(file) + } +} +controlImageSelector.addEventListener("change", loadControlnetImageFromFile) + +function controlImageLoad() { + let w = controlImagePreview.naturalWidth + let h = controlImagePreview.naturalHeight + addImageSizeOption(w) + addImageSizeOption(h) + + widthField.value = w + heightField.value = h + widthField.dispatchEvent(new Event("change")) + heightField.dispatchEvent(new Event("change")) +} +controlImagePreview.addEventListener("load", controlImageLoad) + +function controlImageUnload() { + controlImageSelector.value = null + controlImagePreview.src = "" + controlImagePreview.dispatchEvent(new Event("unload")) +} +controlImageClearBtn.addEventListener("click", controlImageUnload) + promptsFromFileSelector.addEventListener("change", async function() { if (promptsFromFileSelector.files.length === 0) { return diff --git a/ui/media/js/searchable-models.js b/ui/media/js/searchable-models.js index 85b9bbe9..174faf77 100644 --- a/ui/media/js/searchable-models.js +++ b/ui/media/js/searchable-models.js @@ -667,4 +667,7 @@ async function getModels(scanForMalicious = true) { } // reload models button -document.querySelector("#reload-models").addEventListener("click", () => getModels()) +document.querySelector("#reload-models").addEventListener("click", (e) => { + e.stopPropagation() + getModels() +}) From eba7bab15ef9d24da354f624695eb11c0d50cb01 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 16:16:38 +0530 Subject: [PATCH 6/9] Allow named models in the dropdown --- ui/easydiffusion/model_manager.py | 20 ++++++++++++++------ ui/media/js/searchable-models.js | 28 +++++++++------------------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/ui/easydiffusion/model_manager.py b/ui/easydiffusion/model_manager.py index 2e7fef67..7a4ddde3 100644 --- a/ui/easydiffusion/model_manager.py +++ b/ui/easydiffusion/model_manager.py @@ -290,11 +290,11 @@ def is_malicious_model(file_path): def getModels(scan_for_malicious: bool = True): models = { "options": { - "stable-diffusion": ["sd-v1-4"], + "stable-diffusion": [{"sd-v1-4": "SD 1.4"}], "vae": [], "hypernetwork": [], "lora": [], - "codeformer": ["codeformer"], + "codeformer": [{"codeformer": "CodeFormer"}], "embeddings": [], "controlnet": [], }, @@ -305,9 +305,9 @@ def getModels(scan_for_malicious: bool = True): class MaliciousModelException(Exception): "Raised when picklescan reports a problem with a model" - def scan_directory(directory, suffixes, directoriesFirst: bool = True): + def scan_directory(directory, suffixes, directoriesFirst: bool = True, default_entries=[]): + tree = list(default_entries) nonlocal models_scanned - tree = [] for entry in sorted( os.scandir(directory), key=lambda entry: (entry.is_file() == directoriesFirst, entry.name.lower()), @@ -326,7 +326,14 @@ def getModels(scan_for_malicious: bool = True): raise MaliciousModelException(entry.path) if scan_for_malicious: known_models[entry.path] = mtime - tree.append(entry.name[: -len(matching_suffix)]) + model_id = entry.name[: -len(matching_suffix)] + model_exists = False + for m in tree: # allows default "named" models, like CodeFormer and known ControlNet models + if (isinstance(m, str) and model_id == m) or (isinstance(m, dict) and model_id in m): + model_exists = True + break + if not model_exists: + tree.append(model_id) elif entry.is_dir(): scan = scan_directory(entry.path, suffixes, directoriesFirst=False) @@ -343,7 +350,8 @@ def getModels(scan_for_malicious: bool = True): os.makedirs(models_dir) try: - models["options"][model_type] = scan_directory(models_dir, model_extensions) + default_tree = models["options"].get(model_type, []) + models["options"][model_type] = scan_directory(models_dir, model_extensions, default_entries=default_tree) except MaliciousModelException as e: models["scan-error"] = str(e) diff --git a/ui/media/js/searchable-models.js b/ui/media/js/searchable-models.js index 174faf77..299c60dc 100644 --- a/ui/media/js/searchable-models.js +++ b/ui/media/js/searchable-models.js @@ -552,17 +552,23 @@ class ModelDropdown { this.createModelNodeList(`${folderName || ""}/${childFolderName}`, childModels, false) ) } else { + let modelId = model + let modelName = model + if (typeof model === "object") { + modelId = Object.keys(model)[0] + modelName = model[modelId] + } const classes = ["model-file"] if (isRootFolder) { classes.push("in-root-folder") } // Remove the leading slash from the model path - const fullPath = folderName ? `${folderName.substring(1)}/${model}` : model + const fullPath = folderName ? `${folderName.substring(1)}/${modelId}` : modelId modelsMap.set( - model, + modelId, createElement("li", { "data-path": fullPath }, classes, [ createElement("i", undefined, ["fa-regular", "fa-file", "icon"]), - model, + modelName, ]) ) } @@ -643,22 +649,6 @@ async function getModels(scanForMalicious = true) { makeImageBtn.disabled = true } - /* This code should no longer be needed. Commenting out for now, will cleanup later. - const sd_model_setting_key = "stable_diffusion_model" - const vae_model_setting_key = "vae_model" - const hypernetwork_model_key = "hypernetwork_model" - - const stableDiffusionOptions = modelsOptions['stable-diffusion'] - const vaeOptions = modelsOptions['vae'] - const hypernetworkOptions = modelsOptions['hypernetwork'] - - // TODO: set default for model here too - SETTINGS[sd_model_setting_key].default = stableDiffusionOptions[0] - if (getSetting(sd_model_setting_key) == '' || SETTINGS[sd_model_setting_key].value == '') { - setSetting(sd_model_setting_key, stableDiffusionOptions[0]) - } - */ - // notify ModelDropdown objects to refresh document.dispatchEvent(new Event("refreshModels")) } catch (e) { From ae34c9e84b3ce49f8b9e16e9ba5965cbdf380262 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 17:39:04 +0530 Subject: [PATCH 7/9] Download known controlnet models if selected; Auto-pick the recommended controlnet model when a filter is selected --- scripts/check_modules.py | 2 +- ui/easydiffusion/model_manager.py | 27 +++++++++++++++++++++---- ui/index.html | 4 +++- ui/media/js/main.js | 33 ++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/scripts/check_modules.py b/scripts/check_modules.py index 6bf3b1b3..bc0f46e7 100644 --- a/scripts/check_modules.py +++ b/scripts/check_modules.py @@ -18,7 +18,7 @@ os_name = platform.system() modules_to_check = { "torch": ("1.11.0", "1.13.1", "2.0.0"), "torchvision": ("0.12.0", "0.14.1", "0.15.1"), - "sdkit": "1.0.154", + "sdkit": "1.0.155", "stable-diffusion-sdkit": "2.1.4", "rich": "12.6.0", "uvicorn": "0.19.0", diff --git a/ui/easydiffusion/model_manager.py b/ui/easydiffusion/model_manager.py index 7a4ddde3..63f79859 100644 --- a/ui/easydiffusion/model_manager.py +++ b/ui/easydiffusion/model_manager.py @@ -186,6 +186,12 @@ def resolve_model_paths(models_data: ModelsData): continue if model_type == "codeformer": download_if_necessary("codeformer", "codeformer.pth", "codeformer-0.1.0") + elif model_type == "controlnet": + model_id = model_paths[model_type] + model_info = get_model_info_from_db(model_type=model_type, model_id=model_id) + if model_info: + filename = model_info.get("url", "").split("/")[-1] + download_if_necessary("controlnet", filename, model_id, skip_if_others_exist=False) model_paths[model_type] = resolve_model_to_use(model_paths[model_type], model_type=model_type) @@ -209,17 +215,17 @@ def download_default_models_if_necessary(): print(model_type, "model(s) found.") -def download_if_necessary(model_type: str, file_name: str, model_id: str): +def download_if_necessary(model_type: str, file_name: str, model_id: str, skip_if_others_exist=True): model_path = os.path.join(app.MODELS_DIR, model_type, file_name) expected_hash = get_model_info_from_db(model_type=model_type, model_id=model_id)["quick_hash"] - other_models_exist = any_model_exists(model_type) + other_models_exist = any_model_exists(model_type) and skip_if_others_exist known_model_exists = os.path.exists(model_path) known_model_is_corrupt = known_model_exists and hash_file_quick(model_path) != expected_hash if known_model_is_corrupt or (not other_models_exist and not known_model_exists): print("> download", model_type, model_id) - download_model(model_type, model_id, download_base_dir=app.MODELS_DIR) + download_model(model_type, model_id, download_base_dir=app.MODELS_DIR, download_config_if_available=False) def migrate_legacy_model_location(): @@ -296,7 +302,20 @@ def getModels(scan_for_malicious: bool = True): "lora": [], "codeformer": [{"codeformer": "CodeFormer"}], "embeddings": [], - "controlnet": [], + "controlnet": [ + {"control_v11p_sd15_canny": "Canny (*)"}, + {"control_v11p_sd15_openpose": "OpenPose (*)"}, + {"control_v11p_sd15_normalbae": "Normal BAE (*)"}, + {"control_v11f1p_sd15_depth": "Depth (*)"}, + {"control_v11p_sd15_scribble": "Scribble"}, + {"control_v11p_sd15_softedge": "Soft Edge"}, + {"control_v11p_sd15_inpaint": "Inpaint"}, + {"control_v11p_sd15_lineart": "Line Art"}, + {"control_v11p_sd15s2_lineart_anime": "Line Art Anime"}, + {"control_v11p_sd15_mlsd": "Straight Lines"}, + {"control_v11p_sd15_seg": "Segment"}, + {"control_v11e_sd15_shuffle": "Shuffle"}, + ], }, } diff --git a/ui/index.html b/ui/index.html index 68616ed4..29a147ff 100644 --- a/ui/index.html +++ b/ui/index.html @@ -151,7 +151,7 @@
- +
diff --git a/ui/media/js/main.js b/ui/media/js/main.js index 97ce807a..206d1338 100644 --- a/ui/media/js/main.js +++ b/ui/media/js/main.js @@ -119,7 +119,7 @@ let codeformerFidelityField = document.querySelector("#codeformer_fidelity") let stableDiffusionModelField = new ModelDropdown(document.querySelector("#stable_diffusion_model"), "stable-diffusion") let clipSkipField = document.querySelector("#clip_skip") let tilingField = document.querySelector("#tiling") -let controlnetModelField = new ModelDropdown(document.querySelector("#controlnet_model"), "controlnet", "None") +let controlnetModelField = new ModelDropdown(document.querySelector("#controlnet_model"), "controlnet", "None", false) let vaeModelField = new ModelDropdown(document.querySelector("#vae_model"), "vae", "None") let hypernetworkModelField = new ModelDropdown(document.querySelector("#hypernetwork_model"), "hypernetwork", "None") let hypernetworkStrengthSlider = document.querySelector("#hypernetwork_strength_slider") @@ -1880,6 +1880,37 @@ controlImagePreview.addEventListener("load", onControlnetModelChange) controlImagePreview.addEventListener("unload", onControlnetModelChange) onControlnetModelChange() +function onControlImageFilterChange() { + let filterId = controlImageFilterField.value + if (filterId.includes("openpose")) { + controlnetModelField.value = "control_v11p_sd15_openpose" + } else if (filterId === "canny") { + controlnetModelField.value = "control_v11p_sd15_canny" + } else if (filterId === "mlsd") { + controlnetModelField.value = "control_v11p_sd15_mlsd" + } else if (filterId === "mlsd") { + controlnetModelField.value = "control_v11p_sd15_mlsd" + } else if (filterId.includes("scribble")) { + controlnetModelField.value = "control_v11p_sd15_scribble" + } else if (filterId.includes("softedge")) { + controlnetModelField.value = "control_v11p_sd15_softedge" + } else if (filterId === "normal_bae") { + controlnetModelField.value = "control_v11p_sd15_normalbae" + } else if (filterId.includes("depth")) { + controlnetModelField.value = "control_v11f1p_sd15_depth" + } else if (filterId === "lineart_anime") { + controlnetModelField.value = "control_v11p_sd15s2_lineart_anime" + } else if (filterId.includes("lineart")) { + controlnetModelField.value = "control_v11p_sd15_lineart" + } else if (filterId === "shuffle") { + controlnetModelField.value = "control_v11e_sd15_shuffle" + } else if (filterId === "segment") { + controlnetModelField.value = "control_v11p_sd15_seg" + } +} +controlImageFilterField.addEventListener("change", onControlImageFilterChange) +onControlImageFilterChange() + upscaleModelField.disabled = !useUpscalingField.checked upscaleAmountField.disabled = !useUpscalingField.checked useUpscalingField.addEventListener("change", function(e) { From 3d9a9299dc75b9ce0b69a6813781205bec2f2772 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 17:42:15 +0530 Subject: [PATCH 8/9] changelog --- CHANGES.md | 1 + ui/index.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1ac36410..141c0773 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ Our focus continues to remain on an easy installation experience, and an easy user-interface. While still remaining pretty powerful, in terms of features and speed. ### Detailed changelog +* 2.5.48 - 1 Aug 2023 - (beta-only) Full support for ControlNets. You can select a control image to guide the AI. You can pick a filter to pre-process the image, and one of the known (or custom) controlnet models. Supports `OpenPose`, `Canny`, `Straight Lines`, `Depth`, `Line Art`, `Scribble`, `Soft Edge`, `Shuffle` and `Segment`. * 2.5.47 - 30 Jul 2023 - An option to use `Strict Mask Border` while inpainting, to avoid touching areas outside the mask. But this might show a slight outline of the mask, which you will have to touch up separately. * 2.5.47 - 29 Jul 2023 - (beta-only) Fix long prompts with SDXL. * 2.5.47 - 29 Jul 2023 - (beta-only) Fix red dots in some SDXL images. diff --git a/ui/index.html b/ui/index.html index 29a147ff..8bc2a0ba 100644 --- a/ui/index.html +++ b/ui/index.html @@ -32,7 +32,7 @@

Easy Diffusion - v2.5.47 + v2.5.48

From 4e444b418e0d283f5393520b667be15506722789 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 1 Aug 2023 18:23:34 +0530 Subject: [PATCH 9/9] sdkit 1.0.156 - missing jsons for controlnet --- scripts/check_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_modules.py b/scripts/check_modules.py index bc0f46e7..51ecf3e7 100644 --- a/scripts/check_modules.py +++ b/scripts/check_modules.py @@ -18,7 +18,7 @@ os_name = platform.system() modules_to_check = { "torch": ("1.11.0", "1.13.1", "2.0.0"), "torchvision": ("0.12.0", "0.14.1", "0.15.1"), - "sdkit": "1.0.155", + "sdkit": "1.0.156", "stable-diffusion-sdkit": "2.1.4", "rich": "12.6.0", "uvicorn": "0.19.0",