From 7f4786f9dd991dbe2da1c97f87866029f5f9f7c2 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Thu, 27 Oct 2022 20:27:21 -0400 Subject: [PATCH 1/4] Wait until device is fully ready before proceding. --- ui/sd_internal/task_manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/sd_internal/task_manager.py b/ui/sd_internal/task_manager.py index efc53c08..1d0535ab 100644 --- a/ui/sd_internal/task_manager.py +++ b/ui/sd_internal/task_manager.py @@ -20,6 +20,8 @@ ERR_LOCK_FAILED = ' failed to acquire lock within timeout.' LOCK_TIMEOUT = 15 # Maximum locking time in seconds before failing a task. # It's better to get an exception than a deadlock... ALWAYS use timeout in critical paths. +DEVICE_START_TIMEOUT = 60 # seconds - Maximum time to wait for a render device to init. + class SymbolClass(type): # Print nicely formatted Symbol names. def __repr__(self): return self.__qualname__ def __str__(self): return self.__name__ @@ -240,9 +242,6 @@ def thread_get_next_task(): def thread_render(device): global current_state, current_state_error, current_model_path from . import runtime - weak_thread_data[threading.current_thread()] = { - 'device': device - } try: runtime.device_init(device) except: @@ -367,8 +366,8 @@ def start_render_thread(device='auto'): rthread.daemon = True rthread.name = THREAD_NAME_PREFIX + device rthread.start() - timeout = LOCK_TIMEOUT - while not rthread.is_alive(): + timeout = DEVICE_START_TIMEOUT + while not rthread.is_alive() or not rthread in weak_thread_data or not 'device' in weak_thread_data[rthread]: if timeout <= 0: raise Exception('render_thread', rthread.name, 'failed to start before timeout or has crashed.') timeout -= 1 time.sleep(1) From fa4059a4b9adbb567208088c8bb4d612426785d9 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Thu, 27 Oct 2022 21:40:16 -0400 Subject: [PATCH 2/4] Removed all async code since now `start_render_thread` wait for init to complete making this useless. --- ui/server.py | 61 +++++++++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/ui/server.py b/ui/server.py index 2f428894..fae34dec 100644 --- a/ui/server.py +++ b/ui/server.py @@ -31,7 +31,6 @@ APP_CONFIG_DEFAULT_MODELS = [ 'sd-v1-4', # Default fallback. ] -import asyncio from fastapi import FastAPI, HTTPException from fastapi.staticfiles import StaticFiles from starlette.responses import FileResponse, JSONResponse, StreamingResponse @@ -42,7 +41,6 @@ from typing import Any, Generator, Hashable, List, Optional, Union from sd_internal import Request, Response, task_manager -LOOP = asyncio.get_event_loop() app = FastAPI() modifiers_cache = None @@ -362,54 +360,39 @@ logging.getLogger('uvicorn.access').addFilter(LogSuppressFilter()) config = getConfig() -async def check_status(): # Task to Validate user config shortly after startup. - # Check that the loaded config.json yielded a server in a known valid state. - # When issues are found, try to fix them when possible and warn the user. - device_count = 0 - # Wait for devices to register and/or change names. - THREAD_START_DELAY = 5 # seconds - Give time for devices/threads to start. - for i in range(10): # Maximum number of retry. - await asyncio.sleep(THREAD_START_DELAY) - new_count = task_manager.is_alive() - # Stops retry once no more devices show up. - if new_count > 0 and device_count == new_count: break - device_count = new_count - - if 'render_devices' in config and task_manager.is_alive() <= 0: # No running devices, probably invalid user config. Try to apply defaults. - print('WARNING: No active render devices after loading config. Validate "render_devices" in config.json') - task_manager.start_render_thread('auto') # Detect best device for renders - task_manager.start_render_thread('cpu') # Allow CPU to be used for renders - await asyncio.sleep(THREAD_START_DELAY) # delay message after thread start. - print('Default render devices loaded to replace missing render_devices', config['render_devices']) - - display_warning = False - if not 'render_devices' in config and task_manager.is_alive(0) <= 0: # No config set, is on auto mode and without cuda:0 - task_manager.start_render_thread('cuda') # An other cuda device is better and cuda:0 is missing, start it... - display_warning = True # And warn user to update settings... - await asyncio.sleep(THREAD_START_DELAY) # delay message after thread start. - - if display_warning or task_manager.is_alive(0) <= 0: - print('WARNING: GFPGANer only works on GPU:0, use CUDA_VISIBLE_DEVICES if GFPGANer is needed on a specific GPU.') - print('Using CUDA_VISIBLE_DEVICES will remap the selected devices starting at GPU:0 fixing GFPGANer') - print('Add the line "@set CUDA_VISIBLE_DEVICES=N" where N is the GPUs to use to config.bat') - print('Add the line "CUDA_VISIBLE_DEVICES=N" where N is the GPUs to use to config.sh') - # Start the task_manager task_manager.default_model_to_load = resolve_model_to_use() -if 'render_devices' in config: # Start a new thread for each device. +# Check that the loaded config.json yielded a server in a known valid state. +# When issues are found, try to fix them when possible and warn the user. +if 'render_devices' in config: # Start a new thread for each device. if isinstance(config['render_devices'], str): config['render_devices'] = config['render_devices'].split(',') if not isinstance(config['render_devices'], list): raise Exception('Invalid render_devices value in config.') for device in config['render_devices']: task_manager.start_render_thread(device) -else: + if task_manager.is_alive() <= 0: # No running devices, probably invalid user config. + print('WARNING: No active render devices after loading config. Validate "render_devices" in config.json') + print('Loading default render devices to replace invalid render_devices field from config', config['render_devices']) + +display_warning = False +if task_manager.is_alive() <= 0: # Either no defauls or no devices after loading config. # Select best GPU device using free memory, if more than one device. task_manager.start_render_thread('auto') # Detect best device for renders - task_manager.start_render_thread('cpu') # Allow CPU to be used for renders + if task_manager.is_alive(0) <= 0: # without cuda:0 + task_manager.start_render_thread('cuda') # An other cuda device is better and cuda:0 is missing, start it... + display_warning = True # And warn user to update settings... + if task_manager.is_alive('cpu') <= 0: + # Allow CPU to be used for renders + task_manager.start_render_thread('cpu') -# Task to Validate user config shortly after startup. -LOOP.create_task(check_status()) +if display_warning or task_manager.is_alive(0) <= 0: + print('WARNING: GFPGANer only works on GPU:0, use CUDA_VISIBLE_DEVICES if GFPGANer is needed on a specific GPU.') + print('Using CUDA_VISIBLE_DEVICES will remap the selected devices starting at GPU:0 fixing GFPGANer') + print('Add the line "@set CUDA_VISIBLE_DEVICES=N" where N is the GPUs to use to config.bat') + print('Add the line "CUDA_VISIBLE_DEVICES=N" where N is the GPUs to use to config.sh') + +del display_warning # start the browser ui import webbrowser; webbrowser.open('http://localhost:9000') \ No newline at end of file From 7dc7ba9977096b84fc646de184b22ec35a492cc1 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Thu, 27 Oct 2022 21:47:44 -0400 Subject: [PATCH 3/4] Removed old comments. --- ui/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/server.py b/ui/server.py index fae34dec..8befcd7b 100644 --- a/ui/server.py +++ b/ui/server.py @@ -362,8 +362,6 @@ config = getConfig() # Start the task_manager task_manager.default_model_to_load = resolve_model_to_use() -# Check that the loaded config.json yielded a server in a known valid state. -# When issues are found, try to fix them when possible and warn the user. if 'render_devices' in config: # Start a new thread for each device. if isinstance(config['render_devices'], str): config['render_devices'] = config['render_devices'].split(',') From 22a11769fa02d88ac692f5ca0f728ef944b9f2e7 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Thu, 27 Oct 2022 21:57:50 -0400 Subject: [PATCH 4/4] Enable preload on cpu when no other devices are alive. --- ui/sd_internal/task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/sd_internal/task_manager.py b/ui/sd_internal/task_manager.py index 1d0535ab..034e0f69 100644 --- a/ui/sd_internal/task_manager.py +++ b/ui/sd_internal/task_manager.py @@ -250,7 +250,7 @@ def thread_render(device): weak_thread_data[threading.current_thread()] = { 'device': runtime.thread_data.device } - if runtime.thread_data.device != 'cpu': + if runtime.thread_data.device != 'cpu' or is_alive() == 1: preload_model() current_state = ServerStates.Online while True: