From fc2a6567dadb7f01caa13886b1f9707b68f7ea9a Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Fri, 28 Oct 2022 22:51:04 -0400 Subject: [PATCH 1/5] Moved import before use of runtime.thread_data.device --- ui/sd_internal/task_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/sd_internal/task_manager.py b/ui/sd_internal/task_manager.py index e984c6a9..01d7bf0a 100644 --- a/ui/sd_internal/task_manager.py +++ b/ui/sd_internal/task_manager.py @@ -204,13 +204,13 @@ def preload_model(ckpt_file_path=None, vae_file_path=None): print(traceback.format_exc()) def thread_get_next_task(): + from . import runtime if not manager_lock.acquire(blocking=True, timeout=LOCK_TIMEOUT): print('Render thread on device', runtime.thread_data.device, 'failed to acquire manager lock.') return None if len(tasks_queue) <= 0: manager_lock.release() return None - from . import runtime task = None try: # Select a render task. for queued_task in tasks_queue: From 86da27a7a1cd908a70c9922d15f1f07d2647d110 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Fri, 28 Oct 2022 22:52:00 -0400 Subject: [PATCH 2/5] Moved wait outside lock and now returns false on failure. --- ui/sd_internal/task_manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ui/sd_internal/task_manager.py b/ui/sd_internal/task_manager.py index 01d7bf0a..eae17891 100644 --- a/ui/sd_internal/task_manager.py +++ b/ui/sd_internal/task_manager.py @@ -375,14 +375,16 @@ def start_render_thread(device='auto'): rthread.daemon = True rthread.name = THREAD_NAME_PREFIX + device rthread.start() - timeout = DEVICE_START_TIMEOUT - while not rthread.is_alive() or not rthread in weak_thread_data or not 'device' in weak_thread_data[rthread]: - if timeout <= 0: raise Exception('render_thread', rthread.name, 'failed to start before timeout or has crashed.') - timeout -= 1 - time.sleep(1) render_threads.append(rthread) finally: manager_lock.release() + timeout = DEVICE_START_TIMEOUT + while not rthread.is_alive() or not rthread in weak_thread_data or not 'device' in weak_thread_data[rthread]: + if timeout <= 0: + return False + timeout -= 1 + time.sleep(1) + return True def shutdown_event(): # Signal render thread to close on shutdown global current_state_error From 4675da4d167cf0eade5e7a8116128e7cc28a4f53 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Fri, 28 Oct 2022 22:53:55 -0400 Subject: [PATCH 3/5] Display warning on start failure. Removes spam from exception and continue starting other devices. --- ui/server.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ui/server.py b/ui/server.py index b9ea1b48..a38a55c8 100644 --- a/ui/server.py +++ b/ui/server.py @@ -397,7 +397,8 @@ if 'render_devices' in config: # Start a new thread for each device. if not isinstance(config['render_devices'], list): raise Exception('Invalid render_devices value in config.') for device in config['render_devices']: - task_manager.start_render_thread(device) + if not task_manager.start_render_thread(device): + print(device, 'failed to start.') if task_manager.is_alive() <= 0: # No running devices, probably invalid user config. print('WARNING: No active render devices after loading config. Validate "render_devices" in config.json') print('Loading default render devices to replace invalid render_devices field from config', config['render_devices']) @@ -405,13 +406,18 @@ if 'render_devices' in config: # Start a new thread for each device. display_warning = False if task_manager.is_alive() <= 0: # Either no defauls or no devices after loading config. # Select best GPU device using free memory, if more than one device. - task_manager.start_render_thread('auto') # Detect best device for renders - if task_manager.is_alive(0) <= 0: # without cuda:0 - task_manager.start_render_thread('cuda') # An other cuda device is better and cuda:0 is missing, start it... - display_warning = True # And warn user to update settings... + if task_manager.start_render_thread('auto'): # Detect best device for renders + if task_manager.is_alive(0) <= 0: # is not cuda:0 + if task_manager.start_render_thread('cuda'): # An other cuda device is better and cuda:0 is missing, start it... + display_warning = True # And warn user to update settings... + else: + print('Failed to start GPU:0...') + else: + print('Failed to start gpu device.') if task_manager.is_alive('cpu') <= 0: # Allow CPU to be used for renders - task_manager.start_render_thread('cpu') + if not task_manager.start_render_thread('cpu'): + print('Failed to start CPU render device...') if display_warning or task_manager.is_alive(0) <= 0: print('WARNING: GFPGANer only works on GPU:0, use CUDA_VISIBLE_DEVICES if GFPGANer is needed on a specific GPU.') From 902ccbd203018691608dc4828a5fba45ee62d774 Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Sat, 29 Oct 2022 00:36:26 -0400 Subject: [PATCH 4/5] Don't try to start cuda:0 if auto used cpu mode. --- ui/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/server.py b/ui/server.py index a38a55c8..10c2a71c 100644 --- a/ui/server.py +++ b/ui/server.py @@ -407,7 +407,7 @@ display_warning = False if task_manager.is_alive() <= 0: # Either no defauls or no devices after loading config. # Select best GPU device using free memory, if more than one device. if task_manager.start_render_thread('auto'): # Detect best device for renders - if task_manager.is_alive(0) <= 0: # is not cuda:0 + if task_manager.is_alive(0) <= 0 and task_manager.is_alive('cpu') <= 0: # is not cuda:0 and auto didnt use CPU. if task_manager.start_render_thread('cuda'): # An other cuda device is better and cuda:0 is missing, start it... display_warning = True # And warn user to update settings... else: From 9f5202fee33e64afd34d15707d6b584aeca75c2f Mon Sep 17 00:00:00 2001 From: Marc-Andre Ferland Date: Sat, 29 Oct 2022 00:43:02 -0400 Subject: [PATCH 5/5] Improved readability and comments. --- ui/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ui/server.py b/ui/server.py index 10c2a71c..a0dc52ef 100644 --- a/ui/server.py +++ b/ui/server.py @@ -407,8 +407,10 @@ display_warning = False if task_manager.is_alive() <= 0: # Either no defauls or no devices after loading config. # Select best GPU device using free memory, if more than one device. if task_manager.start_render_thread('auto'): # Detect best device for renders - if task_manager.is_alive(0) <= 0 and task_manager.is_alive('cpu') <= 0: # is not cuda:0 and auto didnt use CPU. - if task_manager.start_render_thread('cuda'): # An other cuda device is better and cuda:0 is missing, start it... + if task_manager.is_alive(0) <= 0: # has no cuda:0 + if task_manager.is_alive('cpu') >= 1: # auto used CPU. + pass # Maybe add warning here about CPU mode... + elif task_manager.start_render_thread('cuda'): # An other cuda device is better and cuda:0 is missing, try to start it... display_warning = True # And warn user to update settings... else: print('Failed to start GPU:0...')