diff --git a/ui/sd_internal/task_manager.py b/ui/sd_internal/task_manager.py index e984c6a9..eae17891 100644 --- a/ui/sd_internal/task_manager.py +++ b/ui/sd_internal/task_manager.py @@ -204,13 +204,13 @@ def preload_model(ckpt_file_path=None, vae_file_path=None): print(traceback.format_exc()) def thread_get_next_task(): + from . import runtime if not manager_lock.acquire(blocking=True, timeout=LOCK_TIMEOUT): print('Render thread on device', runtime.thread_data.device, 'failed to acquire manager lock.') return None if len(tasks_queue) <= 0: manager_lock.release() return None - from . import runtime task = None try: # Select a render task. for queued_task in tasks_queue: @@ -375,14 +375,16 @@ def start_render_thread(device='auto'): rthread.daemon = True rthread.name = THREAD_NAME_PREFIX + device rthread.start() - timeout = DEVICE_START_TIMEOUT - while not rthread.is_alive() or not rthread in weak_thread_data or not 'device' in weak_thread_data[rthread]: - if timeout <= 0: raise Exception('render_thread', rthread.name, 'failed to start before timeout or has crashed.') - timeout -= 1 - time.sleep(1) render_threads.append(rthread) finally: manager_lock.release() + timeout = DEVICE_START_TIMEOUT + while not rthread.is_alive() or not rthread in weak_thread_data or not 'device' in weak_thread_data[rthread]: + if timeout <= 0: + return False + timeout -= 1 + time.sleep(1) + return True def shutdown_event(): # Signal render thread to close on shutdown global current_state_error diff --git a/ui/server.py b/ui/server.py index b9ea1b48..a0dc52ef 100644 --- a/ui/server.py +++ b/ui/server.py @@ -397,7 +397,8 @@ if 'render_devices' in config: # Start a new thread for each device. if not isinstance(config['render_devices'], list): raise Exception('Invalid render_devices value in config.') for device in config['render_devices']: - task_manager.start_render_thread(device) + if not task_manager.start_render_thread(device): + print(device, 'failed to start.') if task_manager.is_alive() <= 0: # No running devices, probably invalid user config. print('WARNING: No active render devices after loading config. Validate "render_devices" in config.json') print('Loading default render devices to replace invalid render_devices field from config', config['render_devices']) @@ -405,13 +406,20 @@ if 'render_devices' in config: # Start a new thread for each device. display_warning = False if task_manager.is_alive() <= 0: # Either no defauls or no devices after loading config. # Select best GPU device using free memory, if more than one device. - task_manager.start_render_thread('auto') # Detect best device for renders - if task_manager.is_alive(0) <= 0: # without cuda:0 - task_manager.start_render_thread('cuda') # An other cuda device is better and cuda:0 is missing, start it... - display_warning = True # And warn user to update settings... + if task_manager.start_render_thread('auto'): # Detect best device for renders + if task_manager.is_alive(0) <= 0: # has no cuda:0 + if task_manager.is_alive('cpu') >= 1: # auto used CPU. + pass # Maybe add warning here about CPU mode... + elif task_manager.start_render_thread('cuda'): # An other cuda device is better and cuda:0 is missing, try to start it... + display_warning = True # And warn user to update settings... + else: + print('Failed to start GPU:0...') + else: + print('Failed to start gpu device.') if task_manager.is_alive('cpu') <= 0: # Allow CPU to be used for renders - task_manager.start_render_thread('cpu') + if not task_manager.start_render_thread('cpu'): + print('Failed to start CPU render device...') if display_warning or task_manager.is_alive(0) <= 0: print('WARNING: GFPGANer only works on GPU:0, use CUDA_VISIBLE_DEVICES if GFPGANer is needed on a specific GPU.')