From 80384e6ee1b370dd51324c2385c60d651c12b2e5 Mon Sep 17 00:00:00 2001 From: cmdr2 Date: Tue, 18 Apr 2023 15:42:33 +0530 Subject: [PATCH] Install PyTorch 2.0 by default, but allow existing PyTorch 1.13.1 installations to continue running; Unify and streamline the installation of dependencies --- scripts/check_modules.py | 126 +++++++++++++++++++++++++++++++++------ scripts/on_sd_start.bat | 103 +++----------------------------- scripts/on_sd_start.sh | 110 ++-------------------------------- 3 files changed, 120 insertions(+), 219 deletions(-) diff --git a/scripts/check_modules.py b/scripts/check_modules.py index 991acee7..7365000d 100644 --- a/scripts/check_modules.py +++ b/scripts/check_modules.py @@ -1,23 +1,111 @@ """ -This script checks if the given modules exist - -E.g. python check_modules.py sdkit==1.0.3 sdkit.models ldm transformers numpy antlr4 gfpgan realesrgan +This script checks and installs the required modules. """ -import sys -import pkgutil -from importlib.metadata import version +import os +from importlib.metadata import version as pkg_version +import platform -modules = sys.argv[1:] -missing_modules = [] -for m in modules: - m = m.split("==") - module_name = m[0] - module_version = m[1] if len(m) > 1 else None - is_installed = pkgutil.find_loader(module_name) is not None - if not is_installed: - print("module", module_name, "not found") - exit(1) - elif module_version and version(module_name) != module_version: - print("module version is different! expected: ", module_version, ", actual: ", version(module_name)) - exit(1) +os_name = platform.system() + +modules_to_check = { + "torch": ("1.13.1", "2.0.0"), + "torchvision": ("0.14.1", "0.15.1"), + "sdkit": "1.0.72", + "stable-diffusion-sdkit": "2.1.4", + "rich": "12.6.0", + "uvicorn": "0.19.0", + "fastapi": "0.85.1", +} + + +def version(module_name: str) -> str: + try: + return pkg_version(module_name) + except: + return None + + +def install(module_name: str, module_version: str): + index_url = None + if module_name in ("torch", "torchvision"): + module_version, index_url = apply_torch_install_overrides(module_version) + + install_cmd = f"python -m pip install --upgrade {module_name}=={module_version}" + if index_url: + install_cmd += f" --index-url {index_url}" + if module_name == "sdkit": + install_cmd += " -q" + + print(">", install_cmd) + os.system(install_cmd) + + +def init(): + for module_name, allowed_versions in modules_to_check.items(): + if os.path.exists(f"../src/{module_name}"): + print(f"Skipping {module_name} update, since it's in developer/editable mode") + continue + + allowed_versions, latest_version = get_allowed_versions(module_name, allowed_versions) + if version(module_name) not in allowed_versions: + try: + install(module_name, latest_version) + except: + fail(module_name) + + print(f"{module_name}: {version(module_name)}") + + +### utilities + + +def get_allowed_versions(module_name: str, allowed_versions: tuple[str]): + allowed_versions = (allowed_versions,) if isinstance(allowed_versions, str) else allowed_versions + latest_version = allowed_versions[-1] + + if module_name in ("torch", "torchvision"): + allowed_versions = include_cuda_versions(allowed_versions) + + return allowed_versions, latest_version + + +def apply_torch_install_overrides(module_version: str): + index_url = None + if os_name == "Windows": + module_version += "+cu117" + index_url = "https://download.pytorch.org/whl/cu117" + elif os_name == "Linux": + with open("/proc/bus/pci/devices", "r") as f: + device_info = f.read() + if "amdgpu" in device_info: + index_url = "https://download.pytorch.org/whl/rocm5.4.2" + + return module_version, index_url + + +def include_cuda_versions(module_versions: tuple) -> tuple: + "Adds CUDA-specific versions to the list of allowed version numbers" + + allowed_versions = tuple(module_versions) + allowed_versions += tuple(f"{v}+cu116" for v in module_versions) + allowed_versions += tuple(f"{v}+cu117" for v in module_versions) + + return allowed_versions + + +def fail(module_name): + print( + f"""Error installing {module_name}. Sorry about that, please try to: +1. Run this installer again. +2. If that doesn't fix it, please try the common troubleshooting steps at https://github.com/cmdr2/stable-diffusion-ui/wiki/Troubleshooting +3. If those steps don't help, please copy *all* the error messages in this window, and ask the community at https://discord.com/invite/u9yhsFmEkB +4. If that doesn't solve the problem, please file an issue at https://github.com/cmdr2/stable-diffusion-ui/issues +Thanks!""" + ) + exit(1) + + +### start + +init() diff --git a/scripts/on_sd_start.bat b/scripts/on_sd_start.bat index 52bf3330..0b74e9f0 100644 --- a/scripts/on_sd_start.bat +++ b/scripts/on_sd_start.bat @@ -7,12 +7,6 @@ @copy sd-ui-files\scripts\bootstrap.bat scripts\ /Y @copy sd-ui-files\scripts\check_modules.py scripts\ /Y -set TORCH_VERSION=2.0.0+cu117 -set TORCHVISION_VERSION=0.15.1+cu117 -set TORCH_INDEX_URL=https://download.pytorch.org/whl/cu117 -set SDKIT_VERSION=1.0.72 -set SD_VERSION=2.1.4 - if exist "%cd%\profile" ( set USERPROFILE=%cd%\profile ) @@ -70,98 +64,17 @@ if exist "RealESRGAN_x4plus_anime_6B.pth" move RealESRGAN_x4plus_anime_6B.pth .. if not exist "%INSTALL_ENV_DIR%\DLLs\libssl-1_1-x64.dll" copy "%INSTALL_ENV_DIR%\Library\bin\libssl-1_1-x64.dll" "%INSTALL_ENV_DIR%\DLLs\" if not exist "%INSTALL_ENV_DIR%\DLLs\libcrypto-1_1-x64.dll" copy "%INSTALL_ENV_DIR%\Library\bin\libcrypto-1_1-x64.dll" "%INSTALL_ENV_DIR%\DLLs\" -@rem install torch and torchvision -call python ..\scripts\check_modules.py torch==%TORCH_VERSION% torchvision==%TORCHVISION_VERSION% -if "%ERRORLEVEL%" EQU "0" ( - echo "torch and torchvision have already been installed." -) else ( - echo "Installing torch and torchvision.." - - @REM prevent from using packages from the user's home directory, to avoid conflicts - set PYTHONNOUSERSITE=1 - set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - - call python -m pip install --upgrade torch==%TORCH_VERSION% torchvision==%TORCHVISION_VERSION% --index-url %TORCH_INDEX_URL% || ( - echo "Error installing torch. Sorry about that, please try to:" & echo " 1. Run this installer again." & echo " 2. If that doesn't fix it, please try the common troubleshooting steps at https://github.com/cmdr2/stable-diffusion-ui/wiki/Troubleshooting" & echo " 3. If those steps don't help, please copy *all* the error messages in this window, and ask the community at https://discord.com/invite/u9yhsFmEkB" & echo " 4. If that doesn't solve the problem, please file an issue at https://github.com/cmdr2/stable-diffusion-ui/issues" & echo "Thanks!" - pause - exit /b - ) -) - -call python -c "from importlib.metadata import version; print('torch version:', version('torch'))" - +@rem install or upgrade the required modules set PATH=C:\Windows\System32;%PATH% -@rem install/upgrade sdkit -call python ..\scripts\check_modules.py sdkit sdkit.models ldm transformers numpy antlr4 gfpgan realesrgan -if "%ERRORLEVEL%" EQU "0" ( - echo "sdkit is already installed." +@REM prevent from using packages from the user's home directory, to avoid conflicts +set PYTHONNOUSERSITE=1 +set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - @rem skip sdkit upgrade if in developer-mode - if not exist "..\src\sdkit" ( - @REM prevent from using packages from the user's home directory, to avoid conflicts - set PYTHONNOUSERSITE=1 - set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - - call python -m pip install --upgrade sdkit==%SDKIT_VERSION% -q || ( - echo "Error updating sdkit" - ) - ) -) else ( - echo "Installing sdkit: https://pypi.org/project/sdkit/" - - @REM prevent from using packages from the user's home directory, to avoid conflicts - set PYTHONNOUSERSITE=1 - set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - - call python -m pip install sdkit==%SDKIT_VERSION% || ( - echo "Error installing sdkit. Sorry about that, please try to:" & echo " 1. Run this installer again." & echo " 2. If that doesn't fix it, please try the common troubleshooting steps at https://github.com/cmdr2/stable-diffusion-ui/wiki/Troubleshooting" & echo " 3. If those steps don't help, please copy *all* the error messages in this window, and ask the community at https://discord.com/invite/u9yhsFmEkB" & echo " 4. If that doesn't solve the problem, please file an issue at https://github.com/cmdr2/stable-diffusion-ui/issues" & echo "Thanks!" - pause - exit /b - ) -) - -call python -c "from importlib.metadata import version; print('sdkit version:', version('sdkit'))" - -@rem upgrade stable-diffusion-sdkit -call python -m pip install --upgrade stable-diffusion-sdkit==%SD_VERSION% -q || ( - echo "Error updating stable-diffusion-sdkit" -) -call python -c "from importlib.metadata import version; print('stable-diffusion version:', version('stable-diffusion-sdkit'))" - -@rem install rich -call python ..\scripts\check_modules.py rich -if "%ERRORLEVEL%" EQU "0" ( - echo "rich has already been installed." -) else ( - echo "Installing rich.." - - set PYTHONNOUSERSITE=1 - set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - - call python -m pip install rich || ( - echo "Error installing rich. Sorry about that, please try to:" & echo " 1. Run this installer again." & echo " 2. If that doesn't fix it, please try the common troubleshooting steps at https://github.com/cmdr2/stable-diffusion-ui/wiki/Troubleshooting" & echo " 3. If those steps don't help, please copy *all* the error messages in this window, and ask the community at https://discord.com/invite/u9yhsFmEkB" & echo " 4. If that doesn't solve the problem, please file an issue at https://github.com/cmdr2/stable-diffusion-ui/issues" & echo "Thanks!" - pause - exit /b - ) -) - -set PATH=C:\Windows\System32;%PATH% - -call python ..\scripts\check_modules.py uvicorn fastapi -@if "%ERRORLEVEL%" EQU "0" ( - echo "Packages necessary for Easy Diffusion were already installed" -) else ( - @echo. & echo "Downloading packages necessary for Easy Diffusion..." & echo. - - set PYTHONNOUSERSITE=1 - set PYTHONPATH=%INSTALL_ENV_DIR%\lib\site-packages - - @call conda install -c conda-forge -y uvicorn fastapi || ( - echo "Error installing the packages necessary for Easy Diffusion. Sorry about that, please try to:" & echo " 1. Run this installer again." & echo " 2. If that doesn't fix it, please try the common troubleshooting steps at https://github.com/cmdr2/stable-diffusion-ui/wiki/Troubleshooting" & echo " 3. If those steps don't help, please copy *all* the error messages in this window, and ask the community at https://discord.com/invite/u9yhsFmEkB" & echo " 4. If that doesn't solve the problem, please file an issue at https://github.com/cmdr2/stable-diffusion-ui/issues" & echo "Thanks!" - pause - exit /b - ) +call python ..\scripts\check_modules.py +if "%ERRORLEVEL%" NEQ "0" ( + pause + exit /b ) call WHERE uvicorn > .tmp diff --git a/scripts/on_sd_start.sh b/scripts/on_sd_start.sh index 275dfc1a..fbf96de0 100755 --- a/scripts/on_sd_start.sh +++ b/scripts/on_sd_start.sh @@ -7,11 +7,6 @@ cp sd-ui-files/scripts/check_modules.py scripts/ source ./scripts/functions.sh -TORCH_VERSION="2.0.0" -TORCHVISION_VERSION="0.15.1" -SDKIT_VERSION="1.0.72" -SD_VERSION="2.1.4" - # activate the installer env CONDA_BASEPATH=$(conda info --base) source "$CONDA_BASEPATH/etc/profile.d/conda.sh" # avoids the 'shell not initialized' error @@ -61,108 +56,13 @@ if [ -e "GFPGANv1.3.pth" ]; then mv GFPGANv1.3.pth ../models/gfpgan/; fi if [ -e "RealESRGAN_x4plus.pth" ]; then mv RealESRGAN_x4plus.pth ../models/realesrgan/; fi if [ -e "RealESRGAN_x4plus_anime_6B.pth" ]; then mv RealESRGAN_x4plus_anime_6B.pth ../models/realesrgan/; fi -OS_NAME=$(uname -s) -case "${OS_NAME}" in - Linux*) OS_NAME="linux";; - Darwin*) OS_NAME="macos";; - *) echo "Unknown OS: $OS_NAME! This script runs only on Linux or Mac" && exit -esac - -# Detect GPU types - -if grep -q amdgpu /proc/bus/pci/devices; then - echo AMD GPU detected - HAS_AMD=yes +if ! python ../scripts/check_modules.py; then + read -p "Press any key to continue" + exit 1 fi -if grep -q nvidia /proc/bus/pci/devices; then - echo NVidia GPU detected - HAS_NVIDIA=yes -fi - - - -# install torch and torchvision -if python ../scripts/check_modules.py torch==$TORCH_VERSION torchvision==$TORCHVISION_VERSION; then - echo "torch and torchvision have already been installed." -else - echo "Installing torch and torchvision.." - - export PYTHONNOUSERSITE=1 - export PYTHONPATH="$INSTALL_ENV_DIR/lib/python3.8/site-packages" - - if python -m pip install --upgrade torch==$TORCH_VERSION torchvision==$TORCHVISION_VERSION ; then - echo "Installed." - else - fail "torch install failed" - fi -fi - -python -c "from importlib.metadata import version; print('torch version:', version('torch'))" - -# install/upgrade sdkit -if python ../scripts/check_modules.py sdkit sdkit.models ldm transformers numpy antlr4 gfpgan realesrgan ; then - echo "sdkit is already installed." - - # skip sdkit upgrade if in developer-mode - if [ ! -e "../src/sdkit" ]; then - export PYTHONNOUSERSITE=1 - export PYTHONPATH="$INSTALL_ENV_DIR/lib/python3.8/site-packages" - - python -m pip install --upgrade sdkit==$SDKIT_VERSION -q - fi -else - echo "Installing sdkit: https://pypi.org/project/sdkit/" - - export PYTHONNOUSERSITE=1 - export PYTHONPATH="$INSTALL_ENV_DIR/lib/python3.8/site-packages" - - if python -m pip install sdkit==$SDKIT_VERSION ; then - echo "Installed." - else - fail "sdkit install failed" - fi -fi - -python -c "from importlib.metadata import version; print('sdkit version:', version('sdkit'))" - -# upgrade stable-diffusion-sdkit -python -m pip install --upgrade stable-diffusion-sdkit==$SD_VERSION -q -python -c "from importlib.metadata import version; print('stable-diffusion version:', version('stable-diffusion-sdkit'))" - -# install rich -if python ../scripts/check_modules.py rich; then - echo "rich has already been installed." -else - echo "Installing rich.." - - export PYTHONNOUSERSITE=1 - export PYTHONPATH="$INSTALL_ENV_DIR/lib/python3.8/site-packages" - - if python -m pip install rich ; then - echo "Installed." - else - fail "Install failed for rich" - fi -fi - -if python ../scripts/check_modules.py uvicorn fastapi ; then - echo "Packages necessary for Easy Diffusion were already installed" -else - printf "\n\nDownloading packages necessary for Easy Diffusion..\n\n" - - export PYTHONNOUSERSITE=1 - export PYTHONPATH="$INSTALL_ENV_DIR/lib/python3.8/site-packages" - - if conda install -c conda-forge -y uvicorn fastapi ; then - echo "Installed. Testing.." - else - fail "'conda install uvicorn' failed" - fi - - if ! command -v uvicorn &> /dev/null; then - fail "UI packages not found!" - fi +if ! command -v uvicorn &> /dev/null; then + fail "UI packages not found!" fi if [ -f "../models/stable-diffusion/sd-v1-4.ckpt" ]; then