Merge pull request #389 from cmdr2/installer_umamba

Use micromamba to install git and conda, instead of bundling a copy of git and conda in the installer. This will not affect existing installations, they'll continue using the previously bundled git and conda that came with the old installer

build.bat

@@ -8,40 +8,42 @@
 set /p answer=Are you a developer of this project (Y/N)?
 if /i "%answer:~,1%" NEQ "Y" exit /b
 
-@set PYTHONNOUSERSITE=1
+mkdir dist\win\stable-diffusion-ui\scripts
+@REM mkdir dist\linux-mac\stable-diffusion-ui\scripts
 
-@mkdir dist\stable-diffusion-ui
+@rem copy the installer files for Windows
 
-@echo "Downloading components for the installer.."
+copy scripts\on_env_start.bat dist\win\stable-diffusion-ui\scripts\
+copy scripts\bootstrap.bat dist\win\stable-diffusion-ui\scripts\
+copy "scripts\Start Stable Diffusion UI.cmd" dist\win\stable-diffusion-ui\
+copy LICENSE dist\win\stable-diffusion-ui\
+copy "CreativeML Open RAIL-M License" dist\win\stable-diffusion-ui\
+copy "How to install and run.txt" dist\win\stable-diffusion-ui\
+echo. > dist\win\stable-diffusion-ui\scripts\install_status.txt
 
-@call conda env create --prefix installer -f environment.yaml
+@rem copy the installer files for Linux and Mac
-@call conda activate .\installer
 
-@echo "Creating a distributable package.."
+@REM copy scripts\on_env_start.sh dist\linux-mac\stable-diffusion-ui\scripts\
+@REM copy scripts\bootstrap.sh dist\linux-mac\stable-diffusion-ui\scripts\
+@REM copy scripts\start.sh dist\linux-mac\stable-diffusion-ui\
+@REM copy LICENSE dist\linux-mac\stable-diffusion-ui\
+@REM copy "CreativeML Open RAIL-M License" dist\linux-mac\stable-diffusion-ui\
+@REM copy "How to install and run.txt" dist\linux-mac\stable-diffusion-ui\
+@REM echo. > dist\linux-mac\stable-diffusion-ui\scripts\install_status.txt
 
-@call conda install -c conda-forge -y conda-pack
+@rem make the zip
-@call conda pack --n-threads -1 --prefix installer --format tar
 
-@cd dist\stable-diffusion-ui
+cd dist\win
-@mkdir installer
+call powershell Compress-Archive -Path stable-diffusion-ui -DestinationPath ..\stable-diffusion-ui-win-x64.zip
+cd ..\..
 
-@call tar -xf ..\..\installer.tar -C installer
+@REM cd dist\linux-mac
+@REM call powershell Compress-Archive -Path stable-diffusion-ui -DestinationPath ..\stable-diffusion-ui-linux-x64.zip
+@REM call powershell Compress-Archive -Path stable-diffusion-ui -DestinationPath ..\stable-diffusion-ui-linux-arm64.zip
+@REM call powershell Compress-Archive -Path stable-diffusion-ui -DestinationPath ..\stable-diffusion-ui-mac-x64.zip
+@REM call powershell Compress-Archive -Path stable-diffusion-ui -DestinationPath ..\stable-diffusion-ui-mac-arm64.zip
+@REM cd ..\..
 
-@mkdir scripts
+echo "Build ready. Upload the zip files inside the 'dist' folder."
 
-@copy ..\..\scripts\on_env_start.bat scripts\
+pause
-@copy "..\..\scripts\Start Stable Diffusion UI.cmd" .
-@copy ..\..\LICENSE .
-@copy "..\..\CreativeML Open RAIL-M License" .
-@copy "..\..\How to install and run.txt" .
-@echo. > scripts\install_status.txt
-
-@echo "Build ready. Zip the 'dist\stable-diffusion-ui' folder."
-
-@echo "Cleaning up.."
-
-@cd ..\..
-
-@rmdir /s /q installer
-
-@del installer.tar

build.sh

@@ -11,45 +11,40 @@ case $yn in
     * ) exit;;
 esac
 
-export PYTHONNOUSERSITE=1
+# mkdir -p dist/win/stable-diffusion-ui/scripts
+mkdir -p dist/linux-mac/stable-diffusion-ui/scripts
 
-mkdir -p dist/stable-diffusion-ui
+# copy the installer files for Windows
 
-echo "Downloading components for the installer.."
+# cp scripts/on_env_start.bat dist/win/stable-diffusion-ui/scripts/
+# cp scripts/bootstrap.bat dist/win/stable-diffusion-ui/scripts/
+# cp "scripts/Start Stable Diffusion UI.cmd" dist/win/stable-diffusion-ui/
+# cp LICENSE dist/win/stable-diffusion-ui/
+# cp "CreativeML Open RAIL-M License" dist/win/stable-diffusion-ui/
+# cp "How to install and run.txt" dist/win/stable-diffusion-ui/
+# echo "" > dist/win/stable-diffusion-ui/scripts/install_status.txt
 
-source ~/miniconda3/etc/profile.d/conda.sh
+# copy the installer files for Linux and Mac
 
-conda install -c conda-forge -y conda-pack
+cp scripts/on_env_start.sh dist/linux-mac/stable-diffusion-ui/scripts/
+cp scripts/bootstrap.sh dist/linux-mac/stable-diffusion-ui/scripts/
+cp scripts/start.sh dist/linux-mac/stable-diffusion-ui/
+cp LICENSE dist/linux-mac/stable-diffusion-ui/
+cp "CreativeML Open RAIL-M License" dist/linux-mac/stable-diffusion-ui/
+cp "How to install and run.txt" dist/linux-mac/stable-diffusion-ui/
+echo "" > dist/linux-mac/stable-diffusion-ui/scripts/install_status.txt
 
-conda env create --prefix installer -f environment.yaml
+# make the zip
-conda activate ./installer
 
-echo "Creating a distributable package.."
+# cd dist/win
-
+# zip -r ../stable-diffusion-ui-win-x64.zip stable-diffusion-ui
-conda pack --n-threads -1 --prefix installer --format tar
+# cd ../..
-
-cd dist/stable-diffusion-ui
-mkdir installer
-
-tar -xf ../../installer.tar -C installer
-
-mkdir scripts
-
-cp ../../scripts/on_env_start.sh scripts/
-cp ../../scripts/start.sh .
-cp ../../LICENSE .
-cp "../../CreativeML Open RAIL-M License" .
-cp "../../How to install and run.txt" .
-echo "" > scripts/install_status.txt
-
-chmod u+x start.sh
-
-echo "Build ready. Zip the 'dist/stable-diffusion-ui' folder."
-
-echo "Cleaning up.."
 
+cd dist/linux-mac
+zip -r ../stable-diffusion-ui-linux-x64.zip stable-diffusion-ui
+zip -r ../stable-diffusion-ui-linux-arm64.zip stable-diffusion-ui
+zip -r ../stable-diffusion-ui-mac-x64.zip stable-diffusion-ui
+zip -r ../stable-diffusion-ui-mac-arm64.zip stable-diffusion-ui
 cd ../..
 
-rm -rf installer
+echo "Build ready. Upload the zip files inside the 'dist' folder."
-
-rm installer.tar

scripts/Developer Console.cmd

@@ -2,13 +2,29 @@
 
 echo "Opening Stable Diffusion UI - Developer Console.." & echo.
 
-@call installer\Scripts\activate.bat
+set PATH=C:\Windows\System32;%PATH%
 
-@call conda-unpack
+@rem set legacy and new installer's PATH, if they exist
+if exist "installer" set PATH=%cd%\installer;%cd%\installer\Library\bin;%cd%\installer\Scripts;%cd%\installer\Library\usr\bin;%PATH%
+if exist "installer_files\env" set PATH=%cd%\installer_files\env;%cd%\installer_files\env\Library\bin;%cd%\installer_files\env\Scripts;%cd%\installer_files\Library\usr\bin;%PATH%
 
-@call conda --version
+@rem activate the installer env
-@call git --version
+call conda activate
 
-@call conda activate .\stable-diffusion\env
+@rem Test the environment
+echo "Environment Info:"
+call where git
+call git --version
 
-cmd /k
+call where python
+call python --version
+
+call where conda
+call conda --version
+
+echo.
+
+@rem activate the environment
+call conda activate .\stable-diffusion\env
+
+cmd /k

scripts/Start Stable Diffusion UI.cmd

@@ -1,19 +1,27 @@
 @echo off
 
-@REM Delete the post-activate hook from the old installer
+set PATH=C:\Windows\System32;%PATH%
-if exist "installer\etc\conda\activate.d\post_activate.bat" (
-    echo. > installer\etc\conda\activate.d\post_activate.bat
-)
 
-@call installer\Scripts\activate.bat
+@rem set legacy installer's PATH, if it exists
+if exist "installer" set PATH=%cd%\installer;%cd%\installer\Library\bin;%cd%\installer\Scripts;%cd%\installer\Library\usr\bin;%PATH%
 
-@call conda-unpack
+@rem Setup the packages required for the installer
+call scripts\bootstrap.bat
 
-@call conda --version
+@rem set new installer's PATH, if it downloaded any packages
-@call git --version
+if exist "installer_files\env" set PATH=%cd%\installer_files\env;%cd%\installer_files\env\Library\bin;%cd%\installer_files\env\Scripts;%cd%\installer_files\Library\usr\bin;%PATH%
 
-@cd installer
+@rem Test the bootstrap
+call where git
+call git --version
 
-@call ..\scripts\on_env_start.bat
+call where python
+call python --version
+
+call where conda
+call conda --version
+
+@rem Download the rest of the installer and UI
+call scripts\on_env_start.bat
 
 @pause

scripts/bootstrap.bat

@@ -0,0 +1,55 @@
+@echo off
+
+@rem This script will install git and conda (if not found on the PATH variable)
+@rem  using micromamba (an 8mb static-linked single-file binary, conda replacement).
+@rem For users who already have git and conda, this step will be skipped.
+
+@rem This enables a user to install this project without manually installing conda and git.
+
+@rem config
+set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba
+set INSTALL_ENV_DIR=%cd%\installer_files\env
+set LEGACY_INSTALL_ENV_DIR=%cd%\installer
+set MICROMAMBA_DOWNLOAD_URL=https://github.com/cmdr2/stable-diffusion-ui/releases/download/v1.1/micromamba.exe
+
+@rem figure out whether git and conda needs to be installed
+if exist "%INSTALL_ENV_DIR%" set PATH=%INSTALL_ENV_DIR%;%INSTALL_ENV_DIR%\Library\bin;%INSTALL_ENV_DIR%\Scripts;%INSTALL_ENV_DIR%\Library\usr\bin;%PATH%
+
+set PACKAGES_TO_INSTALL=
+
+if not exist "%LEGACY_INSTALL_ENV_DIR%\etc\profile.d\conda.sh" (
+    if not exist "%INSTALL_ENV_DIR%\etc\profile.d\conda.sh" set PACKAGES_TO_INSTALL=%PACKAGES_TO_INSTALL% conda
+)
+
+call git --version >.tmp1 2>.tmp2
+if "%ERRORLEVEL%" NEQ "0" set PACKAGES_TO_INSTALL=%PACKAGES_TO_INSTALL% git
+
+@rem (if necessary) install git and conda into a contained environment
+if "%PACKAGES_TO_INSTALL%" NEQ "" (
+    @rem download micromamba
+    if not exist "%MAMBA_ROOT_PREFIX%\micromamba.exe" (
+        echo "Downloading micromamba from %MICROMAMBA_DOWNLOAD_URL% to %MAMBA_ROOT_PREFIX%\micromamba.exe"
+
+        mkdir "%MAMBA_ROOT_PREFIX%"
+        call curl -L "%MICROMAMBA_DOWNLOAD_URL%" > "%MAMBA_ROOT_PREFIX%\micromamba.exe"
+
+        @rem test the mamba binary
+        echo Micromamba version:
+        call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version
+    )
+
+    @rem create the installer env
+    if not exist "%INSTALL_ENV_DIR%" (
+        call "%MAMBA_ROOT_PREFIX%\micromamba.exe" create -y --prefix "%INSTALL_ENV_DIR%"
+    )
+
+    echo "Packages to install:%PACKAGES_TO_INSTALL%"
+
+    call "%MAMBA_ROOT_PREFIX%\micromamba.exe" install -y --prefix "%INSTALL_ENV_DIR%" -c conda-forge %PACKAGES_TO_INSTALL%
+
+    if not exist "%INSTALL_ENV_DIR%" (
+        echo "There was a problem while installing%PACKAGES_TO_INSTALL% using micromamba. Cannot continue."
+        pause
+        exit /b
+    )
+)

scripts/bootstrap.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# This script will install git and conda (if not found on the PATH variable)
+#  using micromamba (an 8mb static-linked single-file binary, conda replacement).
+# For users who already have git and conda, this step will be skipped.
+
+# This enables a user to install this project without manually installing conda and git.
+
+
+OS_NAME=$(uname -s)
+case "${OS_NAME}" in
+    Linux*)     OS_NAME="linux";;
+    Darwin*)    OS_NAME="osx";;
+    *)          echo "Unknown OS: $OS_NAME! This script runs only on Linux or Mac" && exit
+esac
+
+OS_ARCH=$(uname -m)
+case "${OS_ARCH}" in
+    x86_64*)    OS_ARCH="64";;
+    arm64*)     OS_ARCH="arm64";;
+    *)          echo "Unknown system architecture: $OS_ARCH! This script runs only on x86_64 or arm64" && exit
+esac
+
+# https://mamba.readthedocs.io/en/latest/installation.html
+if [ "$OS_NAME" == "linux" ] && [ "$OS_ARCH" == "arm64" ]; then OS_ARCH="aarch64"; fi
+
+# config
+export MAMBA_ROOT_PREFIX="$(pwd)/installer_files/mamba"
+INSTALL_ENV_DIR="$(pwd)/installer_files/env"
+LEGACY_INSTALL_ENV_DIR="$(pwd)/installer"
+MICROMAMBA_DOWNLOAD_URL="https://micro.mamba.pm/api/micromamba/${OS_NAME}-${OS_ARCH}/latest"
+
+# figure out whether git and conda needs to be installed
+if [ -e "$INSTALL_ENV_DIR" ]; then export PATH="$INSTALL_ENV_DIR/bin:$PATH"; fi
+
+PACKAGES_TO_INSTALL=""
+
+if [ ! -e "$LEGACY_INSTALL_ENV_DIR/etc/profile.d/conda.sh" ] && [ ! -e "$INSTALL_ENV_DIR/etc/profile.d/conda.sh" ]; then PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL conda"; fi
+if ! hash "git" &>/dev/null; then PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git"; fi
+
+# (if necessary) install git and conda into a contained environment
+if [ "$PACKAGES_TO_INSTALL" != "" ]; then
+    # download micromamba
+    if [ ! -e "$MAMBA_ROOT_PREFIX/micromamba" ]; then
+        echo "Downloading micromamba from $MICROMAMBA_DOWNLOAD_URL to $MAMBA_ROOT_PREFIX/micromamba"
+
+        mkdir -p "$MAMBA_ROOT_PREFIX"
+        curl -L "$MICROMAMBA_DOWNLOAD_URL" | tar -xvj bin/micromamba -O > "$MAMBA_ROOT_PREFIX/micromamba"
+
+        chmod u+x "$MAMBA_ROOT_PREFIX/micromamba"
+
+        # test the mamba binary
+        echo "Micromamba version:"
+        "$MAMBA_ROOT_PREFIX/micromamba" --version
+    fi
+
+    # create the installer env
+    if [ ! -e "$INSTALL_ENV_DIR" ]; then
+        "$MAMBA_ROOT_PREFIX/micromamba" create -y --prefix "$INSTALL_ENV_DIR"
+    fi
+
+    echo "Packages to install:$PACKAGES_TO_INSTALL"
+
+    "$MAMBA_ROOT_PREFIX/micromamba" install -y --prefix "$INSTALL_ENV_DIR" -c conda-forge $PACKAGES_TO_INSTALL
+
+    if [ ! -e "$INSTALL_ENV_DIR" ]; then
+        echo "There was a problem while installing$PACKAGES_TO_INSTALL using micromamba. Cannot continue."
+        exit
+    fi
+fi

scripts/developer_console.sh

@@ -4,14 +4,34 @@ if [ "$0" == "bash" ]; then
   echo "Opening Stable Diffusion UI - Developer Console.."
   echo ""
 
-  source installer/bin/activate
+  # set legacy and new installer's PATH, if they exist
+  if [ -e "installer" ]; then export PATH="$(pwd)/installer/bin:$PATH"; fi
+  if [ -e "installer_files/env" ]; then export PATH="$(pwd)/installer_files/env/bin:$PATH"; fi
 
-  conda-unpack
+  # activate the installer env
+  CONDA_BASEPATH=$(conda info --base)
+  source "$CONDA_BASEPATH/etc/profile.d/conda.sh" # avoids the 'shell not initialized' error
 
-  conda --version
+  conda activate
+
+  # test the environment
+  echo "Environment Info:"
+  which git
   git --version
 
+  which python
+  python --version
+
+  which conda
+  conda --version
+
+  echo ""
+
+  # activate the environment
+  CONDA_BASEPATH=$(conda info --base)
+  source "$CONDA_BASEPATH/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script)
+
   conda activate ./stable-diffusion/env
 else
   bash --init-file developer_console.sh
-fi
+fi

scripts/on_env_start.bat

@@ -4,8 +4,6 @@
 
 set PATH=C:\Windows\System32;%PATH%
 
-@cd ..
-
 if exist "scripts\config.bat" (
     @call scripts\config.bat
 )
@@ -35,7 +33,7 @@ if "%update_branch%"=="" (
     @cd sd-ui-files
 
     @call git reset --hard
-    @call git checkout "%update_branch%"
+    @call git -c advice.detachedHead=false checkout "%update_branch%"
     @call git pull
 
     @cd ..
@@ -54,9 +52,10 @@ if "%update_branch%"=="" (
 
 @xcopy sd-ui-files\ui ui /s /i /Y
 @copy sd-ui-files\scripts\on_sd_start.bat scripts\ /Y
+@copy sd-ui-files\scripts\bootstrap.bat scripts\ /Y
 @copy "sd-ui-files\scripts\Start Stable Diffusion UI.cmd" . /Y
 @copy "sd-ui-files\scripts\Developer Console.cmd" . /Y
 
 @call scripts\on_sd_start.bat
 
-@pause
+@pause

scripts/on_env_start.sh

@@ -16,7 +16,7 @@ if [ -f "scripts/install_status.txt" ] && [ `grep -c sd_ui_git_cloned scripts/in
     cd sd-ui-files
 
     git reset --hard
-    git checkout "$update_branch"
+    git -c advice.detachedHead=false checkout "$update_branch"
     git pull
 
     cd ..
@@ -36,6 +36,7 @@ fi
 rm -rf ui
 cp -Rf sd-ui-files/ui .
 cp sd-ui-files/scripts/on_sd_start.sh scripts/
+cp sd-ui-files/scripts/bootstrap.sh scripts/
 cp sd-ui-files/scripts/start.sh .
 cp sd-ui-files/scripts/developer_console.sh .
 

scripts/on_sd_start.bat

@@ -1,11 +1,19 @@
 @echo off
 
 @copy sd-ui-files\scripts\on_env_start.bat scripts\ /Y
+@copy sd-ui-files\scripts\bootstrap.bat scripts\ /Y
+
+if exist "%cd%\profile" (
+    set USERPROFILE=%cd%\profile
+)
+
+@rem activate the installer env
+call conda activate
 
 @REM Caution, this file will make your eyes and brain bleed. It's such an unholy mess.
 @REM Note to self: Please rewrite this in Python. For the sake of your own sanity.
 
-@copy "sd-ui-files\scripts\Developer Console.cmd" . /Y
+@REM remove the old version of the dev console script, if it's still present
 if exist "Open Developer Console.cmd" del "Open Developer Console.cmd"
 
 @call python -c "import os; import shutil; frm = 'sd-ui-files\\ui\\hotfix\\9c24e6cd9f499d02c4f21a033736dabd365962dc80fe3aeb57a8f85ea45a20a3.26fead7ea4f0f843f6eb4055dfd25693f1a71f3c6871b184042d4b126244e142'; dst = os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'transformers', '9c24e6cd9f499d02c4f21a033736dabd365962dc80fe3aeb57a8f85ea45a20a3.26fead7ea4f0f843f6eb4055dfd25693f1a71f3c6871b184042d4b126244e142'); shutil.copyfile(frm, dst) if os.path.exists(dst) else print(''); print('Hotfixed broken JSON file from OpenAI');"
@@ -18,7 +26,7 @@ if exist "Open Developer Console.cmd" del "Open Developer Console.cmd"
 
     @call git reset --hard
     @call git pull
-    @call git checkout f6cfebffa752ee11a7b07497b8529d5971de916c
+    @call git -c advice.detachedHead=false checkout f6cfebffa752ee11a7b07497b8529d5971de916c
 
     @call git apply ..\ui\sd_internal\ddim_callback.patch
     @call git apply ..\ui\sd_internal\env_yaml.patch
@@ -36,7 +44,7 @@ if exist "Open Developer Console.cmd" del "Open Developer Console.cmd"
     )
 
     @cd stable-diffusion
-    @call git checkout f6cfebffa752ee11a7b07497b8529d5971de916c
+    @call git -c advice.detachedHead=false checkout f6cfebffa752ee11a7b07497b8529d5971de916c
 
     @call git apply ..\ui\sd_internal\ddim_callback.patch
     @call git apply ..\ui\sd_internal\env_yaml.patch

scripts/on_sd_start.sh

@@ -1,10 +1,15 @@
 #!/bin/bash
 
 cp sd-ui-files/scripts/on_env_start.sh scripts/
+cp sd-ui-files/scripts/bootstrap.sh scripts/
 
-source installer/etc/profile.d/conda.sh
+# activate the installer env
+CONDA_BASEPATH=$(conda info --base)
+source "$CONDA_BASEPATH/etc/profile.d/conda.sh" # avoids the 'shell not initialized' error
 
-cp sd-ui-files/scripts/developer_console.sh .
+conda activate
+
+# remove the old version of the dev console script, if it's still present
 if [ -e "open_dev_console.sh" ]; then
     rm "open_dev_console.sh"
 fi
@@ -21,7 +26,7 @@ if [ -e "scripts/install_status.txt" ] && [ `grep -c sd_git_cloned scripts/insta
 
     git reset --hard
     git pull
-    git checkout f6cfebffa752ee11a7b07497b8529d5971de916c
+    git -c advice.detachedHead=false checkout f6cfebffa752ee11a7b07497b8529d5971de916c
 
     git apply ../ui/sd_internal/ddim_callback.patch
     git apply ../ui/sd_internal/env_yaml.patch
@@ -39,7 +44,7 @@ else
     fi
 
     cd stable-diffusion
-    git checkout f6cfebffa752ee11a7b07497b8529d5971de916c
+    git -c advice.detachedHead=false checkout f6cfebffa752ee11a7b07497b8529d5971de916c
 
     git apply ../ui/sd_internal/ddim_callback.patch
     git apply ../ui/sd_internal/env_yaml.patch
@@ -303,7 +308,7 @@ fi
 printf "\n\nStable Diffusion is ready!\n\n"
 
 SD_PATH=`pwd`
-export PYTHONPATH="$SD_PATH;$SD_PATH/env/lib/python3.8/site-packages"
+export PYTHONPATH="$SD_PATH:$SD_PATH/env/lib/python3.8/site-packages"
 echo "PYTHONPATH=$PYTHONPATH"
 
 cd ..

scripts/start.sh

@@ -1,10 +1,23 @@
 #!/bin/bash
 
-source installer/bin/activate
+# set legacy installer's PATH, if it exists
+if [ -e "installer" ]; then export PATH="$(pwd)/installer/bin:$PATH"; fi
 
-conda-unpack
+# Setup the packages required for the installer
+scripts/bootstrap.sh
 
-conda --version
+# set new installer's PATH, if it downloaded any packages
+if [ -e "installer_files/env" ]; then export PATH="$(pwd)/installer_files/env/bin:$PATH"; fi
+
+# Test the bootstrap
+which git
 git --version
 
+which python
+python --version
+
+which conda
+conda --version
+
+# Download the rest of the installer and UI
 scripts/on_env_start.sh