Upload 17 files

.gitattributes

@@ -1,34 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ps1 text eol=crlf

.gitignore

@@ -0,0 +1,20 @@
+.vscode
+.idea
+
+venv
+
+output/*
+!output/.keep
+
+py310
+git
+
+train/*
+logs/*
+sd-models/*
+!sd-models/put stable diffusion model here.txt
+!logs/.keep
+
+tests/
+
+huggingface/hub/models--openai--clip-vit-large-patch14

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "sd-scripts"]
+	path = sd-scripts
+	url = https://github.com/kohya-ss/sd-scripts.git
+[submodule "frontend"]
+	path = frontend
+	url = https://github.com/hanamizuki-ai/lora-gui-dist

README.md

@@ -0,0 +1,47 @@
+# LoRA-scripts
+
+LoRA training scripts for [kohya-ss/sd-scripts](https://github.com/kohya-ss/sd-scripts.git)
+
+## Usage
+
+### Clone repo with submodules
+
+```sh
+git clone --recurse-submodules https://github.com/Akegarasu/lora-scripts
+```
+
+### Required Dependencies
+
+Python 3.10.8 and Git
+
+### Windows
+
+#### Installation
+
+Run `install.ps1` will automaticilly create a venv for you and install necessary deps.
+
+#### Train
+
+Edit `train.ps1`, and run it.
+
+### Linux
+
+#### Installation
+
+Run `install.bash` will create a venv and install necessary deps.
+
+#### Train
+
+Training script `train.sh` **will not** activate venv for you. You should activate venv first.
+
+```sh
+source venv/bin/activate
+```
+
+Edit `train.sh`, and run it.
+
+#### TensorBoard
+
+Run `tensorboard.ps1` will start TensorBoard at http://localhost:6006/
+
+![](./assets/tensorboard-example.png)

gui.py

@@ -0,0 +1,94 @@
+import argparse
+import json
+import os
+import subprocess
+import sys
+import webbrowser
+from datetime import datetime
+from threading import Lock
+
+import uvicorn
+from fastapi import BackgroundTasks, FastAPI, Request
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+
+import toml
+
+app = FastAPI()
+
+lock = Lock()
+
+# fix mimetype error in some fucking systems
+sf = StaticFiles(directory="frontend/dist")
+_o_fr = sf.file_response
+def _hooked_file_response(*args, **kwargs):
+    full_path = args[0]
+    r = _o_fr(*args, **kwargs)
+    if full_path.endswith(".js"):
+        r.media_type = "application/javascript"
+    elif full_path.endswith(".css"):
+        r.media_type = "text/css"
+    return r
+sf.file_response = _hooked_file_response
+
+parser = argparse.ArgumentParser(description="GUI for training network")
+parser.add_argument("--port", type=int, default=28000, help="Port to run the server on")
+
+def run_train(toml_path: str):
+    print(f"Training started with config file / 训练开始，使用配置文件: {toml_path}")
+    args = [
+        "accelerate", "launch", "--num_cpu_threads_per_process", "8",
+        "./sd-scripts/train_network.py",
+        "--config_file", toml_path,
+    ]
+    try:
+        result = subprocess.run(args, shell=True, env=os.environ)
+        if result.returncode != 0:
+            print(f"Training failed / 训练失败")
+        else:
+            print(f"Training finished / 训练完成")
+    except Exception as e:
+        print(f"An error occurred when training / 创建训练进程时出现致命错误: {e}")
+    finally:
+        lock.release()
+
+
+@app.post("/api/run")
+async def create_toml_file(request: Request, background_tasks: BackgroundTasks):
+    acquired = lock.acquire(blocking=False)
+
+    if not acquired:
+        print("Training is already running / 已有正在进行的训练")
+        return {"status": "fail", "detail": "Training is already running"}
+
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    toml_file = f"toml/{timestamp}.toml"
+    toml_data = await request.body()
+    j = json.loads(toml_data.decode("utf-8"))
+    with open(toml_file, "w") as f:
+        f.write(toml.dumps(j))
+    background_tasks.add_task(run_train, toml_file)
+    return {"status": "success"}
+
+@app.middleware("http")
+async def add_cache_control_header(request, call_next):
+    response = await call_next(request)
+    response.headers["Cache-Control"] = "max-age=0"
+    return response
+
+@app.get("/")
+async def index():
+    return FileResponse("./frontend/dist/index.html")
+
+
+app.mount("/", sf, name="static")
+
+if __name__ == "__main__":
+    args, _ = parser.parse_known_args()
+    print(f"Server started at http://127.0.0.1:{args.port}")
+    if sys.platform == "win32":
+        # disable triton on windows
+        os.environ["XFORMERS_FORCE_DISABLE_TRITON"] = "1"
+    
+    webbrowser.open(f"http://127.0.0.1:{args.port}")
+    uvicorn.run(app, host="127.0.0.1", port=28000, log_level="error")

install-cn.ps1

@@ -0,0 +1,55 @@
+$Env:HF_HOME = "huggingface"
+$Env:PIP_DISABLE_PIP_VERSION_CHECK = 1
+$Env:PIP_NO_CACHE_DIR = 1
+function InstallFail {
+    Write-Output "��װʧ�ܡ�"
+    Read-Host | Out-Null ;
+    Exit
+}
+
+function Check {
+    param (
+        $ErrorInfo
+    )
+    if (!($?)) {
+        Write-Output $ErrorInfo
+        InstallFail
+    }
+}
+
+if (!(Test-Path -Path "venv")) {
+    Write-Output "���ڴ������⻷��..."
+    python -m venv venv
+    Check "�������⻷��ʧ�ܣ����� python �Ƿ�װ����Լ� python �汾�Ƿ�Ϊ64λ�汾��python 3.10����python��Ŀ¼�Ƿ��ڻ�������PATH�ڡ�"
+}
+
+.\venv\Scripts\activate
+Check "�������⻷��ʧ�ܡ�"
+
+Set-Location .\sd-scripts
+Write-Output "��װ������������ (�ѽ��й��ڼ��٣����ڹ�����޷�ʹ�ü���Դ�뻻�� install.ps1 �ű�)"
+$install_torch = Read-Host "�Ƿ���Ҫ��װ Torch+xformers? ��������Ϊ�״ΰ�װ��ѡ�� y ��������Ϊ����������װ��ѡ�� n��[y/n] (Ĭ��Ϊ y)"
+if ($install_torch -eq "y" -or $install_torch -eq "Y" -or $install_torch -eq ""){
+    pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 -f https://mirror.sjtu.edu.cn/pytorch-wheels/torch_stable.html -i https://mirrors.bfsu.edu.cn/pypi/web/simple
+    Check "torch ��װʧ�ܣ���ɾ�� venv �ļ��к��������С�"
+    pip install -U -I --no-deps xformers==0.0.17 -i https://mirrors.aliyun.com/pypi/simple/
+    Check "xformers ��װʧ�ܡ�"
+}
+
+pip install --upgrade -r requirements.txt -i https://mirrors.bfsu.edu.cn/pypi/web/simple
+Check "����������װʧ�ܡ�"
+pip install --upgrade lion-pytorch dadaptation -i https://mirrors.bfsu.edu.cn/pypi/web/simple
+Check "Lion��dadaptation �Ż�����װʧ�ܡ�"
+pip install --upgrade lycoris-lora -i https://mirrors.bfsu.edu.cn/pypi/web/simple
+Check "lycoris ��װʧ�ܡ�"
+pip install --upgrade fastapi uvicorn -i https://mirrors.bfsu.edu.cn/pypi/web/simple
+Check "UI ����������װʧ�ܡ�"
+
+
+Write-Output "��װ bitsandbytes..."
+cp .\bitsandbytes_windows\*.dll ..\venv\Lib\site-packages\bitsandbytes\
+cp .\bitsandbytes_windows\cextension.py ..\venv\Lib\site-packages\bitsandbytes\cextension.py
+cp .\bitsandbytes_windows\main.py ..\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py
+
+Write-Output "��װ���"
+Read-Host | Out-Null ;

install.bash

@@ -0,0 +1,34 @@
+echo "Creating python venv..."
+python3 -m venv venv
+source venv/bin/activate
+
+echo "Installing torch & xformers..."
+printf 'Which version of torch do you want to install?
+(1) torch 2.0.0+cu118 with xformers 0.0.17 (suggested)
+(2) torch 1.12.1+cu116, with xformers 0bad001ddd56c080524d37c84ff58d9cd030ebfd
+'
+while true; do
+    read -p "Choose: " version
+    case $version in
+    [1]*)
+        pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 --extra-index-url https://download.pytorch.org/whl/cu118
+        pip install xformers==0.0.17
+        break
+        ;;
+    [2]*)
+        pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
+        pip install --upgrade git+https://github.com/facebookresearch/xformers.git@0bad001ddd56c080524d37c84ff58d9cd030ebfd
+        pip install triton==2.0.0.dev20221202
+        break
+        ;;
+    *) echo "Please enter 1 or 2." ;;
+    esac
+done
+
+echo "Installing deps..."
+cd ./sd-scripts
+
+pip install --upgrade -r requirements.txt
+pip install --upgrade lion-pytorch lycoris-lora dadaptation
+
+echo "Install completed"

install.ps1

@@ -0,0 +1,23 @@
+$Env:HF_HOME = "huggingface"
+
+if (!(Test-Path -Path "venv")) {
+    Write-Output  "Creating venv for python..."
+    python -m venv venv
+}
+.\venv\Scripts\activate
+
+Write-Output "Installing deps..."
+Set-Location .\sd-scripts
+pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 --extra-index-url https://download.pytorch.org/whl/cu118
+pip install --upgrade -r requirements.txt
+pip install --upgrade xformers==0.0.17
+
+Write-Output "Installing bitsandbytes for windows..."
+cp .\bitsandbytes_windows\*.dll ..\venv\Lib\site-packages\bitsandbytes\
+cp .\bitsandbytes_windows\cextension.py ..\venv\Lib\site-packages\bitsandbytes\cextension.py
+cp .\bitsandbytes_windows\main.py ..\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py
+
+pip install --upgrade lion-pytorch dadaptation lycoris-lora
+
+Write-Output "Install completed"
+Read-Host | Out-Null ;

interrogate.ps1

@@ -0,0 +1,29 @@
+# LoRA interrogate script by @bdsqlsz
+
+$v2 = 0 # load Stable Diffusion v2.x model / Stable Diffusion 2.x模型读取
+$sd_model = "./sd-models/sd_model.safetensors" # Stable Diffusion model to load: ckpt or safetensors file | 读取的基础SD模型, 保存格式 cpkt 或 safetensors
+$model = "./output/LoRA.safetensors" # LoRA model to interrogate: ckpt or safetensors file | 需要调查关键字的LORA模型, 保存格式 cpkt 或 safetensors
+$batch_size = 64 # batch size for processing with Text Encoder | 使用 Text Encoder 处理时的批量大小，默认16，推荐64/128
+$clip_skip = 1 # use output of nth layer from back of text encoder (n>=1) | 使用文本编码器倒数第 n 层的输出，n 可以是大于等于 1 的整数
+
+
+# Activate python venv
+.\venv\Scripts\activate
+
+$Env:HF_HOME = "huggingface"
+$ext_args = [System.Collections.ArrayList]::new()
+
+if ($v2) {
+  [void]$ext_args.Add("--v2")
+}
+
+# run interrogate
+accelerate launch --num_cpu_threads_per_process=8 "./sd-scripts/networks/lora_interrogator.py" `
+	--sd_model=$sd_model `
+	--model=$model `
+	--batch_size=$batch_size `
+	--clip_skip=$clip_skip `
+	$ext_args 
+
+Write-Output "Interrogate finished"
+Read-Host | Out-Null ;

resize.ps1

@@ -0,0 +1,41 @@
+# LoRA resize script by @bdsqlsz
+
+$save_precision = "fp16" # precision in saving, default float | 保存精度, 可选 float、fp16、bf16, 默认 float
+$new_rank = 4 # dim rank of output LoRA | dim rank等级, 默认 4
+$model = "./output/lora_name.safetensors" # original LoRA model path need to resize, save as cpkt or safetensors | 需要调整大小的模型路径, 保存格式 cpkt 或 safetensors
+$save_to = "./output/lora_name_new.safetensors" # output LoRA model path, save as ckpt or safetensors | 输出路径, 保存格式 cpkt 或 safetensors
+$device = "cuda" # device to use, cuda for GPU | 使用 GPU跑, 默认 CPU
+$verbose = 1 # display verbose resizing information | rank变更时, 显示详细信息
+$dynamic_method = "" # Specify dynamic resizing method, --new_rank is used as a hard limit for max rank | 动态调节大小，可选"sv_ratio", "sv_fro", "sv_cumulative",默认无
+$dynamic_param = "" # Specify target for dynamic reduction | 动态参数,sv_ratio模式推荐1~2, sv_cumulative模式0~1, sv_fro模式0~1, 比sv_cumulative要高
+
+
+# Activate python venv
+.\venv\Scripts\activate
+
+$Env:HF_HOME = "huggingface"
+$ext_args = [System.Collections.ArrayList]::new()
+
+if ($verbose) {
+  [void]$ext_args.Add("--verbose")
+}
+
+if ($dynamic_method) {
+  [void]$ext_args.Add("--dynamic_method=" + $dynamic_method)
+}
+
+if ($dynamic_param) {
+  [void]$ext_args.Add("--dynamic_param=" + $dynamic_param)
+}
+
+# run resize
+accelerate launch --num_cpu_threads_per_process=8 "./sd-scripts/networks/resize_lora.py" `
+	--save_precision=$save_precision `
+	--new_rank=$new_rank `
+	--model=$model `
+	--save_to=$save_to `
+	--device=$device `
+	$ext_args 
+
+Write-Output "Resize finished"
+Read-Host | Out-Null ;

run_gui.ps1

@@ -0,0 +1,6 @@
+.\venv\Scripts\activate
+
+$Env:HF_HOME = "huggingface"
+$Env:PYTHONUTF8 = "1"
+
+python ./gui.py

tensorboard.ps1

@@ -0,0 +1,3 @@
+.\venv\Scripts\activate
+
+tensorboard --logdir=.\logs

train.ipynb

@@ -0,0 +1,99 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Train data path | 设置训练用模型、图片\n",
+    "pretrained_model = \"./sd-models/model.ckpt\" # base model path | 底模路径\n",
+    "train_data_dir = \"./train/aki\" # train dataset path | 训练数据集路径\n",
+    "\n",
+    "# Train related params | 训练相关参数\n",
+    "resolution = \"512,512\" # image resolution w,h. 图片分辨率，宽,高。支持非正方形，但必须是 64 倍数。\n",
+    "batch_size = 1 # batch size\n",
+    "max_train_epoches = 10 # max train epoches | 最大训练 epoch\n",
+    "save_every_n_epochs = 2 # save every n epochs | 每 N 个 epoch 保存一次\n",
+    "network_dim = 32 # network dim | 常用 4~128，不是越大越好\n",
+    "network_alpha= 32 # network alpha | 常用与 network_dim 相同的值或者采用较小的值，如 network_dim的一半 防止下溢。默认值为 1，使用较小的 alpha 需要提升学习率。\n",
+    "clip_skip = 2 # clip skip | 玄学 一般用 2\n",
+    "train_unet_only = 0 # train U-Net only | 仅训练 U-Net，开启这个会牺牲效果大幅减少显存使用。6G显存可以开启\n",
+    "train_text_encoder_only = 0 # train Text Encoder only | 仅训练 文本编码器\n",
+    "\n",
+    "# Learning rate | 学习率\n",
+    "lr = \"1e-4\"\n",
+    "unet_lr = \"1e-4\"\n",
+    "text_encoder_lr = \"1e-5\"\n",
+    "lr_scheduler = \"cosine_with_restarts\" # \"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"\n",
+    "\n",
+    "# Output settings | 输出设置\n",
+    "output_name = \"aki\" # output model name | 模型保存名称\n",
+    "save_model_as = \"safetensors\" # model save ext | 模型保存格式 ckpt, pt, safetensors"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "!accelerate launch --num_cpu_threads_per_process=8 \"./sd-scripts/train_network.py\" \\\n",
+    "  --enable_bucket \\\n",
+    "  --pretrained_model_name_or_path=$pretrained_model \\\n",
+    "  --train_data_dir=$train_data_dir \\\n",
+    "  --output_dir=\"./output\" \\\n",
+    "  --logging_dir=\"./logs\" \\\n",
+    "  --resolution=$resolution \\\n",
+    "  --network_module=networks.lora \\\n",
+    "  --max_train_epochs=$max_train_epoches \\\n",
+    "  --learning_rate=$lr \\\n",
+    "  --unet_lr=$unet_lr \\\n",
+    "  --text_encoder_lr=$text_encoder_lr \\\n",
+    "  --network_dim=$network_dim \\\n",
+    "  --network_alpha=$network_alpha \\\n",
+    "  --output_name=$output_name \\\n",
+    "  --lr_scheduler=$lr_scheduler \\\n",
+    "  --train_batch_size=$batch_size \\\n",
+    "  --save_every_n_epochs=$save_every_n_epochs \\\n",
+    "  --mixed_precision=\"fp16\" \\\n",
+    "  --save_precision=\"fp16\" \\\n",
+    "  --seed=\"1337\" \\\n",
+    "  --cache_latents \\\n",
+    "  --clip_skip=$clip_skip \\\n",
+    "  --prior_loss_weight=1 \\\n",
+    "  --max_token_length=225 \\\n",
+    "  --caption_extension=\".txt\" \\\n",
+    "  --save_model_as=$save_model_as \\\n",
+    "  --xformers --shuffle_caption --use_8bit_adam"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.7 (tags/v3.10.7:6cc6b13, Sep  5 2022, 14:08:36) [MSC v.1933 64 bit (AMD64)]"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "675b13e958f0d0236d13cdfe08a1df3882cae564fa23a2e7e5eb1f2c6c632b02"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

train.ps1

@@ -0,0 +1,183 @@
+# LoRA train script by @Akegarasu
+
+# Train data path | 设置训练用模型、图片
+$pretrained_model = "./sd-models/model.ckpt" # base model path | 底模路径
+$is_v2_model = 0 # SD2.0 model | SD2.0模型 2.0模型下 clip_skip 默认无效
+$parameterization = 0 # parameterization | 参数化 本参数需要和 V2 参数同步使用 实验性功能
+$train_data_dir = "./train/aki" # train dataset path | 训练数据集路径
+$reg_data_dir = "" # directory for regularization images | 正则化数据集路径，默认不使用正则化图像。
+
+# Network settings | 网络设置
+$network_module = "networks.lora" # 在这里将会设置训练的网络种类，默认为 networks.lora 也就是 LoRA 训练。如果你想训练 LyCORIS（LoCon、LoHa） 等，则修改这个值为 lycoris.kohya
+$network_weights = "" # pretrained weights for LoRA network | 若需要从已有的 LoRA 模型上继续训练，请填写 LoRA 模型路径。
+$network_dim = 32 # network dim | 常用 4~128，不是越大越好
+$network_alpha = 32 # network alpha | 常用与 network_dim 相同的值或者采用较小的值，如 network_dim的一半 防止下溢。默认值为 1，使用较小的 alpha 需要提升学习率。
+
+# Train related params | 训练相关参数
+$resolution = "512,512" # image resolution w,h. 图片分辨率，宽,高。支持非正方形，但必须是 64 倍数。
+$batch_size = 1 # batch size
+$max_train_epoches = 10 # max train epoches | 最大训练 epoch
+$save_every_n_epochs = 2 # save every n epochs | 每 N 个 epoch 保存一次
+
+$train_unet_only = 0 # train U-Net only | 仅训练 U-Net，开启这个会牺牲效果大幅减少显存使用。6G显存可以开启
+$train_text_encoder_only = 0 # train Text Encoder only | 仅训练 文本编码器
+$stop_text_encoder_training = 0 # stop text encoder training | 在第N步时停止训练文本编码器
+
+$noise_offset = 0 # noise offset | 在训练中添加噪声偏移来改良生成非常暗或者非常亮的图像，如果启用，推荐参数为 0.1
+$keep_tokens = 0 # keep heading N tokens when shuffling caption tokens | 在随机打乱 tokens 时，保留前 N 个不变。
+$min_snr_gamma = 0 # minimum signal-to-noise ratio (SNR) value for gamma-ray | 伽马射线事件的最小信噪比（SNR）值  默认为 0
+
+# Learning rate | 学习率
+$lr = "1e-4"
+$unet_lr = "1e-4"
+$text_encoder_lr = "1e-5"
+$lr_scheduler = "cosine_with_restarts" # "linear", "cosine", "cosine_with_restarts", "polynomial", "constant", "constant_with_warmup"
+$lr_warmup_steps = 0 # warmup steps | 学习率预热步数，lr_scheduler 为 constant 或 adafactor 时该值需要设为0。
+$lr_restart_cycles = 1 # cosine_with_restarts restart cycles | 余弦退火重启次数，仅在 lr_scheduler 为 cosine_with_restarts 时起效。
+
+# Output settings | 输出设置
+$output_name = "aki" # output model name | 模型保存名称
+$save_model_as = "safetensors" # model save ext | 模型保存格式 ckpt, pt, safetensors
+
+# Resume training state | 恢复训练设置  
+$save_state = 0 # save training state | 保存训练状态 名称类似于 <output_name>-??????-state ?????? 表示 epoch 数
+$resume = "" # resume from state | 从某个状态文件夹中恢复训练 需配合上方参数同时使用 由于规范文件限制 epoch 数和全局步数不会保存 即使恢复时它们也从 1 开始 与 network_weights 的具体实现操作并不一致
+
+# 其他设置
+$min_bucket_reso = 256 # arb min resolution | arb 最小分辨率
+$max_bucket_reso = 1024 # arb max resolution | arb 最大分辨率
+$persistent_data_loader_workers = 0 # persistent dataloader workers | 容易爆内存，保留加载训练集的worker，减少每个 epoch 之间的停顿
+$clip_skip = 2 # clip skip | 玄学 一般用 2
+$multi_gpu = 0 # multi gpu | 多显卡训练 该参数仅限在显卡数 >= 2 使用
+$lowram = 0 # lowram mode | 低内存模式 该模式下会将 U-net 文本编码器 VAE 转移到 GPU 显存中 启用该模式可能会对显存有一定影响
+
+# 优化器设置
+$optimizer_type = "AdamW8bit" # Optimizer type | 优化器类型 默认为 AdamW8bit，可选：AdamW AdamW8bit Lion SGDNesterov SGDNesterov8bit DAdaptation AdaFactor
+
+# LyCORIS 训练设置
+$algo = "lora" # LyCORIS network algo | LyCORIS 网络算法 可选 lora、loha、lokr、ia3、dylora。lora即为locon
+$conv_dim = 4 # conv dim | 类似于 network_dim，推荐为 4
+$conv_alpha = 4 # conv alpha | 类似于 network_alpha，可以采用与 conv_dim 一致或者更小的值
+$dropout = "0"  # dropout | dropout 概率, 0 为不使用 dropout, 越大则 dropout 越多，推荐 0~0.5， LoHa/LoKr/(IA)^3暂时不支持
+
+# ============= DO NOT MODIFY CONTENTS BELOW | 请勿修改下方内容 =====================
+# Activate python venv
+.\venv\Scripts\activate
+
+$Env:HF_HOME = "huggingface"
+$Env:XFORMERS_FORCE_DISABLE_TRITON = "1"
+$ext_args = [System.Collections.ArrayList]::new()
+$launch_args = [System.Collections.ArrayList]::new()
+
+if ($multi_gpu) {
+  [void]$launch_args.Add("--multi_gpu")
+}
+
+if ($lowram) {
+  [void]$ext_args.Add("--lowram")
+}
+
+if ($is_v2_model) {
+  [void]$ext_args.Add("--v2") 
+}
+else {
+  [void]$ext_args.Add("--clip_skip=$clip_skip")
+}
+
+if ($parameterization) {
+  [void]$ext_args.Add("--v_parameterization")
+}
+
+if ($train_unet_only) {
+  [void]$ext_args.Add("--network_train_unet_only")
+}
+
+if ($train_text_encoder_only) {
+  [void]$ext_args.Add("--network_train_text_encoder_only")
+}
+
+if ($network_weights) {
+  [void]$ext_args.Add("--network_weights=" + $network_weights)
+}
+
+if ($reg_data_dir) {
+  [void]$ext_args.Add("--reg_data_dir=" + $reg_data_dir)
+}
+
+if ($optimizer_type) {
+  [void]$ext_args.Add("--optimizer_type=" + $optimizer_type)
+}
+
+if ($optimizer_type -eq "DAdaptation") {
+  [void]$ext_args.Add("--optimizer_args")
+  [void]$ext_args.Add("decouple=True")
+}
+
+if ($network_module -eq "lycoris.kohya") {
+  [void]$ext_args.Add("--network_args")
+  [void]$ext_args.Add("conv_dim=$conv_dim")
+  [void]$ext_args.Add("conv_alpha=$conv_alpha")
+  [void]$ext_args.Add("algo=$algo")
+  [void]$ext_args.Add("dropout=$dropout")
+}
+
+if ($noise_offset -ne 0) {
+  [void]$ext_args.Add("--noise_offset=$noise_offset")
+}
+
+if ($stop_text_encoder_training -ne 0) {
+  [void]$ext_args.Add("--stop_text_encoder_training=$stop_text_encoder_training")
+}
+
+if ($save_state -eq 1) {
+  [void]$ext_args.Add("--save_state")
+}
+
+if ($resume) {
+  [void]$ext_args.Add("--resume=" + $resume)
+}
+
+if ($min_snr_gamma -ne 0) {
+  [void]$ext_args.Add("--min_snr_gamma=$min_snr_gamma")
+} 
+
+if($persistent_data_loader_workers) {
+  [void]$ext_args.Add("--persistent_data_loader_workers")
+}
+
+# run train
+accelerate launch $launch_args --num_cpu_threads_per_process=8 "./sd-scripts/train_network.py" `
+  --enable_bucket `
+  --pretrained_model_name_or_path=$pretrained_model `
+  --train_data_dir=$train_data_dir `
+  --output_dir="./output" `
+  --logging_dir="./logs" `
+  --log_prefix=$output_name `
+  --resolution=$resolution `
+  --network_module=$network_module `
+  --max_train_epochs=$max_train_epoches `
+  --learning_rate=$lr `
+  --unet_lr=$unet_lr `
+  --text_encoder_lr=$text_encoder_lr `
+  --lr_scheduler=$lr_scheduler `
+  --lr_warmup_steps=$lr_warmup_steps `
+  --lr_scheduler_num_cycles=$lr_restart_cycles `
+  --network_dim=$network_dim `
+  --network_alpha=$network_alpha `
+  --output_name=$output_name `
+  --train_batch_size=$batch_size `
+  --save_every_n_epochs=$save_every_n_epochs `
+  --mixed_precision="fp16" `
+  --save_precision="fp16" `
+  --seed="1337" `
+  --cache_latents `
+  --prior_loss_weight=1 `
+  --max_token_length=225 `
+  --caption_extension=".txt" `
+  --save_model_as=$save_model_as `
+  --min_bucket_reso=$min_bucket_reso `
+  --max_bucket_reso=$max_bucket_reso `
+  --keep_tokens=$keep_tokens `
+  --xformers --shuffle_caption $ext_args
+Write-Output "Train finished"
+Read-Host | Out-Null ;

train.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+# LoRA train script by @Akegarasu
+
+# Train data path | 设置训练用模型、图片
+pretrained_model="./sd-models/model.ckpt" # base model path | 底模路径
+is_v2_model=0                             # SD2.0 model | SD2.0模型 2.0模型下 clip_skip 默认无效
+parameterization=0                        # parameterization | 参数化 本参数需要和 V2 参数同步使用 实验性功能
+train_data_dir="./train/aki"              # train dataset path | 训练数据集路径
+reg_data_dir=""                           # directory for regularization images | 正则化数据集路径，默认不使用正则化图像。
+
+# Network settings | 网络设置
+network_module="networks.lora" # 在这里将会设置训练的网络种类，默认为 networks.lora 也就是 LoRA 训练。如果你想训练 LyCORIS（LoCon、LoHa） 等，则修改这个值为 lycoris.kohya
+network_weights=""             # pretrained weights for LoRA network | 若需要从已有的 LoRA 模型上继续训练，请填写 LoRA 模型路径。
+network_dim=32                 # network dim | 常用 4~128，不是越大越好
+network_alpha=32               # network alpha | 常用与 network_dim 相同的值或者采用较小的值，如 network_dim的一半 防止下溢。默认值为 1，使用较小的 alpha 需要提升学习率。
+
+# Train related params | 训练相关参数
+resolution="512,512"  # image resolution w,h. 图片分辨率，宽,高。支持非正方形，但必须是 64 倍数。
+batch_size=1          # batch size
+max_train_epoches=10  # max train epoches | 最大训练 epoch
+save_every_n_epochs=2 # save every n epochs | 每 N 个 epoch 保存一次
+
+train_unet_only=0         # train U-Net only | 仅训练 U-Net，开启这个会牺牲效果大幅减少显存使用。6G显存可以开启
+train_text_encoder_only=0 # train Text Encoder only | 仅训练 文本编码器
+stop_text_encoder_training=0 # stop text encoder training | 在第N步时停止训练文本编码器
+
+noise_offset="0"  # noise offset | 在训练中添加噪声偏移来改良生成非常暗或者非常亮的图像，如果启用，推荐参数为0.1
+keep_tokens=0   # keep heading N tokens when shuffling caption tokens | 在随机打乱 tokens 时，保留前 N 个不变。
+min_snr_gamma=0 # minimum signal-to-noise ratio (SNR) value for gamma-ray | 伽马射线事件的最小信噪比（SNR）值  默认为 0
+
+# Learning rate | 学习率
+lr="1e-4"
+unet_lr="1e-4"
+text_encoder_lr="1e-5"
+lr_scheduler="cosine_with_restarts" # "linear", "cosine", "cosine_with_restarts", "polynomial", "constant", "constant_with_warmup", "adafactor"
+lr_warmup_steps=0                   # warmup steps | 学习率预热步数，lr_scheduler 为 constant 或 adafactor 时该值需要设为0。
+lr_restart_cycles=1                 # cosine_with_restarts restart cycles | 余弦退火重启次数，仅在 lr_scheduler 为 cosine_with_restarts 时起效。
+
+# Output settings | 输出设置
+output_name="aki"           # output model name | 模型保存名称
+save_model_as="safetensors" # model save ext | 模型保存格式 ckpt, pt, safetensors
+
+# Resume training state | 恢复训练设置
+save_state=0 # save state | 保存训练状态 名称类似于 <output_name>-??????-state ?????? 表示 epoch 数
+resume=""    # resume from state | 从某个状态文件夹中恢复训练 需配合上方参数同时使用 由于规范文件限制 epoch 数和全局步数不会保存 即使恢复时它们也从 1 开始 与 network_weights 的具体实现操作并不一致
+
+# 其他设置
+min_bucket_reso=256              # arb min resolution | arb 最小分辨率
+max_bucket_reso=1024             # arb max resolution | arb 最大分辨率
+persistent_data_loader_workers=0 # persistent dataloader workers | 容易爆内存，保留加载训练集的worker，减少每个 epoch 之间的停顿
+clip_skip=2                      # clip skip | 玄学 一般用 2
+
+# 优化器设置
+optimizer_type="AdamW8bit" # Optimizer type | 优化器类型 默认为 AdamW8bit，可选：AdamW AdamW8bit Lion SGDNesterov SGDNesterov8bit DAdaptation AdaFactor
+
+# LyCORIS 训练设置
+algo="lora"  # LyCORIS network algo | LyCORIS 网络算法 可选 lora、loha、lokr、ia3、dylora。lora即为locon
+conv_dim=4   # conv dim | 类似于 network_dim，推荐为 4
+conv_alpha=4 # conv alpha | 类似于 network_alpha，可以采用与 conv_dim 一致或者更小的值
+dropout="0"  # dropout | dropout 概率, 0 为不使用 dropout, 越大则 dropout 越多，推荐 0~0.5， LoHa/LoKr/(IA)^3暂时不支持
+
+# ============= DO NOT MODIFY CONTENTS BELOW | 请勿修改下方内容 =====================
+export HF_HOME="huggingface"
+export TF_CPP_MIN_LOG_LEVEL=3
+
+extArgs=()
+launchArgs=()
+if [[ $multi_gpu == 1 ]]; then launchArgs+=("--multi_gpu"); fi
+
+if [[ $is_v2_model == 1 ]]; then
+  extArgs+=("--v2");
+else
+  extArgs+=("--clip_skip $clip_skip");
+fi
+
+if [[ $parameterization == 1 ]]; then extArgs+=("--v_parameterization"); fi
+
+if [[ $train_unet_only == 1 ]]; then extArgs+=("--network_train_unet_only"); fi
+
+if [[ $train_text_encoder_only == 1 ]]; then extArgs+=("--network_train_text_encoder_only"); fi
+
+if [[ $network_weights ]]; then extArgs+=("--network_weights $network_weights"); fi
+
+if [[ $reg_data_dir ]]; then extArgs+=("--reg_data_dir $reg_data_dir"); fi
+
+if [[ $optimizer_type ]]; then extArgs+=("--optimizer_type $optimizer_type"); fi
+
+if [[ $optimizer_type == "DAdaptation" ]]; then extArgs+=("--optimizer_args decouple=True"); fi
+
+if [[ $save_state == 1 ]]; then extArgs+=("--save_state"); fi
+
+if [[ $resume ]]; then extArgs+=("--resume $resume"); fi
+
+if [[ $persistent_data_loader_workers == 1 ]]; then extArgs+=("--persistent_data_loader_workers"); fi
+
+if [[ $network_module == "lycoris.kohya" ]]; then
+  extArgs+=("--network_args conv_dim=$conv_dim conv_alpha=$conv_alpha algo=$algo dropout=$dropout")
+fi
+
+if [[ $stop_text_encoder_training -ne 0 ]]; then extArgs+=("--stop_text_encoder_training $stop_text_encoder_training"); fi
+
+if [[ $noise_offset != "0" ]]; then extArgs+=("--noise_offset $noise_offset"); fi
+
+if [[ $min_snr_gamma -ne 0 ]]; then extArgs+=("--min_snr_gamma $min_snr_gamma"); fi
+
+accelerate launch ${launchArgs[@]} --num_cpu_threads_per_process=8 "./sd-scripts/train_network.py" \
+  --enable_bucket \
+  --pretrained_model_name_or_path=$pretrained_model \
+  --train_data_dir=$train_data_dir \
+  --output_dir="./output" \
+  --logging_dir="./logs" \
+  --log_prefix=$output_name \
+  --resolution=$resolution \
+  --network_module=$network_module \
+  --max_train_epochs=$max_train_epoches \
+  --learning_rate=$lr \
+  --unet_lr=$unet_lr \
+  --text_encoder_lr=$text_encoder_lr \
+  --lr_scheduler=$lr_scheduler \
+  --lr_warmup_steps=$lr_warmup_steps \
+  --lr_scheduler_num_cycles=$lr_restart_cycles \
+  --network_dim=$network_dim \
+  --network_alpha=$network_alpha \
+  --output_name=$output_name \
+  --train_batch_size=$batch_size \
+  --save_every_n_epochs=$save_every_n_epochs \
+  --mixed_precision="fp16" \
+  --save_precision="fp16" \
+  --seed="1337" \
+  --cache_latents \
+  --prior_loss_weight=1 \
+  --max_token_length=225 \
+  --caption_extension=".txt" \
+  --save_model_as=$save_model_as \
+  --min_bucket_reso=$min_bucket_reso \
+  --max_bucket_reso=$max_bucket_reso \
+  --keep_tokens=$keep_tokens \
+  --xformers --shuffle_caption ${extArgs[@]}

train_by_toml.ps1

@@ -0,0 +1,33 @@
+# LoRA train script by @Akegarasu
+
+$multi_gpu = 0		 # multi gpu | ���Կ�ѵ�� �ò����������Կ��� >= 2 ʹ��
+$config_file = "./toml/default.toml"		 # config_file | ʹ��toml�ļ�ָ��ѵ������
+$sample_prompts = "./toml/sample_prompts.txt"		 # sample_prompts | ����prompts�ļ�,���������ò�������
+$utf8 = 1		 # utf8 | ʹ��utf-8�����ȡtoml����utf-8�����д�ġ������ĵ�toml���뿪��
+
+
+# ============= DO NOT MODIFY CONTENTS BELOW | �����޸��·����� =====================
+
+# Activate python venv
+.\venv\Scripts\activate
+
+$Env:HF_HOME = "huggingface"
+
+$ext_args = [System.Collections.ArrayList]::new()
+$launch_args = [System.Collections.ArrayList]::new()
+
+if ($multi_gpu) {
+  [void]$launch_args.Add("--multi_gpu")
+}
+if ($utf8 -eq 1) {
+  $Env:PYTHONUTF8 = 1
+}
+
+# run train
+accelerate launch $launch_args --num_cpu_threads_per_process=8 "./sd-scripts/train_network.py" `
+  --config_file=$config_file `
+  --sample_prompts=$sample_prompts `
+  $ext_args
+
+Write-Output "Train finished"
+Read-Host | Out-Null ;

train_by_toml.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# LoRA train script by @Akegarasu
+
+multi_gpu=0                                # multi gpu | 多显卡训练 该参数仅限在显卡数 >= 2 使用
+config_file="./toml/default.toml"          # config_file | 使用toml文件指定训练参数
+sample_prompts="./toml/sample_prompts.txt" # sample_prompts | 采样prompts文件,留空则不启用采样功能
+utf8=1                                     # utf8 | 使用utf-8编码读取toml；以utf-8编码编写的、含中文的toml必须开启
+
+# ============= DO NOT MODIFY CONTENTS BELOW | 请勿修改下方内容 =====================
+
+export HF_HOME="huggingface"
+export TF_CPP_MIN_LOG_LEVEL=3
+
+extArgs=()
+launchArgs=()
+
+if [[ $multi_gpu == 1 ]]; then launchArgs+=("--multi_gpu"); fi
+if [[ $utf8 == 1 ]]; then export PYTHONUTF8=1; fi
+
+# run train
+accelerate launch ${launchArgs[@]} --num_cpu_threads_per_process=8 "./sd-scripts/train_network.py" \
+  --config_file=$config_file \
+  --sample_prompts=$sample_prompts \
+  ${extArgs[@]}