first

README.md

@@ -1,10 +1,10 @@
 ---
-title: DS Fusion Express
-emoji: 😻
-colorFrom: gray
-colorTo: purple
+title: DS-Fusion
+emoji: 👁
+colorFrom: purple
+colorTo: blue
 sdk: gradio
-sdk_version: 3.36.1
+sdk_version: 3.32.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -0,0 +1,245 @@
+import gradio as gr
+import os 
+# os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+from categories import categories, font_list
+import pdb 
+from PIL import Image
+import random
+
+try:
+    import pygsheets
+except Exception as e:
+    print("pygsheets not found", e)
+
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:128"
+
+
+global_index = 0
+mode_global = "DS-Fusion-Express"
+prompt_global = ""
+
+def log_data_to_sheet(pre_style, style_custom, glyph, attribute):
+    try:
+        #authorization
+        gc = pygsheets.authorize(service_file='./huggingface-connector-02adcb4cdf00.json')
+
+        #open the google spreadsheet (where 'PY to Gsheet Test' is the name of my sheet)
+        sh = gc.open('HuggingFace Logs')
+
+        #select the first sheet 
+        wks = sh[0]
+
+        # Send fields list
+        wks.append_table(values=[pre_style, style_custom, glyph, attribute])
+    except:
+        pass
+
+def change_font(evt: gr.SelectData):
+    global global_index
+    global_index = evt.index
+
+def my_main(pre_style, style_custom, glyph, attribute):
+
+    log_data_to_sheet(pre_style, style_custom, glyph, attribute)
+
+    global prompt_global
+    glyph = glyph[0]
+    
+    command = "rm -r out_cur/*"
+    os.system(command)
+    for i in range(1,5):
+        command = "cp initial_show/" + str(i) +".png out_cur/"+str(i)+".png"
+        os.system(command)
+
+    style = pre_style
+    
+    command = "rm -r data_style/"
+    os.system(command)
+
+    if style_custom != "":
+        style = style_custom
+
+    if len(glyph) != 1:
+        prompt_global = f" {style}"
+    else:
+        prompt_global = f" {style} {glyph}"
+
+    li = "ckpt/all2.ckpt"
+    output_path = f"out/express"
+    if attribute == "":
+        prompt = f" '{style} {glyph}'"
+    else:
+        prompt = f" '{attribute} {style} {glyph}'"
+
+    command = "rm -r out/"
+    os.system(command)
+    
+    print(prompt)
+    
+    command = "python txt2img.py --ddim_eta 1.0 \
+                        --n_samples 4 \
+                        --n_iter 1\
+                        --ddim_steps 50 \
+                        --scale 5.0\
+                        --H 256\
+                        --W 256\
+                        --outdir " + output_path + " --ckpt " +li +" --prompt " + prompt
+                        
+    os.system(command)
+
+    command = "rm -r out_cur/*"
+    os.system(command)
+    path = []
+    final_imgs = os.listdir(output_path+"/samples")
+    for i in range(4):
+        path.append(os.path.join(output_path+"/samples", final_imgs[i]))
+        path_in = os.path.join(output_path+"/samples", final_imgs[i])
+        command = "cp " + path_in + " " + "out_cur/"+final_imgs[i]
+        os.system(command)
+
+    return gr.update(value=path)
+
+
+
+def rem_bg():
+    command= "rm -r out_bg/*"
+    os.system(command)
+    files = os.listdir("out_cur")
+    if len(files)>0:
+        command_3 = f"python script_step3.py --input_dir out_cur --method 'rembg'"
+        os.system(command_3)
+
+    for file in files:
+        command = "cp out_bg/"+file +" out_cur/"
+        os.system(command)
+    
+    path = []
+    for file in files:
+        file_path = os.path.join("out_cur", file)
+        image = Image.open(file_path)
+        new_image = Image.new("RGBA", image.size, "WHITE") 
+        new_image.paste(image, (0, 0), image)            
+        new_image.save(file_path, "PNG")  
+        path.append(file_path)
+
+    return gr.update(value = path)
+
+font_list_express = [
+    'Caladea-Regular', #works good
+    # 'Garuda-Bold', #works poorlyß
+    'FreeSansOblique', #works average
+    "Purisa", #works good
+    "Uroob" #worksaverage
+    ]
+
+path_fonts_express = []
+for font in font_list_express:
+    path_in = "font_list/fonts/"+font+".png"
+    path_fonts_express.append(path_in)
+    
+def make_upper(value):
+    if value == "":
+        return ""
+    return value[0].upper()
+
+def get_out_cur():
+    path = []
+    pth = "log_view"
+    for file in os.listdir(pth):
+        file_final = os.path.join(pth, file)
+        path.append(file_final)
+    return gr.update(value=path) 
+
+
+def update_time_mode(value):
+    if value == 'DS-Fusion':
+        return gr.update(value="Generation Time: ~5 mins")
+    else:
+        return gr.update(value="Generation Time: ~30 seconds")
+
+def update_time_cb(value):
+    if value:
+        if mode_global == "DS-Fusion":
+            return gr.update(value="Generation Time: ~8 mins")
+        return gr.update(value="Generation Time: ~30 seconds")
+    else:
+        if mode_global == "DS-Fusion":
+            return gr.update(value="Generation Time: ~5 mins")
+        return gr.update(value="Generation Time: ~30 seconds")
+    
+            
+
+def load_img():
+
+    path = []
+    dir = "out_cur"
+    for file in os.listdir(dir):
+        file_full = os.path.join(dir, file)
+        path.append(file_full)
+    return path
+
+css = '''
+<!-- this makes the items inside the gallery to shrink -->
+div#gallery_64 div.grid {
+    height: 64px;
+    width: 180px;
+}
+
+<!-- this makes the gallery's height to shrink -->
+div#gallery_64 > div:nth-child(3) {
+    min-height: 172px !important;
+}
+
+<!-- this makes the gallery's height to shrink when you click one image to view it bigger -->
+div#gallery_64 > div:nth-child(4) {
+    min-height: 172px !important;
+}
+'''
+
+with gr.Blocks(css=css) as demo:
+    
+    with gr.Column(): 
+        with gr.Row():
+            with gr.Column(): 
+                with gr.Row():
+                    in4 = gr.Text(label="Character (A-Z, 0-9) to Stylize", info = "Only works with capitals. Will pick first letter if more than one", value = "R", interactive = True)
+                    in2 = gr.Dropdown(categories, label="Pre-Defined Style Categories", info = "Categories used to train Express", value = "DRAGON", interactive = True)
+                
+                with gr.Row():
+                    in3 = gr.Text(label="Override Style Category ", info="This will replace the pre-defined style value", value = "", interactive = True)
+                    in5 = gr.Text(label="Additional Style Attribute ",info= "e.g. pixel, grayscale, etc", value = "", interactive = True)
+
+                # with gr.Row():
+                #     with gr.Column():
+                #         in8 = gr.Checkbox(label="MULTI FONT INPUT - font selection below is over-ridden", info="Select for more abstract results", value = False, interactive = True).style(container=True)
+                #         gallery = gr.Gallery([], label="Select Font", show_label=True, elem_id="gallery_64").style(grid=[2,6],  preview=True, height="auto")
+                        
+                with gr.Row():
+                    btn = gr.Button("Let's Stylize It - Generation Time: ~60 seconds", interactive = True)
+                    # btn_bg = gr.Button("Remove Background", interactive = True)
+
+            with gr.Column():
+                gallery_out = gr.Gallery(label="Generated images", elem_id="gallery_out").style(grid=[2,2], height="full")
+        
+    
+    
+    outputs = [gallery_out]    
+    # gallery.select(change_font, None, None)
+        
+
+    inputs = [in2,in3,in4,in5]
+    
+    btn.click(my_main, inputs, outputs)
+    # btn_bg.click(rem_bg, None, outputs)
+
+
+
+
+if __name__ == "__main__":
+    command = "rm -r out_cur/*"
+    os.system(command)
+    for i in range(1,5):
+        command = "cp initial_show/" + str(i) +".png out_cur/"+str(i)+".png"
+        os.system(command)
+    demo.queue()
+    demo.launch(share=False)

background_remover/remove_deeplab.py

@@ -0,0 +1,70 @@
+import torch
+import numpy as np
+import urllib
+from PIL import Image
+from torchvision import transforms
+
+def load_model():
+
+    model = torch.hub.load('pytorch/vision:v0.6.0', 'deeplabv3_resnet101', pretrained=True)
+    model.eval()
+
+    mean = torch.tensor([0.485, 0.456, 0.406])
+    std = torch.tensor([0.229, 0.224, 0.225])
+
+    preprocess = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize(mean=mean, std=std),
+    ])
+
+    postprocess = transforms.Compose([
+        transforms.Normalize(mean=-mean/std, std=1/std),
+        transforms.ToPILImage(),
+    ])
+
+    if torch.cuda.is_available():
+        model.to('cuda')
+    return model, preprocess
+
+def remove_background(img, model, preprocess):
+    input_batch = preprocess(img)[None, ...]
+
+    if torch.cuda.is_available():
+        input_batch = input_batch.to('cuda')
+
+    with torch.no_grad():
+        output = model(input_batch)['out'][0]
+        output_predictions = torch.nn.functional.softmax(output, dim=0)
+        output_predictions = (output_predictions > 0.98).float()
+
+    img.putalpha(255)
+    result_np = np.array(img)
+    result_np[..., 3] = (1-output_predictions[0].cpu().numpy())*255
+    
+    return Image.fromarray(result_np.astype('uint8'))
+
+import os
+def main():
+    model, preprocess = load_model()
+    # fpath = 'data/parrot_2.png'
+    path_in = "/localhome/mta122/PycharmProjects/logo_ai/final_nocherry_score/one/DRAGON/G"
+
+    for fpath_file in os.listdir(path_in):
+        # fpath = 'data/parrot_2.png'
+        fpath = os.path.join(path_in, fpath_file)
+        # fpath_out = fpath.split('.')[0] + '_result_rembg.png'
+        # cmd = f'rembg i {fpath} {fpath_out}'
+        # print(cmd)
+        # os.system(cmd)
+
+        img = Image.open(fpath)
+        if img.size[-1] > 3:
+            img_np = np.array(img)
+            img_rbg = img_np[:, : ,:3]
+        img = Image.fromarray(img_rbg)
+        result = remove_background(img, model, preprocess)
+        result.save(fpath.split('.')[0] + '_result_deeplab.png')
+        print('finished')
+
+
+main()

background_remover/remove_rembg.py

@@ -0,0 +1,24 @@
+# pip install rembg[gpu]
+# read details on https://github.com/brilam/remove-bg
+# Paper: U2-Net: Going deeper with nested U-structure for salient object detection
+
+import os
+
+#chess, c
+#surfing, s
+
+objects = "PLANT"
+letter = "L"
+
+
+for mode in ["one","all"]:
+
+    path_in = f"/localhome/mta122/PycharmProjects/logo_ai/final_nocherry_score/{mode}/{objects}/{letter}"
+
+    for fpath_file in os.listdir(path_in):
+        # fpath = 'data/parrot_2.png'
+        fpath = os.path.join(path_in, fpath_file)
+        fpath_out = fpath.split('.')[0] + '_result_rembg.png'
+        cmd = f'rembg i {fpath} {fpath_out}'
+        print(cmd)
+        os.system(cmd)

background_remover/seg.py

@@ -0,0 +1,48 @@
+import torch
+import numpy as np
+model = torch.hub.load('pytorch/vision:v0.10.0', 'deeplabv3_resnet50', pretrained=True)
+# or any of these variants
+# model = torch.hub.load('pytorch/vision:v0.10.0', 'deeplabv3_resnet101', pretrained=True)
+# model = torch.hub.load('pytorch/vision:v0.10.0', 'deeplabv3_mobilenet_v3_large', pretrained=True)
+model.eval()
+
+
+# file name
+filename = './data/icecream_2.png'
+
+# sample execution (requires torchvision)
+from PIL import Image
+from torchvision import transforms
+input_image = Image.open(filename)
+input_image = input_image.convert("RGB")
+preprocess = transforms.Compose([
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+input_tensor = preprocess(input_image)
+input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model
+
+# move the input and model to GPU for speed if available
+if torch.cuda.is_available():
+    input_batch = input_batch.to('cuda')
+    model.to('cuda')
+
+with torch.no_grad():
+    output = model(input_batch)['out'][0]
+output_predictions = output.argmax(0)
+
+
+# create a color pallette, selecting a color for each class
+palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
+colors = torch.as_tensor([i for i in range(21)])[:, None] * palette
+colors = (colors % 255).numpy().astype("uint8")
+
+# plot the semantic segmentation predictions of 21 classes in each color
+r = Image.fromarray(output_predictions.byte().cpu().numpy()).resize(input_image.size)
+r.putpalette(colors)
+r.save('results.png')
+
+import matplotlib.pyplot as plt
+plt.imshow(r)
+# plt.show()

categories.py

@@ -0,0 +1,56 @@
+categories = [
+    "ASTRONAUT",
+    "MERMAID",
+    "BUTTERFLY",
+    "MUSIC",
+    "OCTOPUS",
+    "CANDLE",
+    "PILOT",
+    "CACTUS",
+    "PLANT",
+    "CHESS",
+    "ROBOT",
+    "COW",
+    "SHARK",
+    "DOLPHIN",
+    "SOCKS",
+    "DRAGON",
+    "UNICORN",
+    "LION",
+    "VIOLIN",
+    "ZOMBIE",
+    "PHOENIX",
+    "CORN",
+    "UMBRELLA",
+    "HARP",
+    "LADYBUG",
+    "KAYAK",
+    "CASTLE",
+    "HOME",
+    "PARROT",
+    "STAR",
+    "SURFING",
+    "HAT",
+    "BOW",
+    "BOOK",
+    "MOUSE",
+    "SHELL",
+    "HORSE",
+    "FOX",
+    "PEACOCK",
+    "DOG",
+    "OWL"
+    ]
+
+
+font_list = [
+    'Garuda-Bold', 
+    'Caladea-Regular', 
+    'OpenSans-ExtraBoldItalic', 
+    'Lato-Black', 
+    'FreeSerifBold',
+    'FreeSansOblique',
+    "Purisa",
+    "Karumbi",
+    "mitra",
+    "Uroob"]

ckpt/all2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fb66ea45f7fa37746f4043c8ed6038fc85371755d36e791ad5efa02c4fd1d0
+size 13165056579

configs/finetune/finetune_bert.yaml

@@ -0,0 +1,128 @@
+model:
+  base_learning_rate: 1.0e-5
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image"
+    cond_stage_key: "caption"
+    image_size: 32
+    channels: 4
+    cond_stage_trainable: False   
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    weight_disc: 0.01
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 1280 
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256 
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+    
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.BERTEmbedder
+      params:
+        n_embed: 1280
+        n_layer: 32
+        device: "cuda"
+
+    discriminator_config:
+      target: ldm.modules.discriminator.Discriminator
+      params:
+        bnorm: True
+        leakyparam: 0.2
+        bias: False
+        generic: False
+
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 1
+    num_workers: 32
+    wrap: false
+    train:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 2001
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+    validation:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 5
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+
+lightning:
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 5000
+  callbacks:
+    image_logger:
+      target: main.ImageLogger
+      params:
+        batch_frequency: 1000
+        max_images: 1
+        increase_log_steps: False
+
+  trainer:
+    benchmark: True
+    max_steps: 500

configs/finetune/finetune_clip.yaml

@@ -0,0 +1,118 @@
+model:
+  base_learning_rate: 1.0e-5 #1e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image"
+    cond_stage_key: "caption"
+    image_size: 64 # 32
+    channels: 4
+    cond_stage_trainable: False   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    weight_disc: 0.01
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 64 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768 # 1280
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 512 #256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+    
+    discriminator_config:
+      target: ldm.modules.discriminator.Discriminator64
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 1
+    num_workers: 32
+    wrap: false
+    train:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 2001
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+    validation:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 5
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+
+lightning:
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 200
+  callbacks:
+    image_logger:
+      target: main.ImageLogger
+      params:
+        batch_frequency: 100
+        max_images: 1
+        increase_log_steps: False
+
+  trainer:
+    benchmark: True
+    max_steps: 1001

configs/finetune/finetune_generic.yaml

@@ -0,0 +1,128 @@
+model:
+  base_learning_rate: 1.0e-5
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image"
+    cond_stage_key: "caption"
+    image_size: 32
+    channels: 4
+    cond_stage_trainable: False   
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    weight_disc: 0.01
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 1280 
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256 
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+    
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.BERTEmbedder
+      params:
+        n_embed: 1280
+        n_layer: 32
+        device: "cuda"
+
+    discriminator_config:
+      target: ldm.modules.discriminator.Discriminator
+      params:
+        bnorm: True
+        leakyparam: 0.2
+        bias: False
+        generic: True
+
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 1
+    num_workers: 32
+    wrap: false
+    train:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 2001
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+    validation:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 5
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+
+lightning:
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 5000
+  callbacks:
+    image_logger:
+      target: main.ImageLogger
+      params:
+        batch_frequency: 1000
+        max_images: 1
+        increase_log_steps: False
+
+  trainer:
+    benchmark: True
+    max_steps: 500

configs/finetune/finetune_multi_bert.yaml

@@ -0,0 +1,127 @@
+model:
+  base_learning_rate: 1.0e-5 #1e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image"
+    cond_stage_key: "caption"
+    image_size: 32 # 32
+    channels: 4
+    cond_stage_trainable: False   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    weight_disc: 0.01
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 1280 # 1280
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256 #256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+    
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.BERTEmbedder
+      params:
+        n_embed: 1280
+        n_layer: 32
+
+    discriminator_config:
+      target: ldm.modules.discriminator.Discriminator
+      params:
+        bnorm: True
+        leakyparam: 0.2
+        bias: False
+        generic: False
+
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 1
+    num_workers: 32
+    wrap: false
+    train:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 2001
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+    validation:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 5
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+
+lightning:
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 2000
+  callbacks:
+    image_logger:
+      target: main.ImageLogger
+      params:
+        batch_frequency: 5000
+        max_images: 1
+        increase_log_steps: False
+
+  trainer:
+    benchmark: True
+    max_steps: 800

configs/finetune/finetune_multi_clip.yaml

@@ -0,0 +1,118 @@
+model:
+  base_learning_rate: 1.0e-5 #1e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "image"
+    cond_stage_key: "caption"
+    image_size: 64 # 32
+    channels: 4
+    cond_stage_trainable: False   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    weight_disc: 0.01
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 64 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768 # 1280
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 512 #256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+    
+    discriminator_config:
+      target: ldm.modules.discriminator.Discriminator64
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 1
+    num_workers: 32
+    wrap: false
+    train:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 2001
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+    validation:
+      target: ldm.data.rasterizer.Rasterizer
+      params:
+        img_size: 256
+        text: "R"
+        style_word: "DRAGON"
+        data_path: "data/cat"
+        alternate_glyph: None
+        num_samples: 5
+        make_black: False
+        one_font: False
+        full_word: False
+        font_name: "Garuda-Bold.ttf"
+        just_use_style: false
+        use_alt: False
+
+lightning:
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 200
+  callbacks:
+    image_logger:
+      target: main.ImageLogger
+      params:
+        batch_frequency: 100
+        max_images: 1
+        increase_log_steps: False
+
+  trainer:
+    benchmark: True
+    max_steps: 1501