Update app.py

app.py

@@ -51,34 +51,54 @@ conv_mode = 'mistral_instruct_system' # Diese Variable wird noch für die Konver
 load_8bit = False
 load_4bit = False
 
-# Laden Sie den Prozessor, der Tokenizer und Bildprozessor kombiniert
-processor = AutoProcessor.from_pretrained(model_path)
-
-# Laden Sie das Modell mit der korrekten Klasse
-model = LlavaMistralForCausalLM.from_pretrained(
-    model_path,
-    torch_dtype=torch.bfloat16, # Ihre config.json spezifiziert bfloat16
-    low_cpu_mem_usage=True,     # Empfohlen für große Modelle
-    load_in_4bit=load_4bit,
-    load_in_8bit=load_8bit,
-)
-
-# Weisen Sie die Komponenten den alten Variablennamen zu, damit der restliche Code funktioniert
-from transformers import AutoTokenizer, AutoImageProcessor
-
-tokenizer = AutoTokenizer.from_pretrained(model_path)
-try:
-    image_processor = AutoImageProcessor.from_pretrained(model_path)
-    if image_processor is None:
-        raise Exception()
-except Exception:
-    # Fallback: Passender Vision Tower!
-    image_processor = AutoImageProcessor.from_pretrained("openai/clip-vit-large-patch14-336")
-assert image_processor is not None, "Could not load image_processor!"
-
-
-# Setzen Sie die Kontextlänge (falls der restliche Code sie benötigt)
-context_len = model.config.max_position_embeddings
+import sys
+import os
+import argparse
+import time
+import subprocess
+import spaces
+import cumo.serve.gradio_web_server as gws
+
+import datetime
+import json
+
+import gradio as gr
+import requests
+from PIL import Image
+
+from cumo.conversation import (default_conversation, conv_templates, SeparatorStyle)
+from cumo.constants import LOGDIR
+from cumo.utils import (build_logger, server_error_msg, violates_moderation, moderation_msg)
+import hashlib
+
+import torch
+import io
+from cumo.constants import WORKER_HEART_BEAT_INTERVAL
+from cumo.utils import (build_logger, server_error_msg,
+    pretty_print_semaphore)
+from cumo.model.builder import load_pretrained_model
+from cumo.mm_utils import process_images, load_image_from_base64, tokenizer_image_token
+from cumo.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from transformers import TextIteratorStreamer
+from threading import Thread
+
+# Execute the pip install command with additional options
+#subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'flash-attn', '--no-build-isolation', '-U']
+
+headers = {"User-Agent": "CuMo"}
+
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True)
+disable_btn = gr.Button(interactive=False)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_path = 'BenkHel/CumoThesis'
+model_base = 'mistralai/Mistral-7B-Instruct-v0.2'
+model_name = 'CuMo-mistral-7b'
+conv_mode = 'mistral_instruct_system'
+load_8bit = False
+load_4bit = False
+tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_base, model_name, load_8bit, load_4bit, device=device, use_flash_attn=False)
 model.config.training = False
              
 def upvote_last_response(state):
@@ -146,10 +166,9 @@ def generate(state, imagebox, textbox, image_process_mode, temperature, top_p, m
             images = process_images(images, image_processor, model.config)
 
             if type(images) is list:
-                images = [image.to(model.device, dtype=torch.bfloat16) for image in images]
+                images = [image.to(model.device, dtype=torch.float16) for image in images]
             else:
-                images = images.to(model.device, dtype=torch.bfloat16)
-
+                images = images.to(model.device, dtype=torch.float16)
 
             replace_token = DEFAULT_IMAGE_TOKEN
             if getattr(model.config, 'mm_use_im_start_end', False):