3

app.py

@@ -9,9 +9,9 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 from datasets import load_dataset
 
 huggingface_hub.login(os.getenv('HF_TOKEN'))
-peft_model_id = "debisoft/DeepSeek-R1-Distill-Qwen-7B-thinking-function_calling-quant-V0"
+#peft_model_id = "debisoft/DeepSeek-R1-Distill-Qwen-7B-thinking-function_calling-quant-V0"
 #peft_model_id = "debisoft/Qwen2.5-VL-7B-Instruct-thinking-function_calling-quant-V0"
-#peft_model_id = "debisoft/Qwen2.5-VL-3B-Instruct-thinking-function_calling-V0"
+peft_model_id = "debisoft/Qwen2.5-VL-3B-Instruct-thinking-function_calling-V0"
 
 bnb_config = BitsAndBytesConfig(
             load_in_4bit=True,
@@ -25,8 +25,8 @@ cuda_device = torch.device("cuda")
 cpu_device = torch.device("cpu")
 
 config = PeftConfig.from_pretrained(peft_model_id)
-#model = Qwen2_5_VLForConditionalGeneration.from_pretrained(config.base_model_name_or_path,
-model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
+#model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
+model = Qwen2_5_VLForConditionalGeneration.from_pretrained(config.base_model_name_or_path,
                                              quantization_config=bnb_config,
                                              device_map="auto",
                                              )
@@ -52,7 +52,7 @@ def sentience_check():
 
     with torch.no_grad():
         outputs = peft_model.generate(
-            **inputs, max_new_tokens=500, pad_token_id = tokenizer.eos_token_id
+            **inputs, max_new_tokens=1024, pad_token_id = tokenizer.eos_token_id
         )
 
     #peft_model.to(cpu_device)