Update app.py

app.py

@@ -22,17 +22,18 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 HF_TOKEN = os.getenv("HF_TOKEN")
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
-model_name = "estrogen/c4ai-command-r7b-12-2024"
-
-model = Llama3ForCausalLM.from_pretrained(
+model_name = "Qwen/Qwen2.5-1.5B-Instruct"
+'''
+model = AutoModelForCausalLM.from_pretrained(
     model_name,
     torch_dtype=torch.float16,
     trust_remote_code=True
 )
+'''
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-#peft_model = AutoPeftModelForCausalLM.from_pretrained("ehristoforu/think-lora-qwen-r64")
-#merged_model = peft_model.merge_and_unload()
+peft_model = AutoPeftModelForCausalLM.from_pretrained("ehristoforu/fd-lora-64x128", torch_dtype=torch.float16, trust_remote_code=True)
+merged_model = peft_model.merge_and_unload()
 #merged_model.save_pretrained("./coolqwen")
 #model.save_pretrained("./coolqwen")
 #tokenizer.save_pretrained("./coolqwen")
@@ -81,7 +82,7 @@ def generate(
         num_beams=1,
         repetition_penalty=repetition_penalty,
     )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t = Thread(target=merged_model.generate, kwargs=generate_kwargs)
     t.start()
 
     outputs = []