Update app.py

app.py

@@ -151,16 +151,16 @@ retriever = db.as_retriever(
 # )
 
 
-llm = ChatOpenAI(
-    base_url="https://openrouter.ai/api/v1",
-    temperature=0.01,
-    api_key=OPENAPI_KEY,
-    model="google/gemini-2.0-flash-exp:free",
-    max_tokens=None,
-    timeout=None,
-    max_retries=3,
+#llm = ChatOpenAI(
+   # base_url="https://openrouter.ai/api/v1",
+    #temperature=0.01,
+   # api_key=OPENAPI_KEY,
+    #model="google/gemini-2.0-flash-exp:free",
+    #max_tokens=None,
+    #timeout=None,
+  #  max_retries=3,
 
-)
+#)
 
 
 # llm = ChatCerebras(
@@ -171,31 +171,31 @@ llm = ChatOpenAI(
 
 
 
-# quantization_config = BitsAndBytesConfig(
-#         load_in_4bit=True,
-#         bnb_4bit_compute_dtype=torch.bfloat16,
-#         bnb_4bit_quant_type="nf4",
-#         bnb_4bit_use_double_quant=True
-#     )
+ quantization_config = BitsAndBytesConfig(
+         load_in_8bit=True,
+         bnb_8bit_compute_dtype=torch.bfloat16,
+         bnb_8bit_quant_type="nf4",
+         bnb_8bit_use_double_quant=True
+     )
 
 
 
 
-# model_id = "unsloth/phi-4"
-# tokenizer = AutoTokenizer.from_pretrained(model_id)
+  model_id = "mistralai/Mistral-Nemo-Instruct-2407"
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-# model = AutoModelForCausalLM.from_pretrained(
-#         model_id,
-#         torch_dtype=torch.float16,
-#         device_map="cuda",
-#         attn_implementation="flash_attention_2",
-#         quantization_config=quantization_config
+ model = AutoModelForCausalLM.from_pretrained(
+         model_id,
+         torch_dtype=torch.float16,
+         device_map="cuda",
+         attn_implementation="flash_attention_2",
+         quantization_config=quantization_config
 
-#     )
+     )
 
-# pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=8192 )
+ pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=8192 )
 
-# llm = HuggingFacePipeline(pipeline=pipe)
+ llm = HuggingFacePipeline(pipeline=pipe)