Update app.py

app.py

@@ -121,41 +121,41 @@ retriever = db.as_retriever(
 
 
 # Set up the LLM
-# llm = ChatOpenAI(
-#     base_url="https://api-inference.huggingface.co/v1/",
-#     temperature=0,
-#     api_key=HF_TOKEN,
-#     model="meta-llama/Llama-3.3-70B-Instruct",
-#     max_tokens=None,
-#     timeout=None
+llm = ChatOpenAI(
+    base_url="https://api-inference.huggingface.co/v1/",
+    temperature=0,
+    api_key=HF_TOKEN,
+    model="mistralai/Mistral-Nemo-Instruct-2407",
+    max_tokens=None,
+    timeout=None
     
-# )
+)
 
-quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.bfloat16,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=True
-    )
+# quantization_config = BitsAndBytesConfig(
+#         load_in_4bit=True,
+#         bnb_4bit_compute_dtype=torch.bfloat16,
+#         bnb_4bit_quant_type="nf4",
+#         bnb_4bit_use_double_quant=True
+#     )
 
 
 
 
-model_id = "unsloth/phi-4"
-tokenizer = AutoTokenizer.from_pretrained(model_id)
+# model_id = "unsloth/phi-4"
+# tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-model = AutoModelForCausalLM.from_pretrained(
-        model_id,
-        torch_dtype=torch.float16,
-        device_map="cuda",
-        attn_implementation="flash_attention_2",
-        quantization_config=quantization_config
+# model = AutoModelForCausalLM.from_pretrained(
+#         model_id,
+#         torch_dtype=torch.float16,
+#         device_map="cuda",
+#         attn_implementation="flash_attention_2",
+#         quantization_config=quantization_config
 
-    )
+#     )
 
-pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=8192 )
+# pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=8192 )
 
-llm = HuggingFacePipeline(pipeline=pipe)
+# llm = HuggingFacePipeline(pipeline=pipe)
 
 
 
@@ -201,7 +201,7 @@ def create_rag_chain(chat_history: str):
 chat_history = ChatHistory()
 
 # Gradio Function
-@spaces.GPU()
+# @spaces.GPU()
 def ask_question_gradio(question, history):
     try:
         # Add user question to chat history