Update app.py

app.py

@@ -20,16 +20,6 @@ from dotenv import load_dotenv
 import os
 import shutil
 import torch
-
-from transformers import AutoModel,AutoTokenizer
-model2 = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
-tokenizer2 = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
-
-
-# this shoub be used when we can not use sentence_transformers (which reqiures transformers==4.39. we cannot use
-# this version since causes using large amount of RAm when loading falcon model)
-# a custom embedding
-#from sentence_transformers import SentenceTransformer
 from langchain_experimental.text_splitter import SemanticChunker
 from typing import List
 import re
@@ -50,6 +40,27 @@ from transformers import (
     pipeline,
 )
 
+
+MODEL_NAME = "tiiuae/falcon-7b-instruct"
+
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_NAME, trust_remote_code=True, device_map="auto",offload_folder="offload"
+)
+model = model.eval()
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+print(f"Model device: {model.device}")
+
+from transformers import AutoModel,AutoTokenizer
+model2 = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+tokenizer2 = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+
+
+# this shoub be used when we can not use sentence_transformers (which reqiures transformers==4.39. we cannot use
+# this version since causes using large amount of RAm when loading falcon model)
+# a custom embedding
+#from sentence_transformers import SentenceTransformer
+
 warnings.filterwarnings("ignore", category=UserWarning)
 
 
@@ -91,15 +102,7 @@ db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embeddings)
 
 
 
-MODEL_NAME = "tiiuae/falcon-7b-instruct"
 
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_NAME, trust_remote_code=True, device_map="auto",offload_folder="offload"
-)
-model = model.eval()
-
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-print(f"Model device: {model.device}")
 
 
 generation_config = model.generation_config
@@ -224,7 +227,7 @@ def get_llama_response(message: str, history: list) -> str:
 
   #print(template)
   chain.prompt=prompt
-  res = chain.invoke(query_text)
+  res = chain(query_text)
   return(res["response"])
 
 import gradio as gr