Update app.py

app.py

@@ -39,7 +39,9 @@ def load_models():
     try:
         # Load embeddings data with custom persistent_load function
         with open("models/embeddings.pkl", "rb") as file:
-            global_models.embeddings_data = pickle.load(file, persistent_load=persistent_load)
+                unpickler = pickle.Unpickler(file)
+                unpickler.persistent_load = persistent_load
+                global_models.embeddings_data = unpickler.load()
             print("Embeddings data loaded successfully.")
     except pickle.UnpicklingError as e:
         raise HTTPException(status_code=500, detail=f"Unpickling error: {e}")
@@ -49,13 +51,54 @@ def load_models():
 app = FastAPI()
 
 @app.on_event("startup")
-async def startup_event():
-    """
-    Load models at application startup.
-    """
-    print("Loading models...")
-    load_models()
-    print("Models loaded.")
+async def load_models():
+    """Initialize all models and data on startup"""
+    try:
+        # Load embedding models
+        global_models.embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+        global_models.cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2', max_length=512)
+        global_models.semantic_model = SentenceTransformer('all-MiniLM-L6-v2')
+
+        # Load BART models
+        global_models.tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base")
+        global_models.model = BartForConditionalGeneration.from_pretrained("facebook/bart-base")
+
+        # Load Orca model
+        model_name = "M4-ai/Orca-2.0-Tau-1.8B"
+        global_models.tokenizer_f = AutoTokenizer.from_pretrained(model_name)
+        global_models.model_f = AutoModelForCausalLM.from_pretrained(model_name)
+
+        # Load translation models
+        global_models.ar_to_en_tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-ar-en")
+        global_models.ar_to_en_model = AutoModelForSeq2SeqLM.from_pretrained("Helsinki-NLP/opus-mt-ar-en")
+        global_models.en_to_ar_tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-ar")
+        global_models.en_to_ar_model = AutoModelForSeq2SeqLM.from_pretrained("Helsinki-NLP/opus-mt-en-ar")
+
+        # Load Medical NER models
+        global_models.bio_tokenizer = AutoTokenizer.from_pretrained("blaze999/Medical-NER")
+        global_models.bio_model = AutoModelForTokenClassification.from_pretrained("blaze999/Medical-NER")
+
+        # Load embeddings data with proper persistent_load handling
+        try:
+            with open('embeddings.pkl', 'rb') as file:
+                unpickler = pickle.Unpickler(file)
+                unpickler.persistent_load = persistent_load
+                global_models.embeddings_data = unpickler.load()
+        except (FileNotFoundError, pickle.UnpicklingError) as e:
+            print(f"Error loading embeddings data: {e}")
+            raise HTTPException(status_code=500, detail="Failed to load embeddings data.")
+        
+        # Load URL mapping data
+        try:
+            df = pd.read_excel('finalcleaned_excel_file.xlsx')
+            global_models.file_name_to_url = {f"article_{index}.html": url for index, url in enumerate(df['Unnamed: 0'])}
+        except Exception as e:
+            print(f"Error loading URL mapping data: {e}")
+            raise HTTPException(status_code=500, detail="Failed to load URL mapping data.")
+        
+    except Exception as e:
+        print(f"Error loading models: {e}")
+        raise HTTPException(status_code=500, detail="Failed to load models.")
 
 @app.get("/")
 async def root():