Update app.py

app.py

@@ -179,38 +179,42 @@ def load_precomputed_resources():
 def precompute_resources():
     global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
     logger.info("Precomputing resources offline")
-    tfidf_vectorizer = TfidfVectorizer(stop_words='english')
-    all_texts = questions_df['Answer'].tolist() + questions_df['Question'].tolist()
-    tfidf_vectorizer.fit(all_texts)
-    
-    skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in questions_df['Skill'].unique()}
-    question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
-    answer_embeddings = universal_model.encode(questions_df['Answer'].tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-    
-    faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
-    faiss_index.add(answer_embeddings)
-    
-    # Precompute course similarities
-    course_skills = courses_df['skills'].fillna("").tolist()
-    course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-    skill_embeddings = universal_model.encode(questions_df['Skill'].unique().tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
-    course_similarity = util.pytorch_cos_sim(skill_embeddings.clone().detach(), course_embeddings.clone().detach()).numpy()
-    
-    # Precompute job similarities
-    job_skills = jobs_df['required_skills'].fillna("").tolist()
-    job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-    job_similarity = util.pytorch_cos_sim(skill_embeddings.clone().detach(), job_embeddings.clone().detach()).numpy()
-    
-    # Save precomputed resources
-    with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
-    with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
-    with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
-    faiss.write_index(faiss_index, FAISS_INDEX_PATH)
-    with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
-    with open(COURSE_SIMILARITY_PATH, 'wb') as f: pickle.dump(course_similarity, f)
-    with open(JOB_SIMILARITY_PATH, 'wb') as f: pickle.dump(job_similarity, f)
-    universal_model.save(UNIVERSAL_MODEL_PATH)
-    logger.info(f"Precomputed resources saved to {chosen_model_dir}")
+    try:
+        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
+        all_texts = questions_df['Answer'].tolist() + questions_df['Question'].tolist()
+        tfidf_vectorizer.fit(all_texts)
+        
+        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in questions_df['Skill'].unique()}
+        question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
+        answer_embeddings = universal_model.encode(questions_df['Answer'].tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
+        
+        faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
+        faiss_index.add(answer_embeddings)
+        
+        # Precompute course similarities
+        course_skills = courses_df['skills'].fillna("").tolist()
+        course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        skill_embeddings = universal_model.encode(questions_df['Skill'].unique().tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        course_similarity = util.pytorch_cos_sim(skill_embeddings, course_embeddings).cpu().numpy()
+        
+        # Precompute job similarities
+        job_skills = jobs_df['required_skills'].fillna("").tolist()
+        job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        job_similarity = util.pytorch_cos_sim(skill_embeddings, job_embeddings).cpu().numpy()
+        
+        # Save precomputed resources
+        with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
+        with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
+        with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
+        faiss.write_index(faiss_index, FAISS_INDEX_PATH)
+        with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
+        with open(COURSE_SIMILARITY_PATH, 'wb') as f: pickle.dump(course_similarity, f)
+        with open(JOB_SIMILARITY_PATH, 'wb') as f: pickle.dump(job_similarity, f)
+        universal_model.save(UNIVERSAL_MODEL_PATH)
+        logger.info(f"Precomputed resources saved to {chosen_model_dir}")
+    except Exception as e:
+        logger.error(f"Error during precomputation: {e}")
+        raise
 
 # Evaluation with precomputed data
 def evaluate_response(args):