Update app.py

app.py

@@ -38,9 +38,19 @@ TFIDF_PATH = os.path.join(chosen_model_dir, "tfidf_vectorizer.pkl")
 SKILL_TFIDF_PATH = os.path.join(chosen_model_dir, "skill_tfidf.pkl")
 QUESTION_ANSWER_PATH = os.path.join(chosen_model_dir, "question_to_answer.pkl")
 FAISS_INDEX_PATH = os.path.join(chosen_model_dir, "faiss_index.index")
+ANSWER_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "answer_embeddings.pkl")
 COURSE_SIMILARITY_PATH = os.path.join(chosen_model_dir, "course_similarity.pkl")
 JOB_SIMILARITY_PATH = os.path.join(chosen_model_dir, "job_similarity.pkl")
 
+# Global variables for precomputed data
+tfidf_vectorizer = None
+skill_tfidf = None
+question_to_answer = None
+faiss_index = None
+answer_embeddings = None
+course_similarity = None
+job_similarity = None
+
 # Improved dataset loading with fallback
 def load_dataset(file_path, required_columns=[], fallback_data=None):
     try:
@@ -108,20 +118,13 @@ def load_universal_model():
             logger.info(f"Loading universal model from {UNIVERSAL_MODEL_PATH}")
             return SentenceTransformer(UNIVERSAL_MODEL_PATH)
         else:
-            logger.info(f"Loading universal model: all-MiniLM-L6-v2")
-            model = SentenceTransformer("all-MiniLM-L6-v2")
-            model.save(UNIVERSAL_MODEL_PATH)
-            return model
-    except Exception as e:
-        logger.error(f"Failed to load universal model all-MiniLM-L6-v2: {e}. Falling back to default.")
-        try:
-            logger.info(f"Loading fallback model: {default_model}")
+            logger.info(f"Loading universal model: {default_model}")
             model = SentenceTransformer(default_model)
             model.save(UNIVERSAL_MODEL_PATH)
             return model
-        except Exception as e:
-            logger.error(f"Failed to load fallback model {default_model}: {e}. Exiting.")
-            exit(1)
+    except Exception as e:
+        logger.error(f"Failed to load universal model {default_model}: {e}. Exiting.")
+        exit(1)
 
 universal_model = load_universal_model()
 
@@ -132,23 +135,16 @@ else:
     detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
     detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
 
-# Global variables for precomputed data
-tfidf_vectorizer = None
-skill_tfidf = None
-question_to_answer = None
-faiss_index = None
-course_similarity = None
-job_similarity = None
-
 # Load Precomputed Resources
 def load_precomputed_resources():
-    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, course_similarity, job_similarity
-    if all(os.path.exists(p) for p in [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH, COURSE_SIMILARITY_PATH, JOB_SIMILARITY_PATH]):
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
+    if all(os.path.exists(p) for p in [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH, ANSWER_EMBEDDINGS_PATH, COURSE_SIMILARITY_PATH, JOB_SIMILARITY_PATH]):
         try:
             with open(TFIDF_PATH, 'rb') as f: tfidf_vectorizer = pickle.load(f)
             with open(SKILL_TFIDF_PATH, 'rb') as f: skill_tfidf = pickle.load(f)
             with open(QUESTION_ANSWER_PATH, 'rb') as f: question_to_answer = pickle.load(f)
             faiss_index = faiss.read_index(FAISS_INDEX_PATH)
+            with open(ANSWER_EMBEDDINGS_PATH, 'rb') as f: answer_embeddings = pickle.load(f)
             with open(COURSE_SIMILARITY_PATH, 'rb') as f: course_similarity = pickle.load(f)
             with open(JOB_SIMILARITY_PATH, 'rb') as f: job_similarity = pickle.load(f)
             logger.info("Loaded precomputed resources successfully")
@@ -160,7 +156,7 @@ def load_precomputed_resources():
 
 # Precompute Resources Offline (to be run separately)
 def precompute_resources():
-    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, course_similarity, job_similarity
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
     logger.info("Precomputing resources offline")
     tfidf_vectorizer = TfidfVectorizer(stop_words='english')
     all_texts = questions_df['Answer'].tolist() + questions_df['Question'].tolist()
@@ -189,6 +185,7 @@ def precompute_resources():
     with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
     with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
     faiss.write_index(faiss_index, FAISS_INDEX_PATH)
+    with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
     with open(COURSE_SIMILARITY_PATH, 'wb') as f: pickle.dump(course_similarity, f)
     with open(JOB_SIMILARITY_PATH, 'wb') as f: pickle.dump(job_similarity, f)
     universal_model.save(UNIVERSAL_MODEL_PATH)
@@ -232,7 +229,7 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
             return []
 
         similarities = course_similarity[skill_indices]
-        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * courses_df.get('popularity', 0.8).values + 0.2 * courses_df.get('completion_rate', 0.7).values
+        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * courses_df['popularity'].values + 0.2 * courses_df['completion_rate'].values
 
         target_level = 'Advanced' if upgrade else user_level
         idx = np.argsort(-total_scores)[:5]
@@ -294,7 +291,7 @@ def assess_skills():
         if len(answers) != len(user_skills):
             return jsonify({"error": "Answers count must match skills count"}), 400
 
-        load_precomputed_resources()
+        load_precomputed_resources()  # Load precomputed resources before processing
 
         user_questions = []
         for skill in user_skills: