Update app.py

app.py

@@ -75,6 +75,13 @@ def load_dataset(file_path, required_columns=[], additional_columns=['popularity
                 else:
                     df[col] = 0.0  # Default for other additional columns
         
+        # Ensure 'level' column has valid values (not empty)
+        if 'level' in df.columns:
+            df['level'] = df['level'].apply(lambda x: 'Intermediate' if pd.isna(x) or x.strip() == "" else x)
+        else:
+            logger.warning(f"'level' column missing in {file_path}. Adding default 'Intermediate'.")
+            df['level'] = 'Intermediate'
+        
         return df
     except ValueError as ve:
         logger.error(f"ValueError loading {file_path}: {ve}. Using fallback data.")
@@ -187,12 +194,12 @@ def precompute_resources():
     course_skills = courses_df['skills'].fillna("").tolist()
     course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
     skill_embeddings = universal_model.encode(questions_df['Skill'].unique().tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
-    course_similarity = util.pytorch_cos_sim(torch.tensor(skill_embeddings), torch.tensor(course_embeddings)).numpy()
+    course_similarity = util.pytorch_cos_sim(skill_embeddings.clone().detach(), course_embeddings.clone().detach()).numpy()
     
     # Precompute job similarities
     job_skills = jobs_df['required_skills'].fillna("").tolist()
     job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-    job_similarity = util.pytorch_cos_sim(torch.tensor(skill_embeddings), torch.tensor(job_embeddings)).numpy()
+    job_similarity = util.pytorch_cos_sim(skill_embeddings.clone().detach(), job_embeddings.clone().detach()).numpy()
     
     # Save precomputed resources
     with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
@@ -236,14 +243,16 @@ def evaluate_response(args):
 def recommend_courses(skills_to_improve, user_level, upgrade=False):
     try:
         if not skills_to_improve or courses_df.empty:
+            logger.info("No skills to improve or courses_df is empty.")
             return []
 
         skill_indices = [list(questions_df['Skill'].unique()).index(skill) for skill in skills_to_improve if skill in questions_df['Skill'].unique()]
         if not skill_indices:
+            logger.info("No matching skill indices found.")
             return []
 
         similarities = course_similarity[skill_indices]
-        # Use get() with default arrays to avoid KeyError
+        # Use default arrays to avoid KeyError
         popularity = courses_df['popularity'].values if 'popularity' in courses_df else np.full(len(courses_df), 0.8)
         completion_rate = courses_df['completion_rate'].values if 'completion_rate' in courses_df else np.full(len(courses_df), 0.7)
         total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * popularity + 0.2 * completion_rate
@@ -251,8 +260,14 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
         target_level = 'Advanced' if upgrade else user_level
         idx = np.argsort(-total_scores)[:5]
         candidates = courses_df.iloc[idx]
-        candidates = candidates[candidates['level'].str.contains(target_level, case=False, na=False)]
-        return candidates[['course_title', 'Organization']].values.tolist()[:3]
+        
+        # Filter by level, but fallback to all courses if none match
+        filtered_candidates = candidates[candidates['level'].str.contains(target_level, case=False, na=False)]
+        if filtered_candidates.empty:
+            logger.warning(f"No courses found for level {target_level}. Returning top courses regardless of level.")
+            filtered_candidates = candidates
+        
+        return filtered_candidates[['course_title', 'Organization']].values.tolist()[:3]
     except Exception as e:
         logger.error(f"Course recommendation error: {e}")
         return []