Update app.py

app.py

@@ -52,15 +52,29 @@ course_similarity = None
 job_similarity = None
 
 # Improved dataset loading with fallback
-def load_dataset(file_path, required_columns=[], fallback_data=None):
+def load_dataset(file_path, required_columns=[], additional_columns=['popularity', 'completion_rate'], fallback_data=None):
     try:
         df = pd.read_csv(file_path)
-        missing_columns = [col for col in required_columns if col not in df.columns]
-        if missing_columns:
-            logger.warning(f"Columns {missing_columns} missing in {file_path}. Using default values.")
-            for col in required_columns:
-                if col not in df.columns:
-                    df[col] = ""
+        missing_required = [col for col in required_columns if col not in df.columns]
+        missing_additional = [col for col in additional_columns if col not in df.columns]
+        
+        # Handle missing required columns
+        if missing_required:
+            logger.warning(f"Required columns {missing_required} missing in {file_path}. Adding empty values.")
+            for col in missing_required:
+                df[col] = ""
+        
+        # Handle missing additional columns (popularity, completion_rate, etc.)
+        if missing_additional:
+            logger.warning(f"Additional columns {missing_additional} missing in {file_path}. Adding default values.")
+            for col in missing_additional:
+                if col == 'popularity':
+                    df[col] = 0.8  # Default value for popularity
+                elif col == 'completion_rate':
+                    df[col] = 0.7  # Default value for completion_rate
+                else:
+                    df[col] = 0.0  # Default for other additional columns
+        
         return df
     except ValueError as ve:
         logger.error(f"ValueError loading {file_path}: {ve}. Using fallback data.")
@@ -76,14 +90,14 @@ def load_dataset(file_path, required_columns=[], fallback_data=None):
         return None
 
 # Load datasets with fallbacks
-questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"], {
+questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"], [], {
     'Skill': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
     'Question': ['Advanced Linux question', 'Advanced Git question', 'Basic Node.js question', 
                  'Intermediate Python question', 'Basic Kubernetes question'],
     'Answer': ['Linux answer', 'Git answer', 'Node.js answer', 'Python answer', 'Kubernetes answer']
 })
 
-courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"], {
+courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"], ['popularity', 'completion_rate'], {
     'skills': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
     'course_title': ['Linux Admin', 'Git Mastery', 'Node.js Advanced', 'Python for Data', 'Kubernetes Basics'],
     'Organization': ['Coursera', 'Udemy', 'Pluralsight', 'edX', 'Linux Foundation'],
@@ -92,7 +106,7 @@ courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "
     'completion_rate': [0.65, 0.7, 0.6, 0.8, 0.75]
 })
 
-jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"], {
+jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"], [], {
     'job_title': ['DevOps Engineer', 'Cloud Architect', 'Software Engineer', 'Data Scientist', 'Security Analyst'],
     'company_name': ['Tech Corp', 'Cloud Inc', 'Tech Solutions', 'Data Co', 'SecuriTech'],
     'location': ['Remote', 'Islamabad', 'Karachi', 'Remote', 'Islamabad'],
@@ -229,7 +243,10 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
             return []
 
         similarities = course_similarity[skill_indices]
-        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * courses_df['popularity'].values + 0.2 * courses_df['completion_rate'].values
+        # Use get() with default arrays to avoid KeyError
+        popularity = courses_df['popularity'].values if 'popularity' in courses_df else np.full(len(courses_df), 0.8)
+        completion_rate = courses_df['completion_rate'].values if 'completion_rate' in courses_df else np.full(len(courses_df), 0.7)
+        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * popularity + 0.2 * completion_rate
 
         target_level = 'Advanced' if upgrade else user_level
         idx = np.argsort(-total_scores)[:5]