Update app.py

app.py

@@ -68,20 +68,21 @@ questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Qu
 })
 
 courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"], {
-    'skills': ['Docker', 'Jenkins', 'Azure', 'Cybersecurity'],
-    'course_title': ['Docker Mastery', 'Jenkins CI/CD', 'Azure Fundamentals', 'Cybersecurity Basics'],
-    'Organization': ['Udemy', 'Coursera', 'Microsoft', 'edX'],
-    'level': ['Intermediate', 'Intermediate', 'Intermediate', 'Advanced'],
-    'popularity': [0.9, 0.85, 0.95, 0.8],
-    'completion_rate': [0.7, 0.65, 0.8, 0.6]
+    'skills': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
+    'course_title': ['Linux Admin', 'Git Mastery', 'Node.js Advanced', 'Python for Data', 'Kubernetes Basics'],
+    'Organization': ['Coursera', 'Udemy', 'Pluralsight', 'edX', 'Linux Foundation'],
+    'level': ['Intermediate', 'Intermediate', 'Advanced', 'Advanced', 'Intermediate'],
+    'popularity': [0.85, 0.9, 0.8, 0.95, 0.9],
+    'completion_rate': [0.65, 0.7, 0.6, 0.8, 0.75]
 })
 
 jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"], {
-    'job_title': ['DevOps Engineer', 'Cloud Architect'],
-    'company_name': ['Tech Corp', 'Cloud Inc'],
-    'location': ['Remote', 'Silicon Valley'],
-    'required_skills': ['Linux, Cloud', 'AWS, Kubernetes'],
-    'job_description': ['DevOps role description', 'Cloud architecture position']
+    'job_title': ['DevOps Engineer', 'Cloud Architect', 'Software Engineer', 'Data Scientist', 'Security Analyst'],
+    'company_name': ['Tech Corp', 'Cloud Inc', 'Tech Solutions', 'Data Co', 'SecuriTech'],
+    'location': ['Remote', 'Islamabad', 'Karachi', 'Remote', 'Islamabad'],
+    'required_skills': ['Linux, Kubernetes', 'AWS, Kubernetes', 'Python, Node.js', 'Python, SQL', 'Cybersecurity, Linux'],
+    'job_description': ['DevOps role description', 'Cloud architecture position', 'Software engineering role', 'Data science position', 'Security analyst role'],
+    'level': ['Intermediate', 'Advanced', 'Intermediate', 'Intermediate', 'Intermediate']  # Added level for job recommendations
 })
 
 # Validate questions_df
@@ -156,7 +157,7 @@ def initialize_resources(user_skills):
         universal_model.save(UNIVERSAL_MODEL_PATH)
         logger.info(f"Resources saved to {chosen_model_dir}")
 
-# Enhanced evaluation with error handling
+# Enhanced evaluation with batch processing
 def evaluate_response(args):
     try:
         skill, user_answer, question = args
@@ -170,9 +171,8 @@ def evaluate_response(args):
         is_ai = probs[1] > 0.5
 
         expected_answer = question_to_answer.get(question, "")
-        user_embedding = universal_model.encode(user_answer, convert_to_tensor=True)
-        expected_embedding = universal_model.encode(expected_answer, convert_to_tensor=True)
-        score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
+        user_embeddings = universal_model.encode([user_answer, expected_answer], batch_size=32, convert_to_tensor=True)
+        score = util.pytorch_cos_sim(user_embeddings[0], user_embeddings[1]).item() * 100
 
         user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
         skill_vec = skill_tfidf.get(skill.lower(), np.zeros_like(user_tfidf))
@@ -184,20 +184,20 @@ def evaluate_response(args):
         logger.error(f"Evaluation error for {skill}: {e}")
         return skill, 0.0, False
 
-# Improved course recommendation
+# Improved course recommendation with batch processing
 def recommend_courses(skills_to_improve, user_level, upgrade=False):
     try:
         if not skills_to_improve or courses_df.empty:
             return []
 
-        # Add missing columns if needed
         if 'popularity' not in courses_df:
             courses_df['popularity'] = 0.8
         if 'completion_rate' not in courses_df:
             courses_df['completion_rate'] = 0.7
 
-        skill_embeddings = universal_model.encode(skills_to_improve, convert_to_tensor=True)
-        course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), convert_to_tensor=True)
+        # Batch encode skills and courses
+        skill_embeddings = universal_model.encode(skills_to_improve, batch_size=32, convert_to_tensor=True)
+        course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), batch_size=32, convert_to_tensor=True)
         similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()
 
         total_scores = 0.6 * similarities + 0.2 * courses_df['popularity'].values + 0.2 * courses_df['completion_rate'].values
@@ -207,7 +207,7 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
         for i, skill in enumerate(skills_to_improve):
             idx = np.argsort(-total_scores[i])[:5]
             candidates = courses_df.iloc[idx]
-            candidates = candidates[candidates['level'].str.contains(target_level, case=False)]
+            candidates = candidates[candidates['level'].str.contains(target_level, case=False, na=False)]
             recommendations.extend(candidates[['course_title', 'Organization']].values.tolist()[:3])
 
         return list(dict.fromkeys(map(tuple, recommendations)))
@@ -215,26 +215,35 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
         logger.error(f"Course recommendation error: {e}")
         return []
 
-# Enhanced job recommendation
+# Enhanced job recommendation with fixed level handling
 def recommend_jobs(user_skills, user_level):
     try:
         if jobs_df.empty:
             return []
 
         job_field = 'required_skills' if 'required_skills' in jobs_df.columns else 'job_description'
-        job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), convert_to_tensor=True)
-        user_embedding = universal_model.encode(" ".join(user_skills), convert_to_tensor=True)
-        similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
-
-        level_scores = jobs_df.get('level', 'Intermediate').apply(
-            lambda x: 1 - abs({'Beginner':0, 'Intermediate':1, 'Advanced':2}.get(x,1) - 
-                            {'Beginner':0, 'Intermediate':1, 'Advanced':2}[user_level])/2
-        )
-        total_scores = 0.6 * similarities + 0.4 * level_scores
-        top_idx = np.argsort(-total_scores)[:5]
-        
+        job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), batch_size=32, convert_to_tensor=True)
+        user_embedding = universal_model.encode(" ".join(user_skills), batch_size=32, convert_to_tensor=True)
+        skill_similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
+
+        # Ensure level column exists and is a Series
+        if 'level' not in jobs_df.columns:
+            jobs_df['level'] = 'Intermediate'
+        level_col = jobs_df['level'].astype(str)  # Ensure it's a string Series
+
+        level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
+        user_level_num = level_map.get(user_level, 1)
+        level_scores = level_col.apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num)/2)
+
+        location_pref = jobs_df.get('location', pd.Series(['Remote'] * len(jobs_df))).apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7)
+        industry_embeddings = universal_model.encode(jobs_df['job_title'].fillna(""), batch_size=32, convert_to_tensor=True)
+        industry_similarities = util.pytorch_cos_sim(user_embedding, industry_embeddings).numpy()[0]
+
+        total_job_scores = 0.5 * skill_similarities + 0.2 * level_scores + 0.1 * location_pref + 0.2 * industry_similarities
+        top_job_indices = np.argsort(-total_job_scores)[:5]
+
         return [(jobs_df.iloc[i]['job_title'], jobs_df.iloc[i]['company_name'], 
-               jobs_df.iloc[i].get('location', 'Remote')) for i in top_idx]
+                jobs_df.iloc[i].get('location', 'Remote')) for i in top_job_indices]
     except Exception as e:
         logger.error(f"Job recommendation error: {e}")
         return []