Update app.py

app.py

@@ -10,6 +10,8 @@ import scipy.special
 from sklearn.feature_extraction.text import TfidfVectorizer
 from flask import Flask, request, jsonify
 import logging
+from pymongo import MongoClient
+import requests
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -18,6 +20,14 @@ logger = logging.getLogger(__name__)
 # Disable tokenizers parallelism to avoid fork-related deadlocks
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
+# MongoDB connection
+MONGO_URI = "mongodb://muhammadbinimran1001:[email protected]:27017,dsm-shard-00-01.inrzs.mongodb.net:27017,dsm-shard-00-02.inrzs.mongodb.net:27017/?ssl=true&replicaSet=atlas-nbg4er-shard-0&authSource=admin&retryWrites=true&w=majority"
+client = MongoClient(MONGO_URI)
+db = client.get_database("test")  # Adjust the database name as needed
+users_collection = db["users"]
+courses_collection = db["courses"]
+jobs_collection = db["jobs"]
+
 # Paths for saving artifacts
 MODEL_DIR = "./saved_models"
 FALLBACK_MODEL_DIR = "/tmp/saved_models"
@@ -58,24 +68,21 @@ def load_dataset(file_path, required_columns=[], additional_columns=['popularity
         missing_required = [col for col in required_columns if col not in df.columns]
         missing_additional = [col for col in additional_columns if col not in df.columns]
         
-        # Handle missing required columns
         if missing_required:
             logger.warning(f"Required columns {missing_required} missing in {file_path}. Adding empty values.")
             for col in missing_required:
                 df[col] = ""
         
-        # Handle missing additional columns (popularity, completion_rate, etc.)
         if missing_additional:
             logger.warning(f"Additional columns {missing_additional} missing in {file_path}. Adding default values.")
             for col in missing_additional:
                 if col == 'popularity':
-                    df[col] = 0.8  # Default value for popularity
+                    df[col] = 0.8
                 elif col == 'completion_rate':
-                    df[col] = 0.7  # Default value for completion_rate
+                    df[col] = 0.7
                 else:
-                    df[col] = 0.0  # Default for other additional columns
+                    df[col] = 0.0
         
-        # Ensure 'level' column has valid values (not empty)
         if 'level' in df.columns:
             df['level'] = df['level'].apply(lambda x: 'Intermediate' if pd.isna(x) or x.strip() == "" else x)
         else:
@@ -104,24 +111,6 @@ questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Qu
     'Answer': ['Linux answer', 'Git answer', 'Node.js answer', 'Python answer', 'Kubernetes answer']
 })
 
-courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"], ['popularity', 'completion_rate'], {
-    'skills': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
-    'course_title': ['Linux Admin', 'Git Mastery', 'Node.js Advanced', 'Python for Data', 'Kubernetes Basics'],
-    'Organization': ['Coursera', 'Udemy', 'Pluralsight', 'edX', 'Linux Foundation'],
-    'level': ['Intermediate', 'Intermediate', 'Advanced', 'Advanced', 'Intermediate'],
-    'popularity': [0.85, 0.9, 0.8, 0.95, 0.9],
-    'completion_rate': [0.65, 0.7, 0.6, 0.8, 0.75]
-})
-
-jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"], [], {
-    'job_title': ['DevOps Engineer', 'Cloud Architect', 'Software Engineer', 'Data Scientist', 'Security Analyst'],
-    'company_name': ['Tech Corp', 'Cloud Inc', 'Tech Solutions', 'Data Co', 'SecuriTech'],
-    'location': ['Remote', 'Islamabad', 'Karachi', 'Remote', 'Islamabad'],
-    'required_skills': ['Linux, Kubernetes', 'AWS, Kubernetes', 'Python, Node.js', 'Python, SQL', 'Cybersecurity, Linux'],
-    'job_description': ['DevOps role description', 'Cloud architecture position', 'Software engineering role', 'Data science position', 'Security analyst role'],
-    'level': ['Intermediate', 'Advanced', 'Intermediate', 'Intermediate', 'Intermediate']
-})
-
 # Validate questions_df
 if questions_df is None or questions_df.empty:
     logger.error("questions_df is empty or could not be loaded. Exiting.")
@@ -175,7 +164,7 @@ def load_precomputed_resources():
     else:
         precompute_resources()
 
-# Precompute Resources Offline (to be run separately)
+# Precompute Resources Offline
 def precompute_resources():
     global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
     logger.info("Precomputing resources offline")
@@ -191,18 +180,6 @@ def precompute_resources():
         faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
         faiss_index.add(answer_embeddings)
         
-        # Precompute course similarities
-        course_skills = courses_df['skills'].fillna("").tolist()
-        course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
-        skill_embeddings = universal_model.encode(questions_df['Skill'].unique().tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
-        course_similarity = util.pytorch_cos_sim(skill_embeddings, course_embeddings).cpu().numpy()
-        
-        # Precompute job similarities
-        job_skills = jobs_df['required_skills'].fillna("").tolist()
-        job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
-        job_similarity = util.pytorch_cos_sim(skill_embeddings, job_embeddings).cpu().numpy()
-        
-        # Save precomputed resources
         with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
         with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
         with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
@@ -243,65 +220,50 @@ def evaluate_response(args):
         logger.error(f"Evaluation error for {skill}: {e}")
         return skill, 0.0, False
 
-# Course recommendation with precomputed similarity
-def recommend_courses(skills_to_improve, user_level, upgrade=False):
+# Fetch questions for given skills
+def get_questions_for_skills(skills):
+    user_questions = []
+    for skill in skills:
+        skill_questions = questions_df[questions_df['Skill'] == skill]
+        if not skill_questions.empty:
+            user_questions.append(skill_questions.sample(1).iloc[0].to_dict())
+        else:
+            user_questions.append({
+                'Skill': skill,
+                'Question': f"What are the best practices for using {skill} in a production environment?",
+                'Answer': f"Best practices for {skill} include proper documentation, monitoring, and security measures."
+            })
+    return user_questions
+
+# Recommend courses from MongoDB
+def recommend_courses_from_mongo(skills_to_improve, user_level, upgrade=False):
     try:
-        if not skills_to_improve or courses_df.empty:
-            logger.info("No skills to improve or courses_df is empty.")
-            return []
-
-        skill_indices = [list(questions_df['Skill'].unique()).index(skill) for skill in skills_to_improve if skill in questions_df['Skill'].unique()]
-        if not skill_indices:
-            logger.info("No matching skill indices found.")
+        if not skills_to_improve:
             return []
 
-        similarities = course_similarity[skill_indices]
-        # Use default arrays to avoid KeyError
-        popularity = courses_df['popularity'].values if 'popularity' in courses_df else np.full(len(courses_df), 0.8)
-        completion_rate = courses_df['completion_rate'].values if 'completion_rate' in courses_df else np.full(len(courses_df), 0.7)
-        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * popularity + 0.2 * completion_rate
-
         target_level = 'Advanced' if upgrade else user_level
-        idx = np.argsort(-total_scores)[:5]
-        candidates = courses_df.iloc[idx]
-        
-        # Filter by level, but fallback to all courses if none match
-        filtered_candidates = candidates[candidates['level'].str.contains(target_level, case=False, na=False)]
-        if filtered_candidates.empty:
-            logger.warning(f"No courses found for level {target_level}. Returning top courses regardless of level.")
-            filtered_candidates = candidates
-        
-        return filtered_candidates[['course_title', 'Organization']].values.tolist()[:3]
+        query = {
+            "skills": {"$in": skills_to_improve},
+            "category": {"$regex": target_level, "$options": "i"}
+        }
+        courses = courses_collection.find(query).limit(3)
+        return [{"title": course["title"], "provider": course.get("provider", "Unknown")} for course in courses]
     except Exception as e:
         logger.error(f"Course recommendation error: {e}")
         return []
 
-# Job recommendation with precomputed similarity
-def recommend_jobs(user_skills, user_level):
+# Recommend jobs from MongoDB
+def recommend_jobs_from_mongo(user_skills, user_level):
     try:
-        if jobs_df.empty:
-            return []
-
-        skill_indices = [list(questions_df['Skill'].unique()).index(skill) for skill in user_skills if skill in questions_df['Skill'].unique()]
-        if not skill_indices:
+        if not user_skills:
             return []
 
-        similarities = job_similarity[skill_indices]
-        total_scores = 0.5 * np.max(similarities, axis=0)
-
-        if 'level' not in jobs_df.columns:
-            jobs_df['level'] = 'Intermediate'
-        level_col = jobs_df['level'].astype(str)
-        level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
-        user_level_num = level_map.get(user_level, 1)
-        level_scores = level_col.apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num)/2)
-
-        location_pref = jobs_df.get('location', pd.Series(['Remote'] * len(jobs_df))).apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7)
-        total_job_scores = total_scores + 0.2 * level_scores + 0.1 * location_pref
-        top_job_indices = np.argsort(-total_job_scores)[:5]
-
-        return [(jobs_df.iloc[i]['job_title'], jobs_df.iloc[i]['company_name'], 
-                jobs_df.iloc[i].get('location', 'Remote')) for i in top_job_indices]
+        query = {
+            "skills": {"$in": user_skills},
+            "status": "active"
+        }
+        jobs = jobs_collection.find(query).limit(5)
+        return [{"jobTitle": job["jobTitle"], "companyName": job["companyName"], "location": job.get("location", "Remote")} for job in jobs]
     except Exception as e:
         logger.error(f"Job recommendation error: {e}")
         return []
@@ -313,13 +275,29 @@ app = Flask(__name__)
 def health_check():
     return jsonify({"status": "active", "model_dir": chosen_model_dir})
 
+@app.route('/get_questions', methods=['POST'])
+def get_questions():
+    try:
+        data = request.get_json()
+        if not data or 'skills' not in data:
+            return jsonify({"error": "Missing skills field"}), 400
+
+        user_skills = [s.strip() for s in data['skills'] if isinstance(s, str)]
+        load_precomputed_resources()
+        questions = get_questions_for_skills(user_skills)
+        return jsonify({"questions": questions})
+    except Exception as e:
+        logger.error(f"Get questions error: {e}")
+        return jsonify({"error": "Internal server error"}), 500
+
 @app.route('/assess', methods=['POST'])
 def assess_skills():
     try:
         data = request.get_json()
-        if not data or 'skills' not in data or 'answers' not in data:
+        if not data or 'skills' not in data or 'answers' not in data or 'userId' not in data:
             return jsonify({"error": "Missing required fields"}), 400
 
+        user_id = data['userId']
         user_skills = [s.strip() for s in data['skills'] if isinstance(s, str)]
         answers = [a.strip() for a in data['answers'] if isinstance(a, str)]
         user_level = data.get('user_level', 'Intermediate').strip()
@@ -327,23 +305,12 @@ def assess_skills():
         if len(answers) != len(user_skills):
             return jsonify({"error": "Answers count must match skills count"}), 400
 
-        load_precomputed_resources()  # Load precomputed resources before processing
-
-        user_questions = []
-        for skill in user_skills:
-            skill_questions = questions_df[questions_df['Skill'] == skill]
-            if not skill_questions.empty:
-                user_questions.append(skill_questions.sample(1).iloc[0])
-            else:
-                user_questions.append({
-                    'Skill': skill,
-                    'Question': f"What are the best practices for using {skill} in a production environment?",
-                    'Answer': f"Best practices for {skill} include proper documentation, monitoring, and security measures."
-                })
-        user_questions = pd.DataFrame(user_questions).reset_index(drop=True)
+        load_precomputed_resources()
+        user_questions = get_questions_for_skills(user_skills)
+        user_questions_df = pd.DataFrame(user_questions).reset_index(drop=True)
 
         user_responses = []
-        for idx, row in user_questions.iterrows():
+        for idx, row in user_questions_df.iterrows():
             answer = answers[idx]
             if not answer or answer.lower() == 'skip':
                 user_responses.append((row['Skill'], None, None))
@@ -366,12 +333,19 @@ def assess_skills():
                 ai_flags[skill] = is_ai
             scores_list.append(score)
 
+        # Update user profile with scores
+        skill_scores = [{"skill": skill, "score": score} for skill, score, _ in results if score > 0]
+        users_collection.update_one(
+            {"_id": user_id},
+            {"$set": {"skillScores": skill_scores}}
+        )
+
         mean_score = np.mean(scores_list) if scores_list else 50
         dynamic_threshold = max(40, mean_score)
         weak_skills = [skill for skill, score in user_scores.items() if score < dynamic_threshold]
 
-        courses = recommend_courses(weak_skills or user_skills, user_level, upgrade=not weak_skills)
-        jobs = recommend_jobs(user_skills, user_level)
+        courses = recommend_courses_from_mongo(weak_skills or user_skills, user_level, upgrade=not weak_skills)
+        jobs = recommend_jobs_from_mongo(user_skills, user_level)
 
         return jsonify({
             "assessment_results": {
@@ -388,8 +362,8 @@ def assess_skills():
                 "weak_skills": weak_skills,
                 "skipped_questions": skipped_questions
             },
-            "recommended_courses": courses[:3],
-            "recommended_jobs": jobs[:5]
+            "recommended_courses": courses,
+            "recommended_jobs": jobs
         })
     except Exception as e:
         logger.error(f"Assessment error: {e}")