Update app.py

app.py

@@ -22,21 +22,18 @@ logger = logging.getLogger(__name__)
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 # Paths for saving artifacts
-MODEL_DIR = "./saved_models"  # Primary location in /app/saved_models
-FALLBACK_MODEL_DIR = "/tmp/saved_models"  # Fallback if ./saved_models fails
+MODEL_DIR = "./saved_models"
+FALLBACK_MODEL_DIR = "/tmp/saved_models"
 
-# Try to use the primary directory, fall back to /tmp if needed
+# Directory handling with improved error handling
 try:
     os.makedirs(MODEL_DIR, exist_ok=True)
-    logger.info(f"Successfully created/accessed directory: {MODEL_DIR}")
+    logger.info(f"Using model directory: {MODEL_DIR}")
     chosen_model_dir = MODEL_DIR
-except PermissionError as e:
-    logger.warning(f"Permission denied creating directory {MODEL_DIR}: {e}. Falling back to {FALLBACK_MODEL_DIR}")
+except Exception as e:
+    logger.warning(f"Failed to create {MODEL_DIR}: {e}. Using fallback directory.")
     os.makedirs(FALLBACK_MODEL_DIR, exist_ok=True)
     chosen_model_dir = FALLBACK_MODEL_DIR
-except Exception as e:
-    logger.error(f"Unexpected error creating directory {MODEL_DIR}: {e}")
-    raise
 
 # Update paths based on the chosen directory
 UNIVERSAL_MODEL_PATH = os.path.join(chosen_model_dir, "universal_model")
@@ -46,316 +43,245 @@ SKILL_TFIDF_PATH = os.path.join(chosen_model_dir, "skill_tfidf.pkl")
 QUESTION_ANSWER_PATH = os.path.join(chosen_model_dir, "question_to_answer.pkl")
 FAISS_INDEX_PATH = os.path.join(chosen_model_dir, "faiss_index.index")
 
-# Load Datasets
+# Improved dataset loading with fallback
 def load_dataset(file_path, required_columns=[]):
     try:
         df = pd.read_csv(file_path)
         for col in required_columns:
             if col not in df.columns:
                 logger.warning(f"Column '{col}' missing in {file_path}. Using default values.")
-                df[col] = "" if col != 'level' else 'Intermediate'
+                df[col] = ""
         return df
-    except FileNotFoundError:
-        logger.error(f"Dataset not found at {file_path}. Exiting.")
+    except Exception as e:
+        logger.error(f"Error loading {file_path}: {e}")
         return None
 
-questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"])
-courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"])
-jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"])
-
-# Simulate courses_df with relevant skills
-if courses_df is None or 'skills' not in courses_df.columns or courses_df['skills'].str.strip().eq('').all():
-    courses_df = pd.DataFrame({
-        'skills': ['Docker', 'Jenkins', 'Azure', 'Cybersecurity'],
-        'course_title': ['Docker Mastery', 'Jenkins CI/CD', 'Azure Fundamentals', 'Cybersecurity Basics'],
-        'Organization': ['Udemy', 'Coursera', 'Microsoft', 'edX'],
-        'level': ['Intermediate', 'Intermediate', 'Intermediate', 'Advanced'],
-        'popularity': [0.9, 0.85, 0.95, 0.8],
-        'completion_rate': [0.7, 0.65, 0.8, 0.6]
-    })
-
-# Validate questions_df
-if questions_df is None or questions_df.empty:
-    logger.error("questions_df is empty or could not be loaded. Exiting.")
-    exit(1)
-if not all(col in questions_df.columns for col in ["Skill", "Question", "Answer"]):
-    logger.error("questions_df is missing required columns. Exiting.")
-    exit(1)
-logger.info(f"questions_df loaded with {len(questions_df)} rows. Skills available: {questions_df['Skill'].unique().tolist()}")
-
-# Load or Initialize Models
-if os.path.exists(UNIVERSAL_MODEL_PATH):
-    universal_model = SentenceTransformer(UNIVERSAL_MODEL_PATH)
-else:
-    universal_model = SentenceTransformer("all-MiniLM-L6-v2")
-
-if os.path.exists(DETECTOR_MODEL_PATH):
-    detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH)
-    detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH)
-else:
-    detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
-    detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
-
-# Precompute Resources with Validation
-def resources_valid(saved_skills, current_skills):
-    return set(saved_skills) == set(current_skills)
+# Load datasets with fallbacks
+questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"]) or pd.DataFrame({
+    'Skill': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
+    'Question': ['Advanced Linux question', 'Advanced Git question', 'Basic Node.js question', 
+                 'Intermediate Python question', 'Basic Kubernetes question'],
+    'Answer': ['Linux answer', 'Git answer', 'Node.js answer', 'Python answer', 'Kubernetes answer']
+})
+
+courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"]) or pd.DataFrame({
+    'skills': ['Docker', 'Jenkins', 'Azure', 'Cybersecurity'],
+    'course_title': ['Docker Mastery', 'Jenkins CI/CD', 'Azure Fundamentals', 'Cybersecurity Basics'],
+    'Organization': ['Udemy', 'Coursera', 'Microsoft', 'edX'],
+    'level': ['Intermediate', 'Intermediate', 'Intermediate', 'Advanced'],
+    'popularity': [0.9, 0.85, 0.95, 0.8],
+    'completion_rate': [0.7, 0.65, 0.8, 0.6]
+})
+
+jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"]) or pd.DataFrame({
+    'job_title': ['DevOps Engineer', 'Cloud Architect'],
+    'company_name': ['Tech Corp', 'Cloud Inc'],
+    'location': ['Remote', 'Silicon Valley'],
+    'required_skills': ['Linux, Cloud', 'AWS, Kubernetes'],
+    'job_description': ['DevOps role description', 'Cloud architecture position']
+})
+
+# Model loading with validation
+def load_model(model_class, path, default_name):
+    try:
+        return model_class.from_pretrained(path)
+    except Exception as e:
+        logger.warning(f"Failed to load model from {path}: {e}. Using default {default_name}.")
+        return model_class.from_pretrained(default_name)
 
+universal_model = SentenceTransformer(UNIVERSAL_MODEL_PATH) if os.path.exists(UNIVERSAL_MODEL_PATH) else SentenceTransformer("all-MiniLM-L6-v2")
+detector_model = load_model(AutoModelForSequenceClassification, DETECTOR_MODEL_PATH, "roberta-base-openai-detector")
+detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH) if os.path.exists(DETECTOR_MODEL_PATH) else AutoTokenizer.from_pretrained("roberta-base-openai-detector")
+
+# Enhanced resource initialization
 def initialize_resources(user_skills):
     global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings
-    if (os.path.exists(TFIDF_PATH) and os.path.exists(SKILL_TFIDF_PATH) and 
-        os.path.exists(QUESTION_ANSWER_PATH) and os.path.exists(FAISS_INDEX_PATH)):
-        with open(TFIDF_PATH, 'rb') as f:
-            tfidf_vectorizer = pickle.load(f)
-        with open(SKILL_TFIDF_PATH, 'rb') as f:
-            skill_tfidf = pickle.load(f)
-        with open(QUESTION_ANSWER_PATH, 'rb') as f:
-            question_to_answer = pickle.load(f)
-        faiss_index = faiss.read_index(FAISS_INDEX_PATH)
-        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
-        
-        if not resources_valid(skill_tfidf.keys(), [s.lower() for s in user_skills]):
-            logger.info("⚠ Saved skill TF-IDF mismatch detected. Recomputing resources.")
-            tfidf_vectorizer = TfidfVectorizer(stop_words='english')
-            all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
-            tfidf_vectorizer.fit(all_texts)
-            skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
-            question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
-            answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
-            faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
-            faiss_index.add(answer_embeddings)
+    
+    user_skills_lower = [s.lower() for s in user_skills]
+    needs_recompute = False
+
+    if all(os.path.exists(p) for p in [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH]):
+        try:
+            with open(TFIDF_PATH, 'rb') as f:
+                tfidf_vectorizer = pickle.load(f)
+            with open(SKILL_TFIDF_PATH, 'rb') as f:
+                skill_tfidf = pickle.load(f)
+            with open(QUESTION_ANSWER_PATH, 'rb') as f:
+                question_to_answer = pickle.load(f)
+            faiss_index = faiss.read_index(FAISS_INDEX_PATH)
+            
+            if set(skill_tfidf.keys()) != set(user_skills_lower):
+                logger.info("Skill mismatch detected, recomputing resources")
+                needs_recompute = True
+        except Exception as e:
+            logger.error(f"Error loading saved resources: {e}")
+            needs_recompute = True
     else:
+        needs_recompute = True
+
+    if needs_recompute:
+        logger.info("Building new resources")
         tfidf_vectorizer = TfidfVectorizer(stop_words='english')
         all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
         tfidf_vectorizer.fit(all_texts)
-        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
+        
+        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in user_skills}
         question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
-        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
+        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True).cpu().numpy()
+        
         faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
         faiss_index.add(answer_embeddings)
-
-        with open(TFIDF_PATH, 'wb') as f:
-            pickle.dump(tfidf_vectorizer, f)
-        with open(SKILL_TFIDF_PATH, 'wb') as f:
-            pickle.dump(skill_tfidf, f)
-        with open(QUESTION_ANSWER_PATH, 'wb') as f:
-            pickle.dump(question_to_answer, f)
+        
+        # Save resources
+        with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
+        with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
+        with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
         faiss.write_index(faiss_index, FAISS_INDEX_PATH)
-        universal_model.save_pretrained(UNIVERSAL_MODEL_PATH)
-        detector_model.save_pretrained(DETECTOR_MODEL_PATH)
-        detector_tokenizer.save_pretrained(DETECTOR_MODEL_PATH)
-        logger.info(f"Models and resources saved to {chosen_model_dir}")
+        universal_model.save(UNIVERSAL_MODEL_PATH)
+        logger.info(f"Resources saved to {chosen_model_dir}")
 
-# Evaluate Responses
+# Enhanced evaluation with error handling
 def evaluate_response(args):
-    skill, user_answer, question = args
-    if not user_answer:
-        return skill, 0, False
-    
-    inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
-    with torch.no_grad():
-        logits = detector_model(**inputs).logits
-    probs = scipy.special.softmax(logits, axis=1).tolist()[0]
-    is_ai_generated = probs[1] > 0.5
-
-    user_embedding = universal_model.encode(user_answer, convert_to_tensor=True)
-    expected_answer = question_to_answer.get(question, "")
-    expected_embedding = universal_model.encode(expected_answer, convert_to_tensor=True)
-    score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
-
-    user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
-    skill_lower = skill.lower()
-    skill_vec = skill_tfidf.get(skill_lower, tfidf_vectorizer.transform([skill_lower]).toarray()[0])
-    skill_relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
-    penalty = min(1.0, max(0.5, skill_relevance))
-    score *= penalty
-
-    logger.debug(f"Evaluated {skill}: score={score:.2f}, is_ai={is_ai_generated}")
-    return skill, round(max(0, score), 2), is_ai_generated
-
-# Recommend Courses
+    try:
+        skill, user_answer, question = args
+        if not user_answer:
+            return skill, 0.0, False
+
+        inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
+        with torch.no_grad():
+            logits = detector_model(**inputs).logits
+        probs = scipy.special.softmax(logits, axis=1).tolist()[0]
+        is_ai = probs[1] > 0.5
+
+        expected_answer = question_to_answer.get(question, "")
+        user_embedding = universal_model.encode(user_answer, convert_to_tensor=True)
+        expected_embedding = universal_model.encode(expected_answer, convert_to_tensor=True)
+        score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
+
+        user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
+        skill_vec = skill_tfidf.get(skill.lower(), np.zeros_like(user_tfidf))
+        relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
+        score *= max(0.5, min(1.0, relevance))
+
+        return skill, round(max(0, score), 2), is_ai
+    except Exception as e:
+        logger.error(f"Evaluation error for {skill}: {e}")
+        return skill, 0.0, False
+
+# Improved course recommendation
 def recommend_courses(skills_to_improve, user_level, upgrade=False):
-    if not skills_to_improve:
+    try:
+        if not skills_to_improve or courses_df.empty:
+            return []
+
+        # Add missing columns if needed
+        if 'popularity' not in courses_df:
+            courses_df['popularity'] = 0.8
+        if 'completion_rate' not in courses_df:
+            courses_df['completion_rate'] = 0.7
+
+        skill_embeddings = universal_model.encode(skills_to_improve, convert_to_tensor=True)
+        course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), convert_to_tensor=True)
+        similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()
+
+        total_scores = 0.6 * similarities + 0.2 * courses_df['popularity'].values + 0.2 * courses_df['completion_rate'].values
+
+        recommendations = []
+        target_level = 'Advanced' if upgrade else user_level
+        for i, skill in enumerate(skills_to_improve):
+            idx = np.argsort(-total_scores[i])[:5]
+            candidates = courses_df.iloc[idx]
+            candidates = candidates[candidates['level'].str.contains(target_level, case=False)]
+            recommendations.extend(candidates[['course_title', 'Organization']].values.tolist()[:3])
+
+        return list(dict.fromkeys(map(tuple, recommendations)))
+    except Exception as e:
+        logger.error(f"Course recommendation error: {e}")
         return []
-    
-    skill_embeddings = universal_model.encode(skills_to_improve, convert_to_tensor=True)
-    course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), convert_to_tensor=True)
-    bert_similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()
-
-    collab_scores = []
-    for skill in skills_to_improve:
-        overlap = 1  # Simplified since user_df is removed
-        collab_scores.append(overlap)
-    collab_similarities = np.array([collab_scores]).repeat(len(courses_df), axis=0).T
-
-    popularity = courses_df['popularity'].fillna(0.5).to_numpy()
-    completion = courses_df['completion_rate'].fillna(0.5).to_numpy()
-    total_scores = (0.6 * bert_similarities + 0.2 * collab_similarities + 0.1 * popularity + 0.1 * completion)
-
-    recommended_courses = []
-    target_level = 'Advanced' if upgrade else user_level
-    for i, skill in enumerate(skills_to_improve):
-        top_indices = total_scores[i].argsort()[-5:][::-1]
-        candidates = courses_df.iloc[top_indices]
-        candidates = candidates[candidates['skills'].str.lower() == skill.lower()]
-        if candidates.empty:
-            candidates = courses_df.iloc[top_indices]
-        candidates.loc[:, "level_match"] = candidates['level'].apply(lambda x: 1 if x == target_level else 0.8 if abs({'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[x] - {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[user_level]) <= 1 else 0.5)
-        level_filtered = candidates.sort_values(by="level_match", ascending=False)
-        recommended_courses.extend(level_filtered[['course_title', 'Organization']].values.tolist()[:3])
-    return list(dict.fromkeys(tuple(course) for course in recommended_courses if course[0].strip()))
-
-# Recommend Jobs
+
+# Enhanced job recommendation
 def recommend_jobs(user_skills, user_level):
-    job_field = 'required_skills' if 'required_skills' in jobs_df.columns and not jobs_df['required_skills'].str.strip().eq('').all() else 'job_description'
-    job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), convert_to_tensor=True)
-    user_embedding = universal_model.encode(" ".join(user_skills), convert_to_tensor=True)
-    skill_similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
-
-    level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
-    user_level_num = level_map[user_level]
-    exp_match = jobs_df['level'].fillna('Intermediate').apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num) / 2) if 'level' in jobs_df.columns else np.ones(len(jobs_df)) * 0.5
-    location_pref = jobs_df['location'].apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7).to_numpy()
-    industry_embeddings = universal_model.encode(jobs_df['job_title'].fillna(""), convert_to_tensor=True)
-    industry_similarities = util.pytorch_cos_sim(user_embedding, industry_embeddings).numpy()[0]
-
-    total_job_scores = (0.5 * skill_similarities + 0.2 * exp_match + 0.1 * location_pref + 0.2 * industry_similarities)
-    top_job_indices = total_job_scores.argsort()[-5:][::-1]
-    return [(jobs_df.iloc[idx]['job_title'], jobs_df.iloc[idx]['company_name'], jobs_df.iloc[idx]['location']) for idx in top_job_indices]
-
-# Main API Endpoint
+    try:
+        if jobs_df.empty:
+            return []
+
+        job_field = 'required_skills' if 'required_skills' in jobs_df.columns else 'job_description'
+        job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), convert_to_tensor=True)
+        user_embedding = universal_model.encode(" ".join(user_skills), convert_to_tensor=True)
+        similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
+
+        level_scores = jobs_df.get('level', 'Intermediate').apply(
+            lambda x: 1 - abs({'Beginner':0, 'Intermediate':1, 'Advanced':2}.get(x,1) - 
+                            {'Beginner':0, 'Intermediate':1, 'Advanced':2}[user_level])/2
+        )
+        total_scores = 0.6 * similarities + 0.4 * level_scores
+        top_idx = np.argsort(-total_scores)[:5]
+        
+        return [(jobs_df.iloc[i]['job_title'], jobs_df.iloc[i]['company_name'], 
+               jobs_df.iloc[i].get('location', 'Remote')) for i in top_idx]
+    except Exception as e:
+        logger.error(f"Job recommendation error: {e}")
+        return []
+
+# Flask application setup
 app = Flask(__name__)
 
+@app.route('/')
+def health_check():
+    return jsonify({"status": "active", "model_dir": chosen_model_dir})
+
 @app.route('/assess', methods=['POST'])
 def assess_skills():
-    data = request.get_json()
-    logger.info(f"Received request: {data}")
-    
-    # Validate required fields
-    if not data or 'user_name' not in data or 'skills' not in data or 'answers' not in data:
-        logger.error("Invalid input: Missing 'user_name', 'skills', or 'answers' in JSON body.")
-        return jsonify({"error": "Invalid input. Provide 'user_name', 'skills', and 'answers' in JSON body."}), 400
-
-    user_name = data['user_name']
-    user_skills = data['skills']
-    answers = data['answers']
-
-    # Validate inputs
-    if not isinstance(user_name, str) or not user_name.strip():
-        logger.error("Invalid user_name: Must be a non-empty string.")
-        return jsonify({"error": "Invalid user_name. Must be a non-empty string."}), 400
-
-    if not isinstance(user_skills, list) or not user_skills or not all(isinstance(skill, str) and skill.strip() for skill in user_skills):
-        logger.error("Invalid skills: Must be a non-empty list of non-empty strings.")
-        return jsonify({"error": "Invalid skills. Must be a non-empty list of non-empty strings."}), 400
-
-    if not isinstance(answers, list):
-        logger.error(f"Answers must be a list, got: {type(answers)}")
-        return jsonify({"error": "Answers must be a list."}), 400
-
-    # Ensure the number of answers matches the number of skills
-    if len(answers) != len(user_skills):
-        logger.error(f"Number of answers ({len(answers)}) does not match number of skills ({len(user_skills)}).")
-        return jsonify({"error": f"Number of answers ({len(answers)}) must match the number of skills ({len(user_skills)})."}), 400
-
-    user_level = 'Intermediate'  # Default level since user_df is removed
-    logger.info(f"User: {user_name}, Skills: {user_skills}, Level: {user_level}")
-
-    initialize_resources(user_skills)
-
-    # Normalize skills for case-insensitive matching
-    filtered_questions = questions_df[questions_df['Skill'].str.lower().isin([skill.lower() for skill in user_skills])]
-    logger.info(f"Filtered questions shape: {filtered_questions.shape}")
-    logger.info(f"Available skills in questions_df: {filtered_questions['Skill'].unique().tolist()}")
-    if filtered_questions.empty:
-        logger.error("No matching questions found for the user's skills.")
-        return jsonify({"error": "No matching questions found!"}), 500
-    
-    user_questions = []
-    for skill in user_skills:
-        skill_questions = filtered_questions[filtered_questions['Skill'].str.lower() == skill.lower()]
-        logger.info(f"Questions for skill '{skill}': {len(skill_questions)}")
-        if not skill_questions.empty:
-            user_questions.append(skill_questions.sample(1).iloc[0])
-        else:
-            logger.warning(f"No questions found for skill '{skill}'. Using a default question.")
-            user_questions.append({
-                'Skill': skill,
-                'Question': f"What are the best practices for using {skill} in a production environment?",
-                'Answer': f"Best practices for {skill} include proper documentation, monitoring, and security measures."
-            })
-    user_questions = pd.DataFrame(user_questions).reset_index(drop=True)  # Reset index to ensure sequential indices
-    logger.info(f"Selected questions: {user_questions[['Skill', 'Question']].to_dict(orient='records')}")
-    logger.info(f"Number of selected questions: {len(user_questions)}")
-
-    if len(user_questions) != len(user_skills):
-        logger.error(f"Number of selected questions ({len(user_questions)}) does not match number of skills ({len(user_skills)}).")
-        return jsonify({"error": f"Internal error: Number of selected questions ({len(user_questions)}) does not match number of skills ({len(user_skills)})."}), 500
-
-    user_responses = []
-    for idx, row in user_questions.iterrows():
-        logger.debug(f"Pairing question for skill '{row['Skill']}' with answer at index {idx}")
-        if idx >= len(answers):
-            logger.error(f"Index out of range: idx={idx}, len(answers)={len(answers)}")
-            return jsonify({"error": f"Internal error: Index {idx} out of range for answers list of length {len(answers)}."}), 500
-        answer = answers[idx]
-        if not answer or answer.lower() == 'skip':
-            user_responses.append((row['Skill'], None, row['Question']))
-        else:
-            user_responses.append((row['Skill'], answer, row['Question']))
-
     try:
-        with Pool(cpu_count()) as pool:
-            eval_args = [(skill, user_code, question) for skill, user_code, question in user_responses if user_code]
-            logger.info(f"Evaluating {len(eval_args)} answers using multiprocessing pool.")
-            results = pool.map(evaluate_response, eval_args)
-            logger.info(f"Evaluation results: {results}")
+        data = request.get_json()
+        if not data or 'skills' not in data or 'answers' not in data:
+            return jsonify({"error": "Missing required fields"}), 400
+
+        user_skills = [s.strip() for s in data['skills'] if isinstance(s, str)]
+        answers = [a.strip() for a in data['answers'] if isinstance(a, str)]
+        user_level = data.get('user_level', 'Intermediate').strip()
+
+        if len(answers) != len(user_skills):
+            return jsonify({"error": "Answers count must match skills count"}), 400
+
+        initialize_resources(user_skills)
+
+        # Get relevant questions
+        user_questions = questions_df[questions_df['Skill'].str.lower().isin([s.lower() for s in user_skills])]
+        if user_questions.empty:
+            user_questions = questions_df.sample(len(user_skills))
+        
+        user_questions = user_questions.sample(len(user_skills)).reset_index(drop=True)
+        responses = list(zip(user_questions['Skill'], answers, user_questions['Question']))
+
+        # Parallel processing with error handling
+        with Pool(processes=min(cpu_count(), 4)) as pool:
+            results = pool.map(evaluate_response, responses)
+
+        # Process results
+        assessment = []
+        scores = []
+        for skill, score, is_ai in results:
+            assessment.append(f"{skill}: {score}% ({'AI' if is_ai else 'Human'})")
+            scores.append(score)
+        
+        mean_score = np.mean(scores) if scores else 0
+        weak_skills = [skill for skill, score, _ in results if score < max(60, mean_score)]
+
+        # Generate recommendations
+        courses = recommend_courses(weak_skills or user_skills, user_level, upgrade=not weak_skills)
+        jobs = recommend_jobs(user_skills, user_level)
+
+        return jsonify({
+            "assessment": assessment,
+            "mean_score": round(mean_score, 1),
+            "weak_skills": weak_skills,
+            "courses": courses[:3],  # Top 3 courses
+            "jobs": jobs[:5]        # Top 5 jobs
+        })
     except Exception as e:
-        logger.error(f"Error in evaluate_response: {str(e)}", exc_info=True)
-        return jsonify({"error": "Failed to evaluate answers due to an internal error."}), 500
-
-    user_scores = {}
-    ai_flags = {}
-    scores_list = []
-    skipped_questions = [f"{skill} ({question})" for skill, user_code, question in user_responses if user_code is None]
-    for skill, score, is_ai in results:
-        if skill in user_scores:
-            user_scores[skill] = max(user_scores[skill], score)
-            ai_flags[skill] = ai_flags[skill] or is_ai
-        else:
-            user_scores[skill] = score
-            ai_flags[skill] = is_ai
-        scores_list.append(score)
-
-    mean_score = np.mean(scores_list) if scores_list else 50
-    dynamic_threshold = max(40, mean_score)
-    weak_skills = [skill for skill, score in user_scores.items() if score < dynamic_threshold]
-
-    assessment_results = [
-        (skill, f"{'■' * int(score//10)}{'-' * (10 - int(score//10))}", f"{score:.2f}%", "AI-Generated" if ai_flags[skill] else "Human-Written")
-        for skill, score in user_scores.items()
-    ]
-    assessment_output = tabulate(assessment_results, headers=["Skill", "Progress", "Score", "Origin"], tablefmt="grid")
-    if skipped_questions:
-        assessment_output += f"\nSkipped Questions: {skipped_questions}"
-    assessment_output += f"\nMean Score: {mean_score:.2f}, Dynamic Threshold: {dynamic_threshold:.2f}"
-    assessment_output += f"\nWeak Skills: {weak_skills if weak_skills else 'None'}"
-
-    skills_to_recommend = weak_skills if weak_skills else user_skills
-    upgrade_flag = not weak_skills
-    recommended_courses = recommend_courses(skills_to_recommend, user_level, upgrade=upgrade_flag)
-    courses_output = tabulate(recommended_courses, headers=["Course", "Organization"], tablefmt="grid") if recommended_courses else "None"
-
-    recommended_jobs = recommend_jobs(user_skills, user_level)
-    jobs_output = tabulate(recommended_jobs, headers=["Job Title", "Company", "Location"], tablefmt="grid")
-
-    response = {
-        "user_info": f"User: {user_name}\nSkills: {user_skills}\nLevel: {user_level}",
-        "assessment_results": assessment_output,
-        "recommended_courses": courses_output,
-        "recommended_jobs": jobs_output
-    }
-    logger.info(f"Response: {response}")
-    return jsonify(response)
+        logger.error(f"Assessment error: {e}")
+        return jsonify({"error": "Internal server error"}), 500
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=7860)
+    app.run(host='0.0.0.0', port=7860, threaded=True)