app.py

import pandas as pd
import torch
from sentence_transformers import SentenceTransformer, util
import faiss
import numpy as np
import os
import pickle
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import scipy.special
from tqdm import tqdm
from tabulate import tabulate
from sklearn.feature_extraction.text import TfidfVectorizer
from multiprocessing import Pool, cpu_count
from flask import Flask, request, jsonify
import logging

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Paths for saving artifacts
MODEL_DIR = "/data/saved_models"  # Use /data for persistent storage in Hugging Face Spaces
UNIVERSAL_MODEL_PATH = os.path.join(MODEL_DIR, "universal_model")
DETECTOR_MODEL_PATH = os.path.join(MODEL_DIR, "detector_model")
TFIDF_PATH = os.path.join(MODEL_DIR, "tfidf_vectorizer.pkl")
SKILL_TFIDF_PATH = os.path.join(MODEL_DIR, "skill_tfidf.pkl")
QUESTION_ANSWER_PATH = os.path.join(MODEL_DIR, "question_to_answer.pkl")
FAISS_INDEX_PATH = os.path.join(MODEL_DIR, "faiss_index.index")

# Ensure the directory exists with error handling
try:
    os.makedirs(MODEL_DIR, exist_ok=True)
    logger.info(f"Successfully created/accessed directory: {MODEL_DIR}")
except PermissionError as e:
    logger.error(f"Permission denied creating directory {MODEL_DIR}: {e}")
    raise
except Exception as e:
    logger.error(f"Unexpected error creating directory {MODEL_DIR}: {e}")
    raise

# Load Datasets
def load_dataset(file_path, required_columns=[]):
    try:
        df = pd.read_csv(file_path)
        for col in required_columns:
            if col not in df.columns:
                print(f"⚠ Warning: Column '{col}' missing in {file_path}. Using default values.")
                df[col] = "" if col != 'level' else 'Intermediate'
        return df
    except FileNotFoundError:
        print(f"❌ Error: Dataset not found at {file_path}. Exiting.")
        return None

user_df = load_dataset("Updated_User_Profile_Dataset.csv", ["name", "skills", "level"])
questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"])
courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"])
jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"])

# Simulate courses_df with relevant skills
if courses_df is None or 'skills' not in courses_df.columns or courses_df['skills'].str.strip().eq('').all():
    courses_df = pd.DataFrame({
        'skills': ['Docker', 'Jenkins', 'Azure', 'Cybersecurity'],
        'course_title': ['Docker Mastery', 'Jenkins CI/CD', 'Azure Fundamentals', 'Cybersecurity Basics'],
        'Organization': ['Udemy', 'Coursera', 'Microsoft', 'edX'],
        'level': ['Intermediate', 'Intermediate', 'Intermediate', 'Advanced'],
        'popularity': [0.9, 0.85, 0.95, 0.8],
        'completion_rate': [0.7, 0.65, 0.8, 0.6]
    })

# Load or Initialize Models
if os.path.exists(UNIVERSAL_MODEL_PATH):
    universal_model = SentenceTransformer(UNIVERSAL_MODEL_PATH)
else:
    universal_model = SentenceTransformer("all-MiniLM-L6-v2")

if os.path.exists(DETECTOR_MODEL_PATH):
    detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH)
    detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH)
else:
    detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
    detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")

# Precompute Resources with Validation
def resources_valid(saved_skills, current_skills):
    return set(saved_skills) == set(current_skills)

def initialize_resources(user_skills):
    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings
    if (os.path.exists(TFIDF_PATH) and os.path.exists(SKILL_TFIDF_PATH) and 
        os.path.exists(QUESTION_ANSWER_PATH) and os.path.exists(FAISS_INDEX_PATH)):
        with open(TFIDF_PATH, 'rb') as f:
            tfidf_vectorizer = pickle.load(f)
        with open(SKILL_TFIDF_PATH, 'rb') as f:
            skill_tfidf = pickle.load(f)
        with open(QUESTION_ANSWER_PATH, 'rb') as f:
            question_to_answer = pickle.load(f)
        faiss_index = faiss.read_index(FAISS_INDEX_PATH)
        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
        
        if not resources_valid(skill_tfidf.keys(), [s.lower() for s in user_skills]):
            logger.info("⚠ Saved skill TF-IDF mismatch detected. Recomputing resources.")
            tfidf_vectorizer = TfidfVectorizer(stop_words='english')
            all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
            tfidf_vectorizer.fit(all_texts)
            skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
            question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
            answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
            faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
            faiss_index.add(answer_embeddings)
    else:
        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
        all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
        tfidf_vectorizer.fit(all_texts)
        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
        question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
        faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
        faiss_index.add(answer_embeddings)

        with open(TFIDF_PATH, 'wb') as f:
            pickle.dump(tfidf_vectorizer, f)
        with open(SKILL_TFIDF_PATH, 'wb') as f:
            pickle.dump(skill_tfidf, f)
        with open(QUESTION_ANSWER_PATH, 'wb') as f:
            pickle.dump(question_to_answer, f)
        faiss.write_index(faiss_index, FAISS_INDEX_PATH)
        universal_model.save_pretrained(UNIVERSAL_MODEL_PATH)
        detector_model.save_pretrained(DETECTOR_MODEL_PATH)
        detector_tokenizer.save_pretrained(DETECTOR_MODEL_PATH)
        logger.info(f"Models and resources saved to {MODEL_DIR}")

# Evaluate Responses
def evaluate_response(args):
    skill, user_answer, question = args
    if not user_answer:
        return skill, 0, False
    
    inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
    with torch.no_grad():
        logits = detector_model(**inputs).logits
    probs = scipy.special.softmax(logits, axis=1).tolist()[0]
    is_ai_generated = probs[1] > 0.5

    user_embedding = universal_model.encode(user_answer, convert_to_tensor=True)
    expected_answer = question_to_answer.get(question, "")
    expected_embedding = universal_model.encode(expected_answer, convert_to_tensor=True)
    score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100

    user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
    skill_lower = skill.lower()
    skill_vec = skill_tfidf.get(skill_lower, tfidf_vectorizer.transform([skill_lower]).toarray()[0])
    skill_relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
    penalty = min(1.0, max(0.5, skill_relevance))
    score *= penalty

    return skill, round(max(0, score), 2), is_ai_generated

# Recommend Courses
def recommend_courses(skills_to_improve, user_level, upgrade=False):
    if not skills_to_improve:
        return []
    
    skill_embeddings = universal_model.encode(skills_to_improve, convert_to_tensor=True)
    course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), convert_to_tensor=True)
    bert_similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()

    collab_scores = []
    for skill in skills_to_improve:
        overlap = sum(1 for user_skills_str in user_df['skills'] if pd.notna(user_skills_str) and skill.lower() in user_skills_str.lower())
        collab_scores.append(overlap / len(user_df))
    collab_similarities = np.array([collab_scores]).repeat(len(courses_df), axis=0).T

    popularity = courses_df['popularity'].fillna(0.5).to_numpy()
    completion = courses_df['completion_rate'].fillna(0.5).to_numpy()
    total_scores = (0.6 * bert_similarities + 0.2 * collab_similarities + 0.1 * popularity + 0.1 * completion)

    recommended_courses = []
    target_level = 'Advanced' if upgrade else user_level
    for i, skill in enumerate(skills_to_improve):
        top_indices = total_scores[i].argsort()[-5:][::-1]
        candidates = courses_df.iloc[top_indices]
        candidates = candidates[candidates['skills'].str.lower() == skill.lower()]
        if candidates.empty:
            candidates = courses_df.iloc[top_indices]
        candidates.loc[:, "level_match"] = candidates['level'].apply(lambda x: 1 if x == target_level else 0.8 if abs({'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[x] - {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[user_level]) <= 1 else 0.5)
        level_filtered = candidates.sort_values(by="level_match", ascending=False)
        recommended_courses.extend(level_filtered[['course_title', 'Organization']].values.tolist()[:3])
    return list(dict.fromkeys(tuple(course) for course in recommended_courses if course[0].strip()))

# Recommend Jobs
def recommend_jobs(user_skills, user_level):
    job_field = 'required_skills' if 'required_skills' in jobs_df.columns and not jobs_df['required_skills'].str.strip().eq('').all() else 'job_description'
    job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), convert_to_tensor=True)
    user_embedding = universal_model.encode(" ".join(user_skills), convert_to_tensor=True)
    skill_similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]

    level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
    user_level_num = level_map[user_level]
    exp_match = jobs_df['level'].fillna('Intermediate').apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num) / 2) if 'level' in jobs_df.columns else np.ones(len(jobs_df)) * 0.5
    location_pref = jobs_df['location'].apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7).to_numpy()
    industry_embeddings = universal_model.encode(jobs_df['job_title'].fillna(""), convert_to_tensor=True)
    industry_similarities = util.pytorch_cos_sim(user_embedding, industry_embeddings).numpy()[0]

    total_job_scores = (0.5 * skill_similarities + 0.2 * exp_match + 0.1 * location_pref + 0.2 * industry_similarities)
    top_job_indices = total_job_scores.argsort()[-5:][::-1]
    return [(jobs_df.iloc[idx]['job_title'], jobs_df.iloc[idx]['company_name'], jobs_df.iloc[idx]['location']) for idx in top_job_indices]

# Main API Endpoint
app = Flask(__name__)

@app.route('/assess', methods=['POST'])
def assess_skills():
    data = request.get_json()
    if not data or 'user_index' not in data or 'answers' not in data:
        return jsonify({"error": "Invalid input. Provide 'user_index' and 'answers' in JSON body."}), 400

    user_index = int(data['user_index'])
    if user_index < 0 or user_index >= len(user_df):
        return jsonify({"error": "Invalid user index."}), 400

    user_text = user_df.loc[user_index, 'skills']
    user_skills = [skill.strip() for skill in user_text.split(",") if skill.strip()] if isinstance(user_text, str) else ["Python", "SQL"]
    user_name = user_df.loc[user_index, 'name']
    user_level = user_df.loc[user_index, 'level'] if 'level' in user_df.columns and pd.notna(user_df.loc[user_index, 'level']) else 'Intermediate'

    initialize_resources(user_skills)

    filtered_questions = questions_df[questions_df['Skill'].isin(user_skills)]
    if filtered_questions.empty:
        return jsonify({"error": "No matching questions found!"}), 500
    
    user_questions = []
    for skill in user_skills:
        skill_questions = filtered_questions[filtered_questions['Skill'] == skill]
        if not skill_questions.empty:
            user_questions.append(skill_questions.sample(1).iloc[0])
    user_questions = pd.DataFrame(user_questions)

    if len(user_questions) != 4:
        return jsonify({"error": "Not enough questions for all skills!"}), 500

    answers = data['answers']
    if len(answers) != 4:
        return jsonify({"error": "Please provide exactly 4 answers."}), 400

    user_responses = []
    for idx, row in user_questions.iterrows():
        answer = answers[idx]
        if not answer or answer.lower() == 'skip':
            user_responses.append((row['Skill'], None, row['Question']))
        else:
            user_responses.append((row['Skill'], answer, row['Question']))

    with Pool(cpu_count()) as pool:
        eval_args = [(skill, user_code, question) for skill, user_code, question in user_responses if user_code]
        results = pool.map(evaluate_response, eval_args)

    user_scores = {}
    ai_flags = {}
    scores_list = []
    skipped_questions = [f"{skill} ({question})" for skill, user_code, question in user_responses if user_code is None]
    for skill, score, is_ai in results:
        if skill in user_scores:
            user_scores[skill] = max(user_scores[skill], score)
            ai_flags[skill] = ai_flags[skill] or is_ai
        else:
            user_scores[skill] = score
            ai_flags[skill] = is_ai
        scores_list.append(score)

    mean_score = np.mean(scores_list) if scores_list else 50
    dynamic_threshold = max(40, mean_score)
    weak_skills = [skill for skill, score in user_scores.items() if score < dynamic_threshold]

    assessment_results = [
        (skill, f"{'■' * int(score//10)}{'-' * (10 - int(score//10))}", f"{score:.2f}%", "AI-Generated" if ai_flags[skill] else "Human-Written")
        for skill, score in user_scores.items()
    ]
    assessment_output = tabulate(assessment_results, headers=["Skill", "Progress", "Score", "Origin"], tablefmt="grid")
    if skipped_questions:
        assessment_output += f"\nSkipped Questions: {skipped_questions}"
    assessment_output += f"\nMean Score: {mean_score:.2f}, Dynamic Threshold: {dynamic_threshold:.2f}"
    assessment_output += f"\nWeak Skills: {weak_skills if weak_skills else 'None'}"

    skills_to_recommend = weak_skills if weak_skills else user_skills
    upgrade_flag = not weak_skills
    recommended_courses = recommend_courses(skills_to_recommend, user_level, upgrade=upgrade_flag)
    courses_output = tabulate(recommended_courses, headers=["Course", "Organization"], tablefmt="grid") if recommended_courses else "None"

    recommended_jobs = recommend_jobs(user_skills, user_level)
    jobs_output = tabulate(recommended_jobs, headers=["Job Title", "Company", "Location"], tablefmt="grid")

    response = {
        "user_info": f"User: {user_name}\nSkills: {user_skills}\nLevel: {user_level}",
        "assessment_results": assessment_output,
        "recommended_courses": courses_output,
        "recommended_jobs": jobs_output
    }
    return jsonify(response)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=7860)