from flask import Flask, render_template, request, redirect, url_for, flash, send_file import os import pandas as pd from werkzeug.utils import secure_filename from joblib import load import numpy as np from sklearn.preprocessing import LabelEncoder from time import time app = Flask(__name__) # Set the secret key for session management app.secret_key = os.urandom(24) # Configurations UPLOAD_FOLDER = "uploads/" DATA_FOLDER = "data/" # Define the model directory and label encoder directory MODEL_DIR = r'./Model' LABEL_ENOCDER_DIR = r'./Label_encoders' # Global file names for outputs; these will be updated per prediction. PRED_OUTPUT_FILE = "data/pred_output.csv" CLASS_OUTPUT_FILE = "data/class_output.csv" ALLOWED_EXTENSIONS = {'csv', 'xlsx'} app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) # ------------------------------ # Load Models and Label Encoders # ------------------------------ gia_model = load(os.path.join(MODEL_DIR, 'linear_regression_model_gia_price.joblib')) grade_model = load(os.path.join(MODEL_DIR, 'linear_regression_model_grade_price.joblib')) bygrade_model = load(os.path.join(MODEL_DIR, 'linear_regression_model_bygrade_price.joblib')) makable_model = load(os.path.join(MODEL_DIR, 'linear_regression_model_makable_price.joblib')) col_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_col.joblib')) cts_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cts.joblib')) cut_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cut.joblib')) qua_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_qua.joblib')) shp_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_shp.joblib')) blk_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_blk.joblib')) wht_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_wht.joblib')) open_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_open.joblib')) pav_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_pav.joblib')) blk_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_blk.joblib')) wht_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_wht.joblib')) open_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_open.joblib')) pav_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_pav.joblib')) blk_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_blk.joblib')) wht_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_wht.joblib')) open_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_open.joblib')) pav_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_pav.joblib')) blk_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_blk.joblib')) wht_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_wht.joblib')) open_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_open.joblib')) pav_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_pav.joblib')) encoder_list = ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngLab','EngBlk', 'EngWht', 'EngOpen','EngPav', 'Change_cts_value', 'Change_shape_value', 'Change_quality_value', 'Change_color_value', 'Change_cut_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Mkbl_value', 'Change_Open_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Grd_value', 'Change_Wht_Eng_to_Grd_value', 'Change_Open_Eng_to_Grd_value', 'Change_Pav_Eng_to_Grd_value', 'Change_Blk_Eng_to_ByGrd_value', 'Change_Wht_Eng_to_ByGrd_value', 'Change_Open_Eng_to_ByGrd_value', 'Change_Pav_Eng_to_ByGrd_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Gia_value', 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Gia_value'] loaded_label_encoder = {} for val in encoder_list: encoder_path = os.path.join(LABEL_ENOCDER_DIR, f"label_encoder_{val}.joblib") loaded_label_encoder[val] = load(encoder_path) # ------------------------------ # Utility: Allowed File Check # ------------------------------ def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS # ------------------------------ # Routes # ------------------------------ @app.route('/') def index(): return render_template('index.html') @app.route('/predict', methods=['POST']) def predict(): if 'file' not in request.files: flash('No file part', 'error') return redirect(request.url) file = request.files['file'] if file.filename == '': flash('No selected file', 'error') return redirect(request.url) if file and allowed_file(file.filename): filename = secure_filename(file.filename) filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) file.save(filepath) # Convert file to DataFrame if filename.endswith('.csv'): df = pd.read_csv(filepath) else: df = pd.read_excel(filepath) # Process the DataFrame and generate predictions and classification analysis. df_pred, dx_class = process_dataframe(df) # Save output files with a timestamp (you can also store in session if needed) current_date = pd.Timestamp.now().strftime("%Y-%m-%d") global PRED_OUTPUT_FILE, CLASS_OUTPUT_FILE PRED_OUTPUT_FILE = f'data/prediction_output_{current_date}.csv' CLASS_OUTPUT_FILE = f'data/classification_output_{current_date}.csv' df_pred.to_csv(PRED_OUTPUT_FILE, index=False) dx_class.to_csv(CLASS_OUTPUT_FILE, index=False) # Redirect to report view; default to prediction report, page 1. return redirect(url_for('report_view', report_type='pred', page=1)) else: flash('Invalid file type. Only CSV and Excel files are allowed.', 'error') return redirect(request.url) def process_dataframe(df): try: # Define the columns needed for two parts required_columns = ['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngAmt'] required_columns_2 = required_columns + ['EngBlk', 'EngWht', 'EngOpen', 'EngPav'] # Create two DataFrames: one for prediction and one for classification. df_pred = df[required_columns].copy() df_class = df[required_columns_2].fillna("NA").copy() # Transform categorical columns for prediction DataFrame using the label encoders. for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly']: df_pred[col] = loaded_label_encoder[col].transform(df_pred[col]) # Update the classification DataFrame with the transformed prediction columns. for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly']: df_class[col] = df_pred[col] # Transform the extra columns in the classification DataFrame. for col in ['EngBlk', 'EngWht', 'EngOpen', 'EngPav']: df_class[col] = loaded_label_encoder[col].transform(df_class[col]) # Convert both DataFrames to float (or handle as needed). df_pred = df_pred.astype(float) df_class = df_class.astype(float) # ------------------------- # Prediction Report Section # ------------------------- # Use the prediction DataFrame for price predictions. x = df_pred.copy() df_pred['GIA_Predicted'] = gia_model.predict(x) df_pred['Grade_Predicted'] = grade_model.predict(x) df_pred['ByGrade_Predicted'] = bygrade_model.predict(x) df_pred['Makable_Predicted'] = makable_model.predict(x) df_pred['GIA_Diff'] = df_pred['EngAmt'] - df_pred['GIA_Predicted'] df_pred['Grade_Diff'] = df_pred['EngAmt'] - df_pred['Grade_Predicted'] df_pred['ByGrade_Diff'] = df_pred['EngAmt'] - df_pred['ByGrade_Predicted'] df_pred['Makable_Diff'] = df_pred['EngAmt'] - df_pred['Makable_Predicted'] # ------------------------- # Classification Report Section # ------------------------- # For classification, use df_class (which has extra columns). x2 = df_class.copy() dx = df_pred.copy() # Start with the prediction data. dx['col_change'] = col_model.predict(x) dx['cts_change'] = cts_model.predict(x) dx['cut_change'] = cut_model.predict(x) dx['qua_change'] = qua_model.predict(x) dx['shp_change'] = shp_model.predict(x) dx['Change_Blk_Eng_to_Mkbl_value'] = blk_eng_to_mkbl_model.predict(x2) dx['Change_Wht_Eng_to_Mkbl_value'] = wht_eng_to_mkbl_model.predict(x2) dx['Change_Open_Eng_to_Mkbl_value'] = open_eng_to_mkbl_model.predict(x2) dx['Change_Pav_Eng_to_Mkbl_value'] = pav_eng_to_mkbl_model.predict(x2) dx['Change_Blk_Eng_to_Grd_value'] = blk_eng_to_grade_model.predict(x2) dx['Change_Wht_Eng_to_Grd_value'] = wht_eng_to_grade_model.predict(x2) dx['Change_Open_Eng_to_Grd_value'] = open_eng_to_grade_model.predict(x2) dx['Change_Pav_Eng_to_Grd_value'] = pav_eng_to_grade_model.predict(x2) dx['Change_Blk_Eng_to_ByGrd_value'] = blk_eng_to_bygrade_model.predict(x2) dx['Change_Wht_Eng_to_ByGrd_value'] = wht_eng_to_bygrade_model.predict(x2) dx['Change_Open_Eng_to_ByGrd_value'] = open_eng_to_bygrade_model.predict(x2) dx['Change_Pav_Eng_to_ByGrd_value'] = pav_eng_to_bygrade_model.predict(x2) dx['Change_Blk_Eng_to_Gia_value'] = blk_eng_to_gia_model.predict(x2) dx['Change_Wht_Eng_to_Gia_value'] = wht_eng_to_gia_model.predict(x2) dx['Change_Open_Eng_to_Gia_value'] = open_eng_to_gia_model.predict(x2) dx['Change_Pav_Eng_to_Gia_value'] = pav_eng_to_gia_model.predict(x2) # Inverse transform classification predictions. dx['col_change'] = loaded_label_encoder['Change_color_value'].inverse_transform(dx['col_change']) dx['cts_change'] = loaded_label_encoder['Change_cts_value'].inverse_transform(dx['cts_change']) dx['cut_change'] = loaded_label_encoder['Change_cut_value'].inverse_transform(dx['cut_change']) dx['qua_change'] = loaded_label_encoder['Change_quality_value'].inverse_transform(dx['qua_change']) dx['shp_change'] = loaded_label_encoder['Change_shape_value'].inverse_transform(dx['shp_change']) dx['Change_Blk_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Blk_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Blk_Eng_to_Mkbl_value']) dx['Change_Wht_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Wht_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Wht_Eng_to_Mkbl_value']) dx['Change_Open_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Open_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Open_Eng_to_Mkbl_value']) dx['Change_Pav_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Pav_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Pav_Eng_to_Mkbl_value']) dx['Change_Blk_Eng_to_Grd_value'] = loaded_label_encoder['Change_Blk_Eng_to_Grd_value'].inverse_transform(dx['Change_Blk_Eng_to_Grd_value']) dx['Change_Wht_Eng_to_Grd_value'] = loaded_label_encoder['Change_Wht_Eng_to_Grd_value'].inverse_transform(dx['Change_Wht_Eng_to_Grd_value']) dx['Change_Open_Eng_to_Grd_value'] = loaded_label_encoder['Change_Open_Eng_to_Grd_value'].inverse_transform(dx['Change_Open_Eng_to_Grd_value']) dx['Change_Pav_Eng_to_Grd_value'] = loaded_label_encoder['Change_Pav_Eng_to_Grd_value'].inverse_transform(dx['Change_Pav_Eng_to_Grd_value']) dx['Change_Blk_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Blk_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Blk_Eng_to_ByGrd_value']) dx['Change_Wht_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Wht_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Wht_Eng_to_ByGrd_value']) dx['Change_Open_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Open_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Open_Eng_to_ByGrd_value']) dx['Change_Pav_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Pav_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Pav_Eng_to_ByGrd_value']) dx['Change_Blk_Eng_to_Gia_value'] = loaded_label_encoder['Change_Blk_Eng_to_Gia_value'].inverse_transform(dx['Change_Blk_Eng_to_Gia_value']) dx['Change_Wht_Eng_to_Gia_value'] = loaded_label_encoder['Change_Wht_Eng_to_Gia_value'].inverse_transform(dx['Change_Wht_Eng_to_Gia_value']) dx['Change_Open_Eng_to_Gia_value'] = loaded_label_encoder['Change_Open_Eng_to_Gia_value'].inverse_transform(dx['Change_Open_Eng_to_Gia_value']) dx['Change_Pav_Eng_to_Gia_value'] = loaded_label_encoder['Change_Pav_Eng_to_Gia_value'].inverse_transform(dx['Change_Pav_Eng_to_Gia_value']) return df_pred, dx.head(len(df_pred)) # Return full DataFrames for pagination later. except Exception as e: flash(f'Error processing file: {e}', 'error') return pd.DataFrame(), pd.DataFrame() # ------------------------------ # Report View Route with Pagination & Toggle # ------------------------------ @app.route('/report') def report_view(): # Get query parameters: report_type (pred or class) and page number. report_type = request.args.get('report_type', 'pred') try: page = int(request.args.get('page', 1)) except ValueError: page = 1 per_page = 15 # records per page # Read the appropriate CSV file. if report_type == 'pred': df = pd.read_csv(PRED_OUTPUT_FILE) else: df = pd.read_csv(CLASS_OUTPUT_FILE) # Calculate pagination indices. start_idx = (page - 1) * per_page end_idx = start_idx + per_page total_records = len(df) # Slice the DataFrame for the current page. df_page = df.iloc[start_idx:end_idx] table_html = df_page.to_html(classes="data-table", index=False) # Determine if previous/next pages exist. has_prev = page > 1 has_next = end_idx < total_records return render_template('output.html', table_html=table_html, report_type=report_type, page=page, has_prev=has_prev, has_next=has_next) # ------------------------------ # Download Routes (remain unchanged) # ------------------------------ @app.route('/download_pred', methods=['GET']) def download_pred(): return send_file(PRED_OUTPUT_FILE, as_attachment=True) @app.route('/download_class', methods=['GET']) def download_class(): return send_file(CLASS_OUTPUT_FILE, as_attachment=True) if __name__ == "__main__": app.run(debug=True)