text: # -*- coding: utf-8 -*-
  """
  Created on in 2022
  
  @author: Group A - Advanced Python - IE MBD
  Hussain Alhajjaj  
  Ali Alqawaeen
  Mohammed Aljamed
  Hadi Alsinan 
  Khalid Nasser
  Maryam Alblushi
  Durrah Alzamil
  Basil Alfakher
  """
  
  from os import stat
  import pandas as pd
  import numpy as np
  import matplotlib.pyplot as plt
  from sklearn.preprocessing import MinMaxScaler
  from sklearn.preprocessing import StandardScaler
  from sklearn.preprocessing import OneHotEncoder
  from sklearn import preprocessing
  from sklearn.preprocessing import StandardScaler, OneHotEncoder
  
  
  from pandas.api.types import is_numeric_dtype, is_object_dtype
  import numpy as np
  import plotly.offline as py
  import plotly.figure_factory as ff
  import plotly.graph_objs as gobj
  from sklearn.model_selection import train_test_split
  
  
  from .stats_helpers import eval_df, clean_data
  from .charting_helper import plot_correlation_matrix, plot_univariate
  from .custom_helpers import WoE_Transformer, Ordinal_Transformer
  
  class MankeyDataFrame(pd.DataFrame):
      """
      The class is used to apply various data wrangling operations to Dataframes which includes Scaling and Encoding.
      In addition, the class provides some statistical based methods and robust functions such as Normality testing, Outlier detection, and Missing Data handling.
      It provides the end user with both general and specific cleaning functions
  
  
      Below is an example of instantiation of the class:
  
      import pandas as pd
      csv_list = pd.read_csv("list_stuff.csv")
      mdf = MankeyDataFrame(csv_list)
  
      """
  
      #Initializing the inherited pd.DataFrame
      def __init__(self, *args, **kwargs):
          super().__init__(*args,**kwargs)
      
      @property
      def _constructor(self):
          def func_(*args,**kwargs):
              df = MankeyDataFrame(*args,**kwargs)
              return df
          return func_
      
  #-----------------------------------------------------------------------------
                          # DATA HANDLING
  #-----------------------------------------------------------------------------
  
      def SetAttributes(self, kwargs):
          """
          The function will update the type of the variable submitted for change.
          It will veify first that the key is present in the desired dataframe.
          If present, it will try to change the type to the desired format.
          If not possible, it will continue to the next element.         
          Parameters
          ----------
          **kwargs : The key-argument pair of field-type relationship that
          wants to be updated.
          Returns
          -------
          None.
          """
          if self.shape[0] > 0:
              for key,vartype in kwargs.items():
                  if key in self.columns:
                      try:
                          self[key] = self[key].astype(vartype)
                      except:
                          print("Undefined type {}".format(str(vartype)))
                  else:
                      print("The dataframe does not contain variable {}.".format(str(key)))
          else:
              print("The dataframe has not yet been initialized")
  
  #-----------------------------------------------------------------------------
                          # SUPERVISED - BINARY CLASSIFICATION - DATA CLEANING
  #-----------------------------------------------------------------------------    
  
  
  
      def plot_charts(self, input_vars = [], force_categorical=['BINARIZED_TARGET']):
          plot_univariate(self, cols = input_vars, force_categorical=force_categorical)
      
  
      def plot_correlation(self, X, y):
          df = pd.DataFrame(X).copy()
          df["target"] = y
          plot_correlation_matrix(df)
  
  
      def explore_stat(self, input_vars=[], normal_test_alpha=0.05, grubbs_alpha=0.05, grubbs_max_outliers=20, iqr_factor=2.5):
          """
          This method describes the each feature of the dataset, for numeric fields, several statistics are 
          calculated including skewness, kurtosis, normality test (based on D'Agostino and Pearson's test), missing values, and quartiles.
          This also includes Grubbs test for outliers
  
          Parameters
          ----------
          input_vars: list of columns (to limit the evaluation to certain features only)
          normal_test_alpha: alpha value to be used for the normality test
          grubbs_alpha: alpha value for the grubbs outlier test
          grubbs_max_outliers: how many outliers to look for
          iqr_factor: IQR factor to determine outlier
  
          Returns
          -------
          None.
          """
          df_analyze = self.copy()
          if(input_vars):
              df_analyze = df_analyze.loc[:,input_vars]
          
          return eval_df(df_analyze)
  
      def create_train_test_sets(self, target_var, input_vars = [], test_size = 0.2, random_state = 42):
          """
          This method divides the dataframe into two datasets (training and test) to be used for ML models
          
          Parameters
          ----------
          input_vars: list of columns (to limit the sets to only certain fields)
          target_var: name of the feature containing the ML target (to be used for y)
          Returns
          -------
            Training and Test data sets including X and y (target) 
            (X_train, y_train, X_test, y_test)
  
          """
  
          X = self.loc[:, self.columns != target_var]
          y = self[target_var]
  
          if(input_vars):
              X = X.loc[:,input_vars]
          
          
  
          # split into input and output elements
          X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=test_size, 
                                                              random_state=random_state) 
          
          return (X_train, X_test, y_train, y_test)
  
      
      def cleaning_missing(self, X_train, y_train, X_test, y_test, target_var, input_vars=[], print_only = True ):
          """
          This method is used for missing data imputation and/or removal of the feature/observation (as per user preferences). 
          This is done using the statistical tests from the explore_stat method, 
          The method can be run in simulation mode or active mode. In simulation mode, only the recommended transformations
          will be highlighted but the data is not modified, while in active mode, the recommended transformations will be applied.
          
          Parameters
          ----------
          input_vars: list of columns (to limit the evaluation to certain features only)
          print_only: only print an analysis of the findings
  
          Returns
          -------
          A print with the analysis or new clean columns .
  
          """
          df_train = X_train.copy()
          df_train[target_var] = y_train
  
          df_test = X_test.copy()
          df_test[target_var] = y_test
  
          if(print_only):
             
              (descriptive_statistics, logic_prep, _) = eval_df(df_train)
              print("Training set")
              print(logic_prep)
              print("===============")
              print(descriptive_statistics)
  
              (descriptive_statistics, _, _) = eval_df(df_test)
              print("===============")
              print("Test set stats")
              print(descriptive_statistics)
  
          else:
              (df_clean, df_clean_test) = clean_data(df_train, df_test)
              #X_train, y_train, X_test, y_test
              return df_clean.loc[:, df_clean.columns != target_var], df_clean[target_var], \
                     df_clean_test.loc[:, df_clean_test.columns != target_var], df_clean_test[target_var]
  
          
      
   
      def recommended_transformation(self, X_train, y_train, X_test, y_test, input_vars = [], ordinal_var = {}, woe_cat_threshold=5, print_only = True, expand_dates = 'all'
      , due_date = "EXPECTED_CLOSE_DATE", start_date = "LOAN_OPEN_DATE", target_var = 'BINARIZED_TARGET'):
          """
          This method can be used to recommend and/or apply transformations including:
          impute missing values, data manipulation, categorical variable handling.
          
          For WoE, this method gets a list of categorical columns and the target variable, it calculates
          the WoE for each column and each class within it (fit) and then replaces the original value of the category with the WoE, this can be used later to transform
          the test set. The transformed categorical variable will be a WOE value and it will be treated same as any continuous variable.
          
          This method can be used to transform categorical variables using one hot encoding, dummy variable encoding, or ordinal encoding.
  
          The method requires two datasets (training and test), all transformation rules are based
          on the training set which is then applied on both datasets.
  
          Note: the method can be used to also print the transformations, without actually generating new data.
         
          Here is an example of specifying ordinal categories for encoding:
          dictionary is as follows: {"feature_name": [list of all classes IN ORDER from low (1) to high(# of classes)]}
          
          Parameters
          ----------
          input_vars: list of columns (to limit the evaluation to certain features only)
          X_train: subset to be used for training
          X_test: subset to be used for test
          y_train: target output related to X_train
          y_test: target output related to X_test
          target_var: name of the feature containing the ML target (to be used for y)
          woe_cat_threshold: the number of classes per category to determine whether WoE should be used for transformation
          ordinal_var: dictionary for specifying ordinal categories as follows: {"feature_name": [list of all classes IN ORDER from low (1) to high(# of classes)]}
          print_only: only print an analysis of the findings
  
          Returns
          -------
            A print with the analysis or new transformed columns.                
          """
          result = {}
          if(input_vars):
              X_train = X_train[input_vars]
              X_test = X_test[input_vars]
  
          if(X_train.isna().sum().values.sum() > 0 or X_test.isna().sum().values.sum() > 0):
              print("X_train and X_test still have missing values, consider the clean missing method ")
              return
  
          if(not ((X_train.columns != X_test.columns).sum() == 0 and (X_train.dtypes != X_test.dtypes).sum() == 0)):
              print("X_train and X_test do not have matching feature list (columns)")
              return
  
          #numeric variables
          descriptive_statistics, _, _ = eval_df(X_train)
          std_scaler = []
          minmax_scaler = []
          for col in X_train.select_dtypes(include=np.number):
              if((descriptive_statistics[descriptive_statistics["Name"] == col]["Is Normal"] == "Normal").bool()):
                  print(f"Feature {col} will be scaled with a normalized scaler (since it is normally distributed)")
                  std_scaler.append(col)
              else:
                  print(f"Feature {col} will be scaled with a Min Max scaler (since it is NOT normally distributed)")
                  minmax_scaler.append(col)
  
          #only change data if we are not running in print_only mode
          if(not print_only):
              if(std_scaler):
                  #scale numeric features (normal)
                  std_scaler_transformer = StandardScaler()
                  std_scaler_transformer.fit(X_train[std_scaler])
                  X_train[std_scaler] = std_scaler_transformer.transform(X_train[std_scaler])
                  X_test[std_scaler] = std_scaler_transformer.transform(X_test[std_scaler])
              if(minmax_scaler):
                  #scale numeric features (non-normal)
                  minmax_scaler_transformer = MinMaxScaler()
                  minmax_scaler_transformer.fit(X_train[minmax_scaler])
                  X_train[minmax_scaler] = minmax_scaler_transformer.transform(X_train[minmax_scaler])
                  X_test[minmax_scaler] = minmax_scaler_transformer.transform(X_test[minmax_scaler])
  
          
  
          # calculate due date - start date
          # only if the the columns exist
          if(due_date in X_train.columns and start_date in X_train.columns):
              print(f"Calulate days difference beteen {due_date} and {start_date}")
              if(not print_only):
                  X_train['due_in_days'] = X_train[due_date] - X_train[start_date]
                  X_test['due_in_days'] = X_test[due_date] - X_test[start_date]
                  X_train['due_in_days'] = X_train['due_in_days'].dt.days
                  X_test['due_in_days'] = X_test['due_in_days'].dt.days
          else:
              print("No due date calculation since columns specified are not in the dataset")
          
          
          # dates expansion
          for col in X_train.select_dtypes(include=['datetime64[ns]']):
              print(f"Exapnding feature {col} to {expand_dates}")
              if(not print_only):
                  if(expand_dates == 'year' or expand_dates == 'month-year' or expand_dates == 'all'):
                      X_train[col+"_year"] = X_train[col].dt.year
                      X_test[col+"_year"] = X_test[col].dt.year
                  if(expand_dates == 'month-year' or expand_dates == 'all'):
                      X_train[col+"_month"] = X_train[col].dt.month
                      X_test[col+"_month"] = X_test[col].dt.month
                  if(expand_dates == 'all'):
                      X_train[col+"_day"] = X_train[col].dt.day
                      X_test[col+"_day"] = X_test[col].dt.day
              
              print(f"Dropping the feature {col} after expansion")
              if(not print_only):
                  X_train.drop(col, axis = 1, inplace = True)
                  X_test.drop(col, axis = 1, inplace = True)
  
          # categorical
                  
          #ordinal
          
          if(ordinal_var):
              print("Ordinal variables specificed will be transformed to numeric according to the specified order")
              if(not print_only):
                  ordinal_transformer = Ordinal_Transformer()
                  ordinal_transformer.fit( ordinal_var, X_train ,None)
  
                  X_train = ordinal_transformer.transform(X_train, None)
                  X_test = ordinal_transformer.transform(X_test, None)
              
          
          #WoE transformations
          woe_vars = []
          for feature in X_train.select_dtypes(exclude=[np.number]):
              if feature in ordinal_var: #we should not need this, but just in case
                  continue
              if X_train[feature].nunique() >= woe_cat_threshold:
                  print(f"Feature {feature} will be transformed to numeric using WoE - Weight of Evidence as it has more classes than the threshold")
                  result[feature] = 'Weight of Evidence (WoE) transformation'
                  woe_vars.append(feature)
          
          
          if(not print_only):
              for feature in woe_vars:
                  X_train[feature] = X_train[feature].cat.remove_unused_categories()
                  X_test[feature] = X_test[feature].cat.remove_unused_categories()
              
              t_woe = WoE_Transformer()
              
              t_woe.fit(X_train, y_train, target_var, woe_vars,  woe_cat_threshold)
              X_train = t_woe.transform(X_train,y_train)
              X_test = t_woe.transform(X_test, y_test)
              
          #remaining - one hot encoding
          
          ohe_vars = [col for col in X_train.columns if not is_numeric_dtype(X_train[col])]
          print("The following features will be transformed using one-hot-encoding:")
          print(ohe_vars)
      # define the data preparation for the columns
          if(not print_only and ohe_vars):
              
              
              ohe_transformer = preprocessing.OneHotEncoder(sparse=False)
              ohe_transformer.fit(X_train[ohe_vars])
              
              X_train_ohe = X_train_ohe = pd.DataFrame(ohe_transformer.transform(X_train[ohe_vars]), columns=ohe_transformer.get_feature_names())
              X_test_ohe = pd.DataFrame(ohe_transformer.transform(X_test[ohe_vars]), columns=ohe_transformer.get_feature_names())
  
              X_train_ohe.set_index(X_train.index, inplace=True)
              X_test_ohe.set_index(X_test.index , inplace=True)
              X_train = pd.concat([X_train, X_train_ohe], axis=1)
              X_test = pd.concat([X_test, X_test_ohe], axis=1)
  
              print("remove remaining Categorical variables after OHE")
              X_train.drop(columns = ohe_vars, axis = 1, inplace = True)
              X_test.drop(columns = ohe_vars, axis = 1, inplace = True)
          
          return X_train, X_test
      
  