text: #!/usr/bin/env python 
  # -*- coding: utf-8 -*-
  """
  Base class for all modules
  """
  
  import time
  import json
  import re
  import copy
  import numpy as np
  import pandas as pd
  import os
  import nibabel as nib
  from scipy import io
  from joblib import Memory
  import abc
  from abc import abstractmethod, ABCMeta
  from imblearn.over_sampling import (RandomOverSampler, SMOTE, ADASYN, BorderlineSMOTE, SMOTENC)
  from imblearn.under_sampling import (RandomUnderSampler,
                                      ClusterCentroids, 
                                      NearMiss,
                                      InstanceHardnessThreshold,
                                      CondensedNearestNeighbour,
                                      EditedNearestNeighbours,
                                      RepeatedEditedNearestNeighbours,
                                      AllKNN,
                                      NeighbourhoodCleaningRule,
                                      OneSidedSelection)
  
  from imblearn.combine import SMOTEENN, SMOTETomek
  from sklearn.metrics import make_scorer, accuracy_score, auc, f1_score, mean_absolute_error
  from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
  from sklearn.preprocessing import StandardScaler, MinMaxScaler
  from sklearn.decomposition import PCA, NMF
  from sklearn.feature_selection import SelectPercentile, SelectKBest, SelectFromModel, f_classif,f_regression, RFE,RFECV, VarianceThreshold, mutual_info_classif, SelectFromModel
  from sklearn.svm import LinearSVC, SVC, SVR
  from sklearn.linear_model import (LinearRegression, LogisticRegression, Lasso, 
                                    LassoCV, RidgeCV, Ridge,
                                    RidgeClassifier, BayesianRidge, ElasticNetCV
  )
  
  from sklearn.gaussian_process import  GaussianProcessClassifier, GaussianProcessRegressor
  from sklearn.tree import DecisionTreeClassifier
  from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, AdaBoostClassifier
  from sklearn.model_selection import KFold, StratifiedKFold,  ShuffleSplit
  from sklearn.pipeline import Pipeline
  
  
  class BaseMachineLearning(object):
      """Base class for all machine learning
  
      Parameters:
      ----------
      configuration_file: file string
          configuration file containing all inputs
  
      Attributes:
      ----------
      method_feature_preprocessing_: list of sklearn object or None
      param_feature_preprocessing_: list of sklearn object or None
      
      method_dim_reduction_: list of sklearn object or None
      param_dim_reduction_: list of sklearn object or None
      
      method_feature_selection_: list of sklearn object or None
      param_feature_selection_: list of sklearn object or None
      
      method_unbalance_treatment_: list of sklearn object or None
      param_unbalance_treatment_: list of sklearn object or None
      
      machine_learning_type_: str
      method_machine_learning_: list of sklearn object or None
      param_machine_learning_: list of sklearn object or None
  
      method_model_evaluation_: list of sklearn object or None
      param_model_evaluation_: list of sklearn object or None
  
      model_: machine learning model, e.g., sklearn gridSearch model
      param_search_: parameter for search of machine learning pipeline
      """
  
      def __init__(self, configuration_file):
          self.configuration_file = configuration_file
          self._random_state = 0
          self._gridcv_k = 3
          self._search_strategy = "grid"
          self._n_jobs = 2
  
      def get_configuration_(self):
          """Get and parse the configuration file
          """
  
          with open(self.configuration_file, 'r', encoding='utf-8') as config:
                      configuration = config.read()
          self.configuration = json.loads(configuration)
          return self
  
      def get_preprocessing_parameters(self):
          self.method_feature_preprocessing_ = None
          self.param_feature_preprocessing_ = {}
                  
          feature_preprocessing = self.configuration.get('feature_engineering', {}).get('feature_preprocessing', None)
          if feature_preprocessing and (list(feature_preprocessing.keys())[0] != 'None'):
              self.method_feature_preprocessing_ = [list(feature_preprocessing.keys())[0] if list(feature_preprocessing.keys())[0] != 'None' else None]
              self.method_feature_preprocessing_ = [self.security_eval(self.method_feature_preprocessing_[0])]
      
              for key in feature_preprocessing.keys():
                  for key_ in feature_preprocessing.get(key).keys():
                      if key_ != []:
                          for key__ in feature_preprocessing.get(key).get(key_).keys():
  
                              param = feature_preprocessing.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              param = self.security_eval(param)
                              self.param_feature_preprocessing_.update({"feature_preprocessing__"+key_: [param]})
  
          # Fix the random_state for replication of results
          if self.method_feature_preprocessing_ and ("random_state" in self.method_feature_preprocessing_[0].get_params().keys()):
              self.param_feature_preprocessing_.update({"feature_preprocessing__"+'random_state': [self._random_state]})
          self.param_feature_preprocessing_ = None if self.param_feature_preprocessing_ == {} else self.param_feature_preprocessing_
               
          return self
  
      def get_dimension_reduction_parameters(self):
          self.method_dim_reduction_ = None
          self.param_dim_reduction_ = {}
                  
          dimension_reduction = self.configuration.get('feature_engineering', {}).get('dimreduction', None)
          if dimension_reduction and (list(dimension_reduction.keys())[0] != 'None'):
              self.method_dim_reduction_ = [self.security_eval(list(dimension_reduction.keys())[0] if list(dimension_reduction.keys())[0] != 'None' else None)]
      
              for key in dimension_reduction.keys():
                  for key_ in dimension_reduction.get(key).keys():
                      if key_ != []:
                          for key__ in dimension_reduction.get(key).get(key_).keys():
                              param = dimension_reduction.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              param = self.security_eval(param)
                              if not isinstance(param, (list, tuple)):
                                  param = [param]
                              self.param_dim_reduction_.update({"dim_reduction__"+key_: param})
          
          # Fix the random_state for replication of results
          if self.method_dim_reduction_ and ("random_state" in self.method_dim_reduction_[0].get_params().keys()):
              self.param_dim_reduction_.update({"dim_reduction__"+'random_state': [self._random_state]})
          self.param_dim_reduction_ = None if self.param_dim_reduction_ == {} else self.param_dim_reduction_
          return self  
  
      def get_feature_selection_parameters(self):
          self.method_feature_selection_ = None
          self.param_feature_selection_ = {}
          
          feature_selection = self.configuration.get('feature_engineering', {}).get('feature_selection', None)
          if feature_selection and (list(feature_selection.keys())[0] != 'None'): 
              for key in feature_selection.keys():
                  for key_ in feature_selection.get(key).keys():
                      if key_ != []:
                          for key__ in feature_selection.get(key).get(key_).keys():
                              param = feature_selection.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              param = self.security_eval(param)
                              if not isinstance(param, (list, tuple)):
                                  param = [param]
                              self.param_feature_selection_.update({"feature_selection__"+key_:param})
  
              # Methods
              self.method_feature_selection_ = list(feature_selection.keys())[0] if list(feature_selection.keys())[0] != 'None' else None
              # Update point
              if "RFE" in self.method_feature_selection_:
                  self.method_feature_selection_ = "RFE(estimator=SVC(kernel='linear'))"
              
              if self.method_feature_selection_ == 'SelectFromModel(LassoCV())':
                  self.param_feature_selection_ = None
              
              if self.method_feature_selection_ == 'SelectFromModel(ElasticNetCV())':
                  self.method_feature_selection_ = 'SelectFromModel(ElasticNetCV('
                  for keys in list(self.param_feature_selection_.keys()):
                      param_ = keys.split('__')[1]
                      value_ = self.param_feature_selection_[keys]
                      self.method_feature_selection_ = self.method_feature_selection_+ f'{param_}={value_},'  
                  self.method_feature_selection_ = self.method_feature_selection_ + '))'
                  self.param_feature_selection_ = None
                  
              self.method_feature_selection_ = [self.security_eval(self.method_feature_selection_)]
          
          # Fix the random_state for replication of results
          if self.method_feature_selection_ and ("random_state" in self.method_feature_selection_[0].get_params().keys()):
              self.param_feature_selection_.update({"feature_selection__"+'random_state': [self._random_state]})
          self.param_feature_selection_ = None if self.param_feature_selection_ == {} else self.param_feature_selection_
          return self
  
      def get_unbalance_treatment_parameters(self):
          self.method_unbalance_treatment_ = None
          self.param_unbalance_treatment_ = {}
  
          unbalance_treatment = self.configuration.get('feature_engineering', {}).get('unbalance_treatment', None)
          if unbalance_treatment and (list(unbalance_treatment.keys())[0] != 'None'):
              self.method_unbalance_treatment_ = (self.security_eval(list(unbalance_treatment.keys())[0]) if list(unbalance_treatment.keys())[0] != 'None' else None)
      
              for key in unbalance_treatment.keys():
                  for key_ in unbalance_treatment.get(key).keys():
                      if key_ != []:
                          for key__ in unbalance_treatment.get(key).get(key_).keys():
  
                              param = unbalance_treatment.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              param = self.security_eval(param)
                              if not isinstance(param, (list, tuple)):
                                  param = [param]
                              self.param_unbalance_treatment_.update({"unbalance_treatment__"+key_:param})
                       
          # Fix the random_state for replication of results
          if self.method_unbalance_treatment_ and "random_state" in self.method_unbalance_treatment_.get_params().keys():
              self.method_unbalance_treatment_.set_params(**{"random_state": self._random_state})
              self.param_unbalance_treatment_.update({"unbalance_treatment__"+'random_state': [self._random_state]})
          self.param_unbalance_treatment_ = None if self.param_unbalance_treatment_ == {} else self.param_unbalance_treatment_
          
          return self
  
      def get_machine_learning_parameters(self):
          self.method_machine_learning_ = None
          self.param_machine_learning_ = {}
          
          machine_learning = self.configuration.get('machine_learning', None)
          self.machine_learning_type_ = list(machine_learning.keys()) if machine_learning else None
          if self.machine_learning_type_ is None:
              raise ValueError("There is no keys for machine_learning")
          elif len(self.machine_learning_type_) > 1:
              raise RuntimeError("Currently, easylearn only supports one type of machine learning")
              
          for keys in machine_learning:
              machine_learning = machine_learning.get(keys, None)
  
          if machine_learning and (list(machine_learning.keys())[0] != 'None'):
              # TODO: This place will update for supporting multiple estimators
              self.method_machine_learning_ = [self.security_eval(list(machine_learning.keys())[0] if list(machine_learning.keys())[0] != 'None' else None)]
      
              for key in machine_learning.keys():
                  for key_ in machine_learning.get(key).keys():
                      if key_ != []:
                          for key__ in machine_learning.get(key).get(key_).keys():
  
                              param = machine_learning.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              # for example, DecisionTreeClassifier(max_depth=1) is a parameter of AdaBoostClassifier()
                              # Because a [sklearn] object has a
                              param = self.security_eval(param)
                              if not isinstance(param, (list, tuple)):
                                  param = [param]
                              # TODO: Design a method to set params
                              self.param_machine_learning_.update({"estimator__"+key_: param})
           
          # Fix the random_state for replication of results
          if self.method_machine_learning_ and "random_state" in self.method_machine_learning_[0].get_params().keys():
              self.param_machine_learning_.update({"estimator__"+'random_state': [self._random_state]})
          self.param_machine_learning_ = None if self.param_machine_learning_ == {} else self.param_machine_learning_
          return self
  
      def get_model_evaluation_parameters(self):
          self.method_model_evaluation_ = None
          self.param_model_evaluation_ = {}
          self.statistical_analysis = self.configuration.get('model_evaluation', {}).get("Statistical_analysis", None)
          if self.statistical_analysis:
              self.configuration.get('model_evaluation', {}).pop("Statistical_analysis")
          model_evaluation = self.configuration.get('model_evaluation', None)
          
          if model_evaluation and (list(model_evaluation.keys())[0] != 'None'):
              self.method_model_evaluation_ = list(model_evaluation.keys())[0] if list(model_evaluation.keys())[0] != 'None' else None
              
              for key in model_evaluation.keys():
                  for key_ in model_evaluation.get(key).keys():
                      if key_ != []:
                          for key__ in model_evaluation.get(key).get(key_).keys():                            
                              param = model_evaluation.get(key).get(key_).get(key__)
                              param = 'None' if param == '' else param
                              # Parse parameters: if param is digits str or containing "(" and ")", we will eval the param
                              # for example, DecisionTreeClassifier(max_depth=1) is a parameter of AdaBoostClassifier()
                              # Because a [sklearn] object has a
                              if type(param) is str:  # selected_dataset is list
                                      param = self.security_eval(param)
                              self.param_model_evaluation_.update({key_: param})
               
              # ------Give parameter to method------
              pme = ""
              ik_end = len(self.param_model_evaluation_)  - 1
              for ik, key_pme in enumerate(self.param_model_evaluation_):
                  if ik != ik_end:
                      pme = pme + f"{key_pme}={self.param_model_evaluation_[key_pme]}" + ", "
                  else:
                      pme = pme + f"{key_pme}={self.param_model_evaluation_[key_pme]}"
              
              self.method_model_evaluation_ = self.method_model_evaluation_.split("(")[0] + "(" + pme + self.method_model_evaluation_.split("(")[1]
              self.method_model_evaluation_ = self.security_eval(self.method_model_evaluation_)
              
          return self
  
      def get_statistical_analysis_parameters(self):
          self.method_statistical_analysis_ = list(self.statistical_analysis.keys())[0]
          # parameters
          # FIX the logical and extending to...
          self.param_statistical_analysis_ = None
          for key1 in self.statistical_analysis.keys():
              for key2 in self.statistical_analysis.get(key1, {}).keys():
                  for key3 in self.statistical_analysis[key1].get(key2, {}).keys():
                      self.param_statistical_analysis_ = self.statistical_analysis.get(key1, {}).get(key2, {}).get(key3, {})
                      self.param_statistical_analysis_ = self.security_eval(self.param_statistical_analysis_)
                  
          return self
  
      def get_visualization_parameters(self):
          self.configuration.get('visualization', None)
  
      def get_all_inputs(self):
          self.get_configuration_()
          self.get_preprocessing_parameters()
          self.get_dimension_reduction_parameters()
          self.get_feature_selection_parameters()
          self.get_unbalance_treatment_parameters()
          self.get_machine_learning_parameters()
          self.get_model_evaluation_parameters()
          self.get_statistical_analysis_parameters()
          self.make_sklearn_search_model_()
          return self
  
  
      @staticmethod
      def security_eval(expression):
          """Security evaluation of python expression
          
          FIX: 'eval' had security hole
          """
          
          iseval = (
                      (
                          bool(re.search(r'\d', expression)) or 
                          (expression == 'None') or 
                          (bool(re.search(r'\(', expression)) and bool(re.search(r'\)', expression))) 
                      ) 
                      and
                      (
                          expression != 'l1' and
                          expression != 'l2'
                          # not bool(re.search('del',  expression)) and
                          # not bool(re.search('open',  expression)) and
                          # not bool(re.search('move',  expression)) and
                          # not bool(re.search('copy',  expression))     
                      )
          )
  
          if iseval:
              evaluated_expression = eval(expression)
          else:
              evaluated_expression = expression
  
          return evaluated_expression
  
      def make_sklearn_search_model_(self, metric=accuracy_score):
          
          """Construct pipeline_
  
          Currently, the pipeline_ only supports one specific method for corresponding method, 
          e.g., only supports one dimension reduction method for dimension reduction.
          In the next version, the pipeline_ will support multiple methods for each corresponding method.
          
          Parameters:
          ----------
          metric: sklearn metric object, such as accuracy_score, auc, f1_score. Default is accuracy_score
              Metric is used evaluate model using cross validation in search strategy, such as GridSearchCV.
  
          Returns:
          -------
          model_
          """
          
          self.memory = Memory(location=os.path.dirname(self.configuration_file), verbose=False)
  
          # Construct sklearn pipeline
          self.pipeline_ = Pipeline(steps=[
              ('feature_preprocessing','passthrough'),
              ('dim_reduction', 'passthrough'),
              ('feature_selection', 'passthrough'),
              ('estimator', 'passthrough'),
              ], 
              memory=self.memory
          )
  
          # Set parameters of search CV
          self.param_search_ = {}
          if self.method_feature_preprocessing_:
              self.param_search_.update({'feature_preprocessing':self.method_feature_preprocessing_})
          if self.param_feature_preprocessing_:   
              self.param_search_.update(self.param_feature_preprocessing_)
              
          if self.method_dim_reduction_:
              self.param_search_.update({'dim_reduction':self.method_dim_reduction_})
          if self.param_dim_reduction_:
              self.param_search_.update(self.param_dim_reduction_)
                  
          if self.method_feature_selection_:
              self.param_search_.update({'feature_selection': self.method_feature_selection_})
          if self.param_feature_selection_:
              self.param_search_.update(self.param_feature_selection_)
              
          if self.method_machine_learning_:
              self.param_search_.update({'estimator': self.method_machine_learning_})
          if self.param_machine_learning_:
              self.param_search_.update(self.param_machine_learning_)
          
          # If no parameters' length greater than 1, using sklearn pipeline for speed up, instead of GridSearchCV or RandomizedSearchCV.
          self.is_search = self.get_is_search(self.param_search_)
          if not self.is_search:
              if self.method_feature_preprocessing_:
                  self.pipeline_.set_params(**{'feature_preprocessing':self.method_feature_preprocessing_[0]})
              if self.param_feature_preprocessing_:   
                  mapping = self.parse_search_params(self.param_feature_preprocessing_)
                  self.pipeline_['feature_preprocessing'].set_params(**mapping)
                  
              if self.method_dim_reduction_:
                  self.pipeline_.set_params(**{'dim_reduction':self.method_dim_reduction_[0]})
              if self.param_dim_reduction_:
                  mapping = self.parse_search_params(self.param_dim_reduction_)
                  self.pipeline_['dim_reduction'].set_params(**mapping)   
                  
              if self.method_feature_selection_:
                  self.pipeline_.set_params(**{'feature_selection': self.method_feature_selection_[0]})
              if self.param_feature_selection_:
                  mapping = self.parse_search_params(self.param_feature_selection_)
                  self.pipeline_['feature_selection'].set_params(**mapping)
  
              if self.method_machine_learning_:
                  self.pipeline_.set_params(**{'estimator': self.method_machine_learning_[0]})
              if self.param_machine_learning_:
                  mapping = self.parse_search_params(self.param_machine_learning_)
                  self.pipeline_['estimator'].set_params(**mapping)
  
          # Building model
          if "Classification" in self.machine_learning_type_:
              cv = StratifiedKFold(n_splits=self._gridcv_k, random_state=self._random_state, shuffle=True)  # Default is StratifiedKFold
          else:
              cv = KFold(n_splits=self._gridcv_k, random_state=self._random_state, shuffle=True)
          
          if self.is_search:
              if self._search_strategy == 'grid':
                  self.model_ = GridSearchCV(
                      self.pipeline_, n_jobs=self._n_jobs, param_grid=self.param_search_, cv=cv, 
                      scoring = make_scorer(metric), refit=True
                  )
              elif self._search_strategy == 'random':
                  self.model_ = RandomizedSearchCV(
                      self.pipeline_, n_jobs=self._n_jobs, param_distributions=self.param_search_, cv=cv, 
                      scoring = make_scorer(metric), refit=True, n_iter=self.n_iter_of_randomedsearch,
                  )
              else:
                  print("Please specify which search strategy!\n")
                  return
          else:
              self.model_ = self.pipeline_
  
          return self
      
      @staticmethod
      def get_is_search(dictionary):
          """ Identify whether search params (grid search or random search) or just using pipeline
          """
          
          is_search = False
          for key in dictionary:
              if dictionary[key] and len(dictionary[key]) > 1:
                  is_search = True
                  break
          return is_search
      
      @staticmethod
      def parse_search_params(dictionary):
          """ When just using pipeline and not search parameters
          I use 'set_params' to set parameters for pipeline to save running time.       
          """
  
          mapping = {}
          for key in dictionary:
              mapping.update({key.split("__")[1]:dictionary[key][0]})
          return mapping
              
      def save_weight(self, weights=None, out_dir=None):
          """Save contribution weight of features for each modality
          
          Parameters:
          ----------
          
          weights: list of numpy.ndarray
              Contribution weights of each fold of each modality (e.g. 5-fold cross validation)
  
          out_dir: str
              Output directory
  
          Returns:
          -------
          None
          """
  
          mean_wei = np.reshape(np.mean(weights, axis=0), [-1,])
          
          for group in self.mask_:
              loc_start = 0
              for im, modality in enumerate(self.mask_[group]):
                  mask = self.mask_[group][modality]
                  mean_weight = np.zeros(mask.shape)
  
                  # Updating mask location index
                  n_features = mask.sum()
                  loc_end = loc_start + n_features
  
                  mean_weight[mask] = mean_wei[loc_start:loc_end]
  
                  # Save
                  if self.data_format_[group][modality] in ["nii","gz"]:
                      out_name_wei = os.path.join(out_dir, f"weight_{modality}.nii.gz")
                      if os.path.exists(out_name_wei):
                          time_ = time.strftime('%Y%m%d%H%M%S')
                          out_name_wei = os.path.join(out_dir, f"weight_{modality}_{time_}.nii.gz")
                      mean_weight2nii = nib.Nifti1Image(mean_weight, self.affine_[group][modality])
                      mean_weight2nii.to_filename(out_name_wei)
                  else:
                      out_name_wei = os.path.join(out_dir, f"weight_{modality}.csv")
                      if os.path.exists(out_name_wei):
                          time_ = time.strftime('%Y%m%d%H%M%S')
                          out_name_wei = os.path.join(out_dir, f"weight_{modality}_{time_}.csv")
                          
                      if len(np.shape(mean_weight)) > 1:
                          np.savetxt(out_name_wei, mean_weight, delimiter=',')  
                      else:
                          pd.Series(mean_weight).to_csv(out_name_wei, header=False)
                  
                  # Updating mask location index
                  loc_start += n_features  # n_features in this point is the value in previous iteration
                  
              break  # Assuming the size of the same modality in different group are matching for each other
  
  
  # TODO
  class MakeModel():
      """Make a machine learning model
  
      make_sklearn_search_model_ should be integrated into the this class
  
      Parameters:
      ----------
      pass
  
      Attributes:
      ----------
      fit:
      predict:
      
      returns:
      -------
      pass
      """
  
      def __init__(self):
          baseml = BaseMachineLearning()
  
      def make_sklearn_search_model_(self, metric=accuracy_score):
          
          """Construct pipeline_
  
          Currently, the pipeline_ only supports one specific method for corresponding method, 
          e.g., only supports one dimension reduction method for dimension reduction.
          In the next version, the pipeline_ will support multiple methods for each corresponding method.
          
          Parameters:
          ----------
          metric: sklearn metric object, such as accuracy_score, auc, f1_score. Default is accuracy_score
              Metric is used evaluate model using cross validation in search strategy, such as GridSearchCV.
  
          Returns:
          -------
          model_
          """
          
          self.memory = Memory(location=os.path.dirname(self.configuration_file), verbose=False)
  
          # Construct sklearn pipeline
          self.pipeline_ = Pipeline(steps=[
              ('feature_preprocessing','passthrough'),
              ('dim_reduction', 'passthrough'),
              ('feature_selection', 'passthrough'),
              ('estimator', 'passthrough'),
              ], 
              memory=self.memory
          )
  
          # Set parameters of search CV
          self.param_search_ = {}
          if baseml.method_feature_preprocessing_:
              self.param_search_.update({'feature_preprocessing':self.method_feature_preprocessing_})
          if baseml.param_feature_preprocessing_:   
              self.param_search_.update(self.param_feature_preprocessing_)
              
          if baseml.method_dim_reduction_:
              self.param_search_.update({'dim_reduction':self.method_dim_reduction_})
          if baseml.param_dim_reduction_:
              self.param_search_.update(self.param_dim_reduction_)
                  
          if baseml.method_feature_selection_:
              self.param_search_.update({'feature_selection': self.method_feature_selection_})
          if baseml.param_feature_selection_:
              self.param_search_.update(self.param_feature_selection_)
              
          if baseml.method_machine_learning_:
              self.param_search_.update({'estimator': self.method_machine_learning_})
          if baseml.param_machine_learning_:
              self.param_search_.update(self.param_machine_learning_)
          
          # If no parameters' length greater than 1, using sklearn pipeline for speed up, instead of GridSearchCV or RandomizedSearchCV.
          self.is_search = self.get_is_search(self.param_search_)
          if not self.is_search:
              if self.method_feature_preprocessing_:
                  self.pipeline_.set_params(**{'feature_preprocessing':self.method_feature_preprocessing_[0]})
              if self.param_feature_preprocessing_:   
                  mapping = self.parse_search_params(self.param_feature_preprocessing_)
                  self.pipeline_['feature_preprocessing'].set_params(**mapping)
                  
              if self.method_dim_reduction_:
                  self.pipeline_.set_params(**{'dim_reduction':self.method_dim_reduction_[0]})
              if self.param_dim_reduction_:
                  mapping = self.parse_search_params(self.param_dim_reduction_)
                  self.pipeline_['dim_reduction'].set_params(**mapping)   
                  
              if self.method_feature_selection_:
                  self.pipeline_.set_params(**{'feature_selection': self.method_feature_selection_[0]})
              if self.param_feature_selection_:
                  mapping = self.parse_search_params(self.param_feature_selection_)
                  self.pipeline_['feature_selection'].set_params(**mapping)
  
              if self.method_machine_learning_:
                  self.pipeline_.set_params(**{'estimator': self.method_machine_learning_[0]})
              if self.param_machine_learning_:
                  mapping = self.parse_search_params(self.param_machine_learning_)
                  self.pipeline_['estimator'].set_params(**mapping)
  
          # Building model
          if "Classification" in list(self.configuration.get("machine_learning").keys()):
              cv = StratifiedKFold(n_splits=self._gridcv_k, random_state=self._random_state, shuffle=True)  # Default is StratifiedKFold
          else:
              cv = KFold(n_splits=self._gridcv_k, random_state=self._random_state, shuffle=True)  # Default is StratifiedKFold
          
          if self.is_search:
              if self._search_strategy == 'grid':
                  self.model_ = GridSearchCV(
                      self.pipeline_, n_jobs=self._n_jobs, param_grid=self.param_search_, cv=cv, 
                      scoring = make_scorer(metric), refit=True
                  )
              elif self._search_strategy == 'random':
                  self.model_ = RandomizedSearchCV(
                      self.pipeline_, n_jobs=self._n_jobs, param_distributions=self.param_search_, cv=cv, 
                      scoring = make_scorer(metric), refit=True, n_iter=self.n_iter_of_randomedsearch,
                  )
              else:
                  print("Please specify which search strategy!\n")
                  return
          else:
              self.model_ = self.pipeline_
  
          return self
  
  
  #%% ==========================================================================
  class DataLoader():
      """Load datasets according to different data types and handle extreme values
  
      Parameters:
      ----------
      configuration_file: file string
          configuration file containing all inputs
  
      Attributes:
      ----------
      targets_: ndarray of shape (n_samples, )
      
      features_: ndarray of shape (n_samples, n_features) 
  
      mask_: dictionary, each element contains a mask of a modality of a group
      
      data_format_: str, data format such as 'nii', 'mat'
      
      affine_: 4 by 4 matrix, image affine
  
      id_: subject id
  
      Notes:
      -----
      1. Easylearn allows users to input multiple modalities for one group. 
         Then, easylearn will feed features combined multiple modalities into machine learning model.
      
      2. If there is only one input file for one modality, 
         then data in this file should have a column of "__ID__" (unique idenfity), 
         otherwise easylearn will take the first column as "__ID__".
         So that easylearn can match cases between modalities and match modalities with targets and covariates.
         If this file
      
      3. If ther are multiple input files for one modality, then the files name must contain r'.*(sub.?[0-9].*).*' for 
         extracting unique idenfity information to match like above. 
         For example one file name contains strings of "sub-008.nii".
      
      4. Easylearn only allows users to input targets as one integer by type in the GUI 
         (only for classification) or a file path for one group. 
         If the input targets is a file, then data in the file should have a column of "__ID__" and a column of "__Targets__", 
         otherwise easylearn will take the first column as "__ID__", and the second as "__Targets__".
         
         If the input targets is a integer(only for classification) , 
         then easylearn will assign the integer as target for all case in the group.
  
      5. Easylearn allows users to input covariates, such as age, gender. Now, only file can be input as the covariates.
         If user given easylearn a covariates file, then data in the file should have a column of "__ID__", 
         otherwise easylearn will take the first column as "__ID__".
  
      """
      
      def __init__(self, configuration_file):
          self.configuration_file = configuration_file
          
          # Generate type2fun dictionary
          # TODO: Extended to handle other formats
          self.type2fun = {
                      ".nii": self.read_nii, 
                      ".img": self.read_nii,
                      ".mat": self.read_mat, 
                      ".txt": self.read_csv,
                      ".csv": self.read_csv,
                      ".xlsx": self.read_excel,
                      ".xls": self.read_excel,
                      ".npy": self.read_ndarray,
          }
      
      def get_configuration_(self):
          """Get and parse the configuration file
          """
  
          with open(self.configuration_file, 'r', encoding='utf-8') as config:
                      configuration = config.read()
          self.configuration = json.loads(configuration)
          return self
  
      def load_data(self):
          self.get_configuration_()
          load_data = self.configuration.get('data_loading', None)
          
          #%% ==========================================Check datasets=================================
          # NOTE.: That check whether the feature dimensions of the same modalities in different groups are equal
          # is placed in the next section.
          targets = {}
          self.covariates_ = {}
          for i, gk in enumerate(load_data.keys()):
              
              # Check the number of modality across all group is equal
              if i == 0:
                  n_mod = len(load_data.get(gk).get("modalities").keys())
              else:
                  if n_mod != len(load_data.get(gk).get("modalities").keys()):
                      raise ValueError("The number of modalities in each group is not equal, check your inputs")
                      return
                  n_mod = len(load_data.get(gk).get("modalities").keys())
                  
              # Get targets
              targets_input = load_data.get(gk).get("targets")
              targets[gk] = self.read_targets(targets_input)  
  
      
              # Get covariates
              covariates_input = load_data.get(gk).get("covariates")
              if (isinstance(covariates_input, str) and
                  covariates_input.strip() != "" and
                  (not os.path.isfile(covariates_input))):  # Easylearn only supports file input for covariates
                  raise ValueError("Easylearn only supports file input for covariates, check your covariates for {gk}")
              self.covariates_[gk] = self.base_read(covariates_input)
              
              # Check the number of files in each modalities in the same group is equal
              for j, mk in enumerate(load_data.get(gk).get("modalities").keys()):
                  modality = load_data.get(gk).get("modalities").get(mk)
                  
                  # Filses
                  input_files = modality.get("file")
                  if j == 0:
                      n_file = self.get_file_len(input_files)  # Initialize n_file
                  else:
                      if n_file != self.get_file_len(input_files):  # Left is previous, right is current loop
                          raise ValueError(f"The number of files in each modalities in {gk} is not equal, check your inputs")
                          return
                  n_file = self.get_file_len(input_files)  # Update n_file
  
                  # Check the number of targets in each modalities is equal to the number of files          
                  # If the type of targets is list, and number of files are not equal to targets, then raise error
                  if (isinstance(targets[gk],list)) and (n_file != len(targets[gk])):
                      raise ValueError(f"The number of files in {mk} of {gk} is not equal to the number of targets, check your inputs")
                      return
          
                  # Check the number of lines of covariates in each modalities is equal to the number of files
                  # If covariates is not int (0), and number of files are not equal to covariates, then raise error
                  if (not isinstance(self.covariates_[gk],int)) and (n_file != len(self.covariates_[gk])):
                      raise ValueError(f"The number of files in {mk} of {gk} is not equal to its' number of covariates, check your inputs")
                      return
                  
          #%% ==========================================Get selected datasets =================================
          shape_of_data = {}
          feature_applied_mask_all = {}
          feature_applied_mask_and_add_otherinfo = {}
          col_drop = {}
          self.mask_ = {}
          self.data_format_ = {}
          self.affine_ =  {}
  
          for gi, gk in enumerate(load_data.keys()): 
              col_drop[gk] = ["__Targets__"]
              shape_of_data[gk] = {}
              feature_applied_mask_and_add_otherinfo[gk] = {}
              feature_applied_mask_all[gk] = {}
              self.mask_[gk] = {}
              self.data_format_[gk] = {}
              self.affine_[gk] = {}
              
              for jm, mk in enumerate(load_data.get(gk).get("modalities").keys()):
                  modality = load_data.get(gk).get("modalities").get(mk)
                 
                  # Get files
                  # If only input one file for one modality, 
                  # then I think the file contained multiple cases' data
                  input_files = modality.get("file")
                  n_file = self.get_file_len(input_files)
                  if len(input_files) == 1:
                      one_file_per_modality = True
                  else:
                      one_file_per_modality = False
                  
                  # Get features' format and affine for each modality
                  # I think all files in on modality are in the same format
                  # So I take the first file in corresponding modality as example file
                  # TODO: other situations
                  self.data_format_[gk][mk], self.affine_[gk][mk] = self.get_data_format(input_files[0])
  
                  # Get Features
                  all_features = self.read_file(input_files, False)
                  if one_file_per_modality:
                      all_features_ = list(all_features)[0]
                  else:
                      all_features_ = False
  
                  # Get cases' name (unique ID) in this modality
                  # If one_file_per_modality = False, then each file name must contain r'.*(sub.?[0-9].*).*'
                  # If one_file_per_modality = True and all_features_ is DataFrame, 
                  # then the DataFrame must have header of "__ID__" which contain the unique_identifier,
                  # otherwise easylearn will take the first column as "__ID__".
                  if isinstance(all_features_, pd.core.frame.DataFrame) and ("__ID__" not in all_features_.columns):
                      # raise ValueError(f"The dataset of {input_files} did not have '__ID__' column, check your dataset")
                      unique_identifier = all_features_.iloc[:,0]  # Take the first column as __ID__
                      print(f"The dataset of {input_files} did not have '__ID__' column, easylearn take the first column as ID\n")
                  elif isinstance(all_features_, pd.core.frame.DataFrame) and ("__ID__" in all_features_.columns):
                      unique_identifier = pd.DataFrame(all_features_["__ID__"])
                      all_features_.drop("__ID__", axis=1, inplace=True)
                      all_features = [all_features_]
                  elif isinstance(all_features_, np.ndarray):
                      all_features_ = pd.DataFrame(all_features_)
                      all_features = [all_features_]
                      unique_identifier = pd.DataFrame(all_features_.iloc[:,0], dtype=np.str) # Take the first column as __ID__
                      unique_identifier.columns = ["__ID__"]
                      all_features = [all_features_.iloc[:,1:]]
                  else:
                      autogen = True if (isinstance(targets[gk],int)) else False
                      unique_identifier = self.extract_id(input_files, autogen)  # Multiple files
                          
                  # Apply mask to feature
                  mask_input = modality.get("mask")
                  # Do not extract triangule matrix when read mask file
                  self.mask_[gk][mk] = self.base_read(mask_input)
                  if not isinstance(self.mask_[gk][mk], int):  # If have mask
                     # TODO:  Allow uses to set threshold for mask, now code only set zero threshold.
                     self.mask_[gk][mk] = self.mask_[gk][mk] != 0 
                     # Apply mask
                     feature_applied_mask = [fa[self.mask_[gk][mk]] for fa in all_features]
                     feature_applied_mask = np.array(feature_applied_mask)
                  else:
                     feature_applied_mask = [fa for fa in all_features]
                     feature_applied_mask, self.mask_[gk][mk] = self.get_upper_tri_mat(feature_applied_mask, one_file_per_modality)
                     feature_applied_mask = np.array(feature_applied_mask)
                     feature_applied_mask = feature_applied_mask.reshape(n_file,-1)
  
                  # Concat feature across different modalities and groups 
                  # In addition, the second and later modality are sorted according with the first one using pd.merge method
                  feature_applied_mask = pd.concat([unique_identifier, pd.DataFrame(feature_applied_mask)], axis=1) 
                  if jm == 0:
                      feature_applied_mask_all[gk] = feature_applied_mask
                  else:
                      feature_applied_mask_all[gk] = pd.merge(feature_applied_mask_all[gk], feature_applied_mask, left_on="__ID__", right_on="__ID__")  
  
              # Dropout __ID__ from feature_applied_mask_all
              unique_identifier_ = feature_applied_mask_all[gk]["__ID__"]  # Get the final unique_identifier
              feature_applied_mask_all[gk].drop("__ID__", axis=1, inplace=True)
              
              #%% =====================Match targets and covariates with unique_identifier_==============================
              # NOTE. subj-name is come from the first modality due to the second and later modality are sorted 
              # according with the first one using pd.merge method 
  
              # Sort targets and check
              if (isinstance(targets[gk],int)):
                  targets[gk] = [targets[gk] for ifile in range(n_file)]
                  targets[gk] = pd.DataFrame(targets[gk])
                  targets[gk]["__ID__"] = unique_identifier_
                  targets[gk].rename(columns={0: "__Targets__"}, inplace=True)
              elif isinstance(targets[gk], pd.core.frame.DataFrame) and ("__ID__" not in targets[gk].columns):
                  # raise ValueError(f"The targets of {gk} did not have '__ID__' column, check your targets") 
                  print(f"The targets of {gk} did not have '__ID__' column, easylearn take the first column as ID\n") 
                  # Take the first column as __ID__, and the second column as __Targets__
                  targets[gk].columns = ["__ID__", "__Targets__"] 
              elif isinstance(targets[gk], np.ndarray):
                  targets[gk] = pd.DataFrame(targets[gk])
                  # Take the first column as __ID__, and the second as __Targets__
                  targets[gk].rename(columns={0:"__ID__", 1:"__Targets__"}, inplace=True)
              
              targets[gk] = pd.merge(unique_identifier_, targets[gk], left_on="__ID__", right_on="__ID__", how='inner')
              if targets[gk].shape[0] != n_file:
                      raise ValueError(f"The subjects' ID in targets is not totally matched with its' data file name in {mk} of {gk} , check your ID in targets or check your data file name")
  
              # Sort covariates and check                
              if (not isinstance(self.covariates_[gk],int)):  # User have given covariates
                  if isinstance(self.covariates_[gk], pd.core.frame.DataFrame) and ("__ID__" not in self.covariates_[gk].columns):
                      # raise ValueError(f"The covariates of {gk} did not have 'ID' column, check your covariates")
                      print(f"The covariates of {gk} did not have 'ID' column, easylearn take the first column as ID\n")
                      colname = list(self.covariates_[gk].columns)
                      colname[0] = "__ID__"
                      self.covariates_[gk].columns = colname
                  elif isinstance(self.covariates_[gk], np.ndarray): 
                      self.covariates_[gk] = pd.DataFrame(self.covariates_[gk])
                      self.covariates_[gk].rename(columns={0:"__ID__"}, inplace=True) # Take the first column as __ID__
                  
                  self.covariates_[gk] = pd.merge(unique_identifier_, self.covariates_[gk], left_on="__ID__", right_on="__ID__") 
                  if self.covariates_[gk].shape[0] != n_file:
                      raise ValueError(f"The subjects' ID in covariates is not totally matched with its' data file name in {mk} of {gk} , check your ID in covariates or check your data file name")
                          
              # Check whether the feature dimensions of the same modalities in different groups are equal
              shape_of_data[gk][mk] = feature_applied_mask_all[gk].shape
              if gi == 0:
                  gk_pre = gk
              else:
                  if shape_of_data[gk_pre][mk][-1] != shape_of_data[gk][mk][-1]:
                      raise ValueError(f"Feature dimension of {mk} in {gk_pre} is {shape_of_data[gk_pre][mk][-1]} which is not equal to {mk} in {gk}: {shape_of_data[gk][mk][-1]}, check your inputs")
               
  
              # Concat datasets across different group
              unique_identifier_ = pd.DataFrame([f"{gk}_{ui}" for ui in unique_identifier_])
              if gi == 0:
                  self.id_ = unique_identifier_
                  self.targets_ = targets[gk]["__Targets__"]
                  self.features_ = feature_applied_mask_all[gk]
              else:
                  self.id_ = pd.concat([self.id_, unique_identifier_])
                  self.targets_ = pd.concat([self.targets_,  targets[gk]["__Targets__"]])
                  self.features_ = pd.concat([self.features_, feature_applied_mask_all[gk]], axis=0)
  
          self.id_ = self.id_.values
          self.targets_ = np.float64(self.targets_.values)
          self.features_ = np.float64(self.features_.values)
          return self
         
      #%% ========================utilt functions========================
      def get_file_len(self, files):
          """If the files lenght is 1, then the length is the length of content of the files
          """
          
          file_len = len(files)
          if file_len == 1:
              all_features = self.read_file(files, False)
              all_features = [fe for fe in all_features][0]
              file_len = len(all_features)
          return file_len
          
      def del_id(self, all_features, input_files):
          """Delete "__ID__" in each DataFrame in all_features
  
          At last, the all_features contains only the feature
  
          Parameters:
          ----------
          all_features: list of DataFrames or ndarray
              All features
  
          input_files: list of
          """
          
          all_features_ = list()
          for df, file in zip(all_features, input_files):
              if isinstance(df, pd.core.frame.DataFrame) and ("__ID__" not in df.columns):
                  raise ValueError(f"The dataset of {file} did not have '__ID__' column, check your dataset")
              elif isinstance(df, pd.core.frame.DataFrame) and ("__ID__" in df.columns):
                  df.drop("__ID__", axis=1, inplace=True)
                  all_features_.append(df)
              else:
                  all_features_.append(df[:,1:])  # Take the first column as __ID__, and drop the first column.
                  
          return all_features_
  
      def read_targets(self, targets_input):
          """Read targets from GUI
  
          Easylearn only supports one integer input or a file input for targets
          """
  
          if (targets_input == []) or (targets_input == ''):
              targets = None  # Clustering algorithm does not need targets
          
          if os.path.isfile(targets_input):
              targets = self.base_read(targets_input) 
          elif len(re.findall(r'[A-Za-z]', targets_input)):  # Contain alphabet
              raise ValueError(f"The '{targets_input}' is not exist OR The targets(labels) must be an Arabic numbers or file, but it contain alphabet, check your targets: '{targets_input}'")
              return
          elif ' ' in targets_input:
              targets = targets_input.split(' ')
              targets = [int(targets_) for targets_ in targets]
              if len(targets) > 1:
                raise ValueError(f"The targets(labels) must be an integer or a file, check your targets: {targets_input}\n")
          elif ',' in targets_input:
              targets = targets_input.split(',')
              targets = [int(targets_) for targets_ in targets]
              if len(targets) > 1:
                raise ValueError(f"The targets(labels) must be an integer or a file, check your targets: {targets_input}\n")
          else:
              targets = eval(targets_input)  
              if not isinstance(targets, int):
                raise ValueError(f"The targets(labels) must be an integer or a file for classification, check your targets: {targets_input}\n")      
  
          return targets
  
      def read_file(self, input_files, to1d=False):  
          """Read all input files
          """
  
          data = (self.base_read(file, to1d) for file in input_files)
          return data
  
  
      def base_read(self, file, to1d=False):
          """Read one file
          
          Parameters:
          ----------
          file: Path str
              input file of one case
          
          to1d: Bool
              whether transform data to 1 dimension
          
          Return:
          ------
          data: ndarray
          """
          
          if (file == []) or (file == ''):
              return 0
          elif not os.path.isfile(file):
              raise ValueError(f" Cannot find the file:'{file}'")
              return
          else:
              # Identify file type
              [path, filename] = os.path.split(file)
              suffix = os.path.splitext(filename)[-1]
              # Read
              data = self.type2fun[suffix](file)
  
              if to1d:
                  data = np.reshape(data, [-1,])
                  
          return data
      
      @staticmethod
      def get_data_format(example_file):
          """Get  features' data format and affine
          """
  
          # TODO: considering such as nii.gz in the future
          affine = None
          data_format = example_file.split(".")[-1]
          if data_format in ["nii","gz"]:
              obj = nib.load(example_file)
              affine = obj.affine
          return data_format, affine
  
      @ staticmethod
      def read_nii(file):      
          obj = nib.load(file)
          data = obj.get_fdata()
          return data
  
      @ staticmethod
      def read_mat(file):
          dataset_struct = io.loadmat(file)
          return dataset_struct[list(dataset_struct.keys())[3]]
  
      @ staticmethod
      def read_csv(file):
          data = pd.read_csv(file)                    
          return data
  
      @ staticmethod
      def read_excel(file):
          """
          Not consider symmetric matrix
          """
          
          data = pd.read_excel(file)
          return data
      
      @staticmethod
      def read_ndarray(file):
          try:
              data = np.load(file,  allow_pickle=False)
          except ValueError:
              data = np.load(file,  allow_pickle=True)
          return data
  
      @ staticmethod
      def get_upper_tri_mat(data, one_file_per_modality):
          """Get upper triangular matrix
  
          If the matrix is symmetric, then I extract the upper triangular matrix.
  
          Parameters:
          ----------
          data: list of ndarray or DataFrame
          
          one_file_per_modality: bool
              If one file per modality, which used for generating mask.
  
          Return:
          -------
          data_: list of ndarray or DataFrame
              data that the upper triangular matrix was extracted or the original data.
              
          mask: ndarray
              Internal mask of data (mask sure that all item in data have the same dimension and type) 
          """
  
          # If data is symmetric matrix, then only extract upper triangule matrix
          data_ = []
          for dd in data:
              dd_ = dd.copy()
              if ((len(dd.shape) == 2) and (dd.shape[0] == dd.shape[1])):
                  dd_[np.eye(dd.shape[0]) == 1] = 0
                  if np.all(np.abs(dd_-dd_.T) < 1e-8):  
                      mask = np.triu(np.ones(dd.shape),1) == 1
                      data_.append(dd[mask])
                  else:
                      mask = np.ones(np.shape(dd)) if not one_file_per_modality else np.ones(np.shape(dd)[1]) 
                      data_.append(dd)
              else:
                  mask = np.ones(np.shape(dd)) if not one_file_per_modality else np.ones(np.shape(dd)[1])
                  data_.append(dd)
  
          return data_, mask == 1
              
  
      @staticmethod
      def extract_id(files, autogen=False):
          """Extract subject unique ID from file names
  
          Parameters:
          ----------
          files : list of file paths
  
          Returns:
          -------
          unique_identifier: pd.DataFrame
              Subjects' name or unique idenfity
          """
          if not autogen:
              unique_identifier = [os.path.basename(file).split(".")[0] for file in files]
              unique_identifier = [re.findall(r'.*(sub.?[0-9]+).*', name)[0] 
                                   if re.findall(r'.*(sub.?[0-9]+).*', name) != [] 
                                   else "" for name in unique_identifier]
              unique_identifier = pd.DataFrame(unique_identifier)
              unique_identifier.columns = ["__ID__"]
          else:
              unique_identifier = [i for i, file in enumerate(files)]
              unique_identifier = pd.DataFrame(unique_identifier)
              unique_identifier.columns = ["__ID__"]
          return unique_identifier
  
      