build/lib/AutoMxL/__main__.py
def preprocess(self, date_ref=None, process_outliers=False,
                   cat_method='deep_encoder', verbose=False):
        """Prepare the data before feeding it to the model :

            - remove low variance features
            - remove identifiers and verbatims features
            - transform date features to timedelta
            - fill missing values
            - process categorical and boolean data (one-hot-encoding or Pytorch NN encoder)
            - replace outliers (optional)

            create self.d_preprocess : dict {step : transformation}
                - remove: list of the features to remove
                - date: fitted DateEncoder object
                - NA: fitted NAEncoder object
                - categorical: fitted CategoricalEncoder object
                - outlier: fitted OutlierEncoder object

        Parameters
        ----------
        date_ref : string '%d/%m/%y' (Default : None)
            ref date to compute date features timedelta.
            If None, today date
        process_outliers : boolean (Default : False)
            Enable outliers replacement
        cat_method : string (Default : 'deep_encoder')
            Categorical features encoding method
        verbose : boolean (Default False)
            Get logging information

        """
        # check pipe step
        assert self.step in ['explore'], 'apply explore method first'
        assert not self.is_fitted_preprocessing, 'preprocessing encoders already fitted'

        ###############################
        # Fit and apply preprocessing #
        ###############################
        if verbose:
            start_time = time()
            print_title1('Fit and apply preprocessing')

        target = self.target
        df_local = self.copy()

        # Features Removing 'zero variance / verbatims / identifiers)
        if verbose:
            color_print("Features removing (zero variance / verbatims / identifiers)")

        l_remove = self.d_features['low_variance'] + self.d_features['verbatim'] + self.d_features['identifier']
        if len(l_remove) > 0:
            df_local = df_local.drop(l_remove, axis=1)

        if verbose:
            print("  >", len(l_remove), "features to remove")
            if len(l_remove) > 0:
                print(" ", l_remove)

        # Transform date -> time between date and date_ref
        if verbose:
            color_print("Transform date")

        date_encoder = DateEncoder(method='timedelta', date_ref=date_ref)
        date_encoder.fit(self, l_var=self.d_features['date'], verbose=False)
        df_local = date_encoder.transform(df_local, verbose=verbose)

        # Missing Values
        if verbose:
            color_print('Missing values')

        NA_encoder = NAEncoder()
        NA_encoder.fit(df_local, l_var=None, verbose=False)
        df_local = NA_encoder.transform(df_local, verbose=verbose)

        # replace outliers
        if process_outliers:
            if verbose:
                color_print('Outliers')
            out_encoder = OutliersEncoder()
            out_encoder.fit(df_local, l_var=None, verbose=False)
            df_local = out_encoder.transform(df_local, verbose=verbose)
        else:
            out_encoder = None

        # categorical processing
        if verbose:
            color_print('Encode Categorical and boolean')

        cat_col = self.d_features['categorical'] + self.d_features['boolean']
        # apply one-hot encoding if target not filled in class parameters
        if self.target is None:
            cat_method = 'one_hot'
            color_print('No target -> one_hot encoding !', 31)

        # get embedding
        cat_encoder = CategoricalEncoder(method=cat_method)
        cat_encoder.fit(self, l_var=cat_col, target=self.target, verbose=verbose)
        df_local = cat_encoder.transform(df_local, verbose=verbose)

        # store preprocessing params
        self.d_preprocess = {'remove': l_remove, 'date': date_encoder, 'NA': NA_encoder, 'categorical': cat_encoder}
        if out_encoder is not None:
            self.d_preprocess['outlier'] = out_encoder

        if verbose:
            color_print("\nCreated attributes :  d_preprocess (dict) ")
            print("Keys :")
            print("  -> remove")
            print("  -> date")
            print("  -> NA")
            print("  -> categorical")
            print("  -> outlier (optional)")

        # is_fitted
        self.is_fitted_preprocessing = True

        # update self
        self.__dict__.update(df_local.__dict__)
        self.target = target
        self.step = 'preprocess'

        if verbose:
            color_print("New DataFrame size ")
            print("  > row number : ", self.shape[0], "\n  > col number : ", self.shape[1])
            print('\n\t\t>>>', 'proprocess execution time:', round(time() - start_time, 4), 'secs. <<<')
Prepare the data before feeding it to the model :

            - remove low variance features
            - remove identifiers and verbatims features
            - transform date features to timedelta
            - fill missing values
            - process categorical and boolean data (one-hot-encoding or Pytorch NN encoder)
            - replace outliers (optional)

            create self.d_preprocess : dict {step : transformation}
                - remove: list of the features to remove
                - date: fitted DateEncoder object
                - NA: fitted NAEncoder object
                - categorical: fitted CategoricalEncoder object
                - outlier: fitted OutlierEncoder object

        Parameters
        ----------
        date_ref : string '%d/%m/%y' (Default : None)
            ref date to compute date features timedelta.
            If None, today date
        process_outliers : boolean (Default : False)
            Enable outliers replacement
        cat_method : string (Default : 'deep_encoder')
            Categorical features encoding method
        verbose : boolean (Default False)
            Get logging information

        
Prepare the data before feeding it to the model :
remove low variance features
remove identifiers and verbatims features
transform date features to timedelta
fill missing values
process categorical and boolean data (one-hot-encoding or Pytorch NN encoder)
replace outliers (optional)



Parameters

date_ref : string '%d/%m/%y' (Default : None)
ref date to compute date features timedelta.
Prepare the data before feeding it to the model : remove low variance features remove identifiers and verbatims features transform date features to timedelta fill missing values process categorical and boolean data ( one - hot - encoding or Pytorch NN encoder ) replace outliers ( optional ) Parameters date_ref : string ' %d / %m / %y ' ( Default : None ) ref date to compute date features timedelta .
def preprocess(self, date_ref=None, process_outliers=False,
                   cat_method='deep_encoder', verbose=False):
        assert self.step in ['explore'], 'apply explore method first'
        assert not self.is_fitted_preprocessing, 'preprocessing encoders already fitted'
        if verbose:
            start_time = time()
            print_title1('Fit and apply preprocessing')
        target = self.target
        df_local = self.copy()
        if verbose:
            color_print("Features removing (zero variance / verbatims / identifiers)")
        l_remove = self.d_features['low_variance'] + self.d_features['verbatim'] + self.d_features['identifier']
        if len(l_remove) > 0:
            df_local = df_local.drop(l_remove, axis=1)
        if verbose:
            print("  >", len(l_remove), "features to remove")
            if len(l_remove) > 0:
                print(" ", l_remove)
        if verbose:
            color_print("Transform date")
        date_encoder = DateEncoder(method='timedelta', date_ref=date_ref)
        date_encoder.fit(self, l_var=self.d_features['date'], verbose=False)
        df_local = date_encoder.transform(df_local, verbose=verbose)
        if verbose:
            color_print('Missing values')
        NA_encoder = NAEncoder()
        NA_encoder.fit(df_local, l_var=None, verbose=False)
        df_local = NA_encoder.transform(df_local, verbose=verbose)
        if process_outliers:
            if verbose:
                color_print('Outliers')
            out_encoder = OutliersEncoder()
            out_encoder.fit(df_local, l_var=None, verbose=False)
            df_local = out_encoder.transform(df_local, verbose=verbose)
        else:
            out_encoder = None
        if verbose:
            color_print('Encode Categorical and boolean')
        cat_col = self.d_features['categorical'] + self.d_features['boolean']
        if self.target is None:
            cat_method = 'one_hot'
            color_print('No target -> one_hot encoding !', 31)
        cat_encoder = CategoricalEncoder(method=cat_method)
        cat_encoder.fit(self, l_var=cat_col, target=self.target, verbose=verbose)
        df_local = cat_encoder.transform(df_local, verbose=verbose)
        self.d_preprocess = {'remove': l_remove, 'date': date_encoder, 'NA': NA_encoder, 'categorical': cat_encoder}
        if out_encoder is not None:
            self.d_preprocess['outlier'] = out_encoder
        if verbose:
            color_print("\nCreated attributes :  d_preprocess (dict) ")
            print("Keys :")
            print("  -> remove")
            print("  -> date")
            print("  -> NA")
            print("  -> categorical")
            print("  -> outlier (optional)")
        self.is_fitted_preprocessing = True
        self.__dict__.update(df_local.__dict__)
        self.target = target
        self.step = 'preprocess'
        if verbose:
            color_print("New DataFrame size ")
            print("  > row number : ", self.shape[0], "\n  > col number : ", self.shape[1])
            print('\n\t\t>>>', 'proprocess execution time:', round(time() - start_time, 4), 'secs. <<<')
def preprocess ( self , date_ref = None , process_outliers = False , cat_method = 'deep_encoder' , verbose = False ) : assert self . step in [ 'explore' ] , 'apply explore method first' assert not self . is_fitted_preprocessing , 'preprocessing encoders already fitted' if verbose : start_time = time ( ) print_title1 ( 'Fit and apply preprocessing' ) target = self . target df_local = self . copy ( ) if verbose : color_print ( "Features removing (zero variance / verbatims / identifiers)" ) l_remove = self . d_features [ 'low_variance' ] + self . d_features [ 'verbatim' ] + self . d_features [ 'identifier' ] if len ( l_remove ) > 0 : df_local = df_local . drop ( l_remove , axis = 1 ) if verbose : print ( "  >" , len ( l_remove ) , "features to remove" ) if len ( l_remove ) > 0 : print ( " " , l_remove ) if verbose : color_print ( "Transform date" ) date_encoder = DateEncoder ( method = 'timedelta' , date_ref = date_ref ) date_encoder . fit ( self , l_var = self . d_features [ 'date' ] , verbose = False ) df_local = date_encoder . transform ( df_local , verbose = verbose ) if verbose : color_print ( 'Missing values' ) NA_encoder = NAEncoder ( ) NA_encoder . fit ( df_local , l_var = None , verbose = False ) df_local = NA_encoder . transform ( df_local , verbose = verbose ) if process_outliers : if verbose : color_print ( 'Outliers' ) out_encoder = OutliersEncoder ( ) out_encoder . fit ( df_local , l_var = None , verbose = False ) df_local = out_encoder . transform ( df_local , verbose = verbose ) else : out_encoder = None if verbose : color_print ( 'Encode Categorical and boolean' ) cat_col = self . d_features [ 'categorical' ] + self . d_features [ 'boolean' ] if self . target is None : cat_method = 'one_hot' color_print ( 'No target -> one_hot encoding !' , 31 ) cat_encoder = CategoricalEncoder ( method = cat_method ) cat_encoder . fit ( self , l_var = cat_col , target = self . target , verbose = verbose ) df_local = cat_encoder . transform ( df_local , verbose = verbose ) self . d_preprocess = { 'remove' : l_remove , 'date' : date_encoder , 'NA' : NA_encoder , 'categorical' : cat_encoder } if out_encoder is not None : self . d_preprocess [ 'outlier' ] = out_encoder if verbose : color_print ( "\nCreated attributes :  d_preprocess (dict) " ) print ( "Keys :" ) print ( "  -> remove" ) print ( "  -> date" ) print ( "  -> NA" ) print ( "  -> categorical" ) print ( "  -> outlier (optional)" ) self . is_fitted_preprocessing = True self . __dict__ . update ( df_local . __dict__ ) self . target = target self . step = 'preprocess' if verbose : color_print ( "New DataFrame size " ) print ( "  > row number : " , self . shape [ 0 ] , "\n  > col number : " , self . shape [ 1 ] ) print ( '\n\t\t>>>' , 'proprocess execution time:' , round ( time ( ) - start_time , 4 ) , 'secs. <<<' )
Prepare the data before feeding it to the model :
 remove low variance features
 remove identifiers and verbatims features
 transform date features to timedelta
 fill missing values
 process categorical and boolean data (one-hot-encoding or Pytorch NN encoder)
 replace outliers (optional)
Prepare the data before feeding it to the model : remove low variance features remove identifiers and verbatims features transform date features to timedelta fill missing values process categorical and boolean data ( one - hot - encoding or Pytorch NN encoder ) replace outliers ( optional )
"""Prepare the data before feeding it to the model :

            - remove low variance features
            - remove identifiers and verbatims features
            - transform date features to timedelta
            - fill missing values
            - process categorical and boolean data (one-hot-encoding or Pytorch NN encoder)
            - replace outliers (optional)

            create self.d_preprocess : dict {step : transformation}
                - remove: list of the features to remove
                - date: fitted DateEncoder object
                - NA: fitted NAEncoder object
                - categorical: fitted CategoricalEncoder object
                - outlier: fitted OutlierEncoder object

        Parameters
        ----------
        date_ref : string '%d/%m/%y' (Default : None)
            ref date to compute date features timedelta.
            If None, today date
        process_outliers : boolean (Default : False)
            Enable outliers replacement
        cat_method : string (Default : 'deep_encoder')
            Categorical features encoding method
        verbose : boolean (Default False)
            Get logging information

        """ # check pipe step ############################### # Fit and apply preprocessing # ############################### # Features Removing 'zero variance / verbatims / identifiers) # Transform date -> time between date and date_ref # Missing Values # replace outliers # categorical processing # apply one-hot encoding if target not filled in class parameters # get embedding # store preprocessing params # is_fitted # update self
self
date_ref
process_outliers
cat_method
verbose