feature-engineering-and-data-transformations/using-toy-exmaples/feature_cleaning/rare_values.py
def grouping(self, X_in, threshold, mapping=None, cols=None):
        """
        Grouping the observations that show rare labels into a unique category ('rare')

        """

        X = X_in.copy(deep=True)

#        if cols is None:
#            cols = X.columns.values

        if mapping is not None:  # transform
            mapping_out = mapping
            for i in mapping:
                column = i.get('col') # get the column name
                X[column] = X[column].map(i['mapping'])

#                try:
#                    X[column] = X[column].astype(int)
#                except ValueError as e:
#                    X[column] = X[column].astype(float)
        else: # fit
            mapping_out = []
            for col in cols:
#                if util.is_category(X[col].dtype):
#                    categories = X[col].cat.categories
#                else:
                temp_df = pd.Series(X[col].value_counts()/len(X))
                mapping = { k: ('rare' if k not in temp_df[temp_df >= threshold].index else k)
                          for k in temp_df.index}

                mapping = pd.Series(mapping)
                mapping_out.append({'col': col, 'mapping': mapping, 'data_type': X[col].dtype}, )

        return X, mapping_out

        Grouping the observations that show rare labels into a unique category ('rare')

        
Grouping the observations that show rare labels into a unique category ('rare')
Grouping the observations that show rare labels into a unique category ( ' rare ' )
def grouping(self, X_in, threshold, mapping=None, cols=None):
        X = X_in.copy(deep=True)
        if mapping is not None:  
            mapping_out = mapping
            for i in mapping:
                column = i.get('col') 
                X[column] = X[column].map(i['mapping'])
        else: 
            mapping_out = []
            for col in cols:
                temp_df = pd.Series(X[col].value_counts()/len(X))
                mapping = { k: ('rare' if k not in temp_df[temp_df >= threshold].index else k)
                          for k in temp_df.index}
                mapping = pd.Series(mapping)
                mapping_out.append({'col': col, 'mapping': mapping, 'data_type': X[col].dtype}, )
        return X, mapping_out
def grouping ( self , X_in , threshold , mapping = None , cols = None ) : X = X_in . copy ( deep = True ) if mapping is not None : mapping_out = mapping for i in mapping : column = i . get ( 'col' ) X [ column ] = X [ column ] . map ( i [ 'mapping' ] ) else : mapping_out = [ ] for col in cols : temp_df = pd . Series ( X [ col ] . value_counts ( ) / len ( X ) ) mapping = { k : ( 'rare' if k not in temp_df [ temp_df >= threshold ] . index else k ) for k in temp_df . index } mapping = pd . Series ( mapping ) mapping_out . append ( { 'col' : col , 'mapping' : mapping , 'data_type' : X [ col ] . dtype } , ) return X , mapping_out
Grouping the observations that show rare labels into a unique category ('rare')
Grouping the observations that show rare labels into a unique category ( ' rare ' )
"""
        Grouping the observations that show rare labels into a unique category ('rare')

        """ #        if cols is None: #            cols = X.columns.values # transform # get the column name #                try: #                    X[column] = X[column].astype(int) #                except ValueError as e: #                    X[column] = X[column].astype(float) # fit #                if util.is_category(X[col].dtype): #                    categories = X[col].cat.categories #                else:
self
X_in
threshold
mapping
cols