models/train_classifier.py
def oversample_nlp(dataframe):
    """
    Oversample the minor categories using synonyms.

    :param dataframe: pandas format dataframe to be oversampled
    :return: oversampled dataframe
    """
    obj = dataframe.mean()
    obj = obj.sort_values(ascending=False)

    columns_perc = {}

    new_dataframe = pd.DataFrame(columns=dataframe.columns)

    for key, value in obj.iteritems():
        if key not in ['id', 'message', 'original', 'genre']:
            columns_perc[key] = round(value * 100, 1)

    for column in dataframe.columns:

        if column not in ['id', 'message', 'original', 'genre']:

            if columns_perc[column] < 10:

                # Selecting class slice
                feature = column
                mask = dataframe[feature] == 1
                df_temp = dataframe.loc[mask].copy()

                rows_target = int(dataframe.shape[0] * 0.10) - df_temp.shape[0]

                obj_temp = df_temp.mean()
                obj_temp = obj_temp.sort_values(ascending=False)

                # Filtering data
                for key, value in obj_temp.iteritems():
                    if key not in ['id', 'message', 'original', 'genre']:
                        if (value * 100) > 50:
                            if value * 100 == 100:
                                pass
                            else:
                                mask = df_temp[key] == 0
                                df_temp = df_temp.loc[mask]

                df_temp.reset_index(inplace=True, drop=True)
                new_data = pd.DataFrame(columns=df_temp.columns)

                if df_temp.shape[0] > 0:

                    for _ in range(rows_target):
                        row_number = randint(0, df_temp.shape[0] - 1)

                        new_row = df_temp.iloc[row_number]
                        transformed_msg = synonym_tokenize(new_row['message'])
                        transformed_msg = ' '.join(transformed_msg)
                        new_row[0:]['message'] = transformed_msg
                        new_row[0:]['id'] = 'Artificial Data'
                        new_data = new_data.append(new_row)

                    print(f"{column} ADDED: {new_data.shape[0]}")

                    new_dataframe = new_dataframe.append(new_data)

    dataframe = dataframe.append(new_dataframe)

    return dataframe

    Oversample the minor categories using synonyms.

    :param dataframe: pandas format dataframe to be oversampled
    :return: oversampled dataframe
    
Oversample the minor categories using synonyms.
Oversample the minor categories using synonyms .
def oversample_nlp(dataframe):
    obj = dataframe.mean()
    obj = obj.sort_values(ascending=False)
    columns_perc = {}
    new_dataframe = pd.DataFrame(columns=dataframe.columns)
    for key, value in obj.iteritems():
        if key not in ['id', 'message', 'original', 'genre']:
            columns_perc[key] = round(value * 100, 1)
    for column in dataframe.columns:
        if column not in ['id', 'message', 'original', 'genre']:
            if columns_perc[column] < 10:
                feature = column
                mask = dataframe[feature] == 1
                df_temp = dataframe.loc[mask].copy()
                rows_target = int(dataframe.shape[0] * 0.10) - df_temp.shape[0]
                obj_temp = df_temp.mean()
                obj_temp = obj_temp.sort_values(ascending=False)
                for key, value in obj_temp.iteritems():
                    if key not in ['id', 'message', 'original', 'genre']:
                        if (value * 100) > 50:
                            if value * 100 == 100:
                                pass
                            else:
                                mask = df_temp[key] == 0
                                df_temp = df_temp.loc[mask]
                df_temp.reset_index(inplace=True, drop=True)
                new_data = pd.DataFrame(columns=df_temp.columns)
                if df_temp.shape[0] > 0:
                    for _ in range(rows_target):
                        row_number = randint(0, df_temp.shape[0] - 1)
                        new_row = df_temp.iloc[row_number]
                        transformed_msg = synonym_tokenize(new_row['message'])
                        transformed_msg = ' '.join(transformed_msg)
                        new_row[0:]['message'] = transformed_msg
                        new_row[0:]['id'] = 'Artificial Data'
                        new_data = new_data.append(new_row)
                    print(f"{column} ADDED: {new_data.shape[0]}")
                    new_dataframe = new_dataframe.append(new_data)
    dataframe = dataframe.append(new_dataframe)
    return dataframe
def oversample_nlp ( dataframe ) : obj = dataframe . mean ( ) obj = obj . sort_values ( ascending = False ) columns_perc = { } new_dataframe = pd . DataFrame ( columns = dataframe . columns ) for key , value in obj . iteritems ( ) : if key not in [ 'id' , 'message' , 'original' , 'genre' ] : columns_perc [ key ] = round ( value * 100 , 1 ) for column in dataframe . columns : if column not in [ 'id' , 'message' , 'original' , 'genre' ] : if columns_perc [ column ] < 10 : feature = column mask = dataframe [ feature ] == 1 df_temp = dataframe . loc [ mask ] . copy ( ) rows_target = int ( dataframe . shape [ 0 ] * 0.10 ) - df_temp . shape [ 0 ] obj_temp = df_temp . mean ( ) obj_temp = obj_temp . sort_values ( ascending = False ) for key , value in obj_temp . iteritems ( ) : if key not in [ 'id' , 'message' , 'original' , 'genre' ] : if ( value * 100 ) > 50 : if value * 100 == 100 : pass else : mask = df_temp [ key ] == 0 df_temp = df_temp . loc [ mask ] df_temp . reset_index ( inplace = True , drop = True ) new_data = pd . DataFrame ( columns = df_temp . columns ) if df_temp . shape [ 0 ] > 0 : for _ in range ( rows_target ) : row_number = randint ( 0 , df_temp . shape [ 0 ] - 1 ) new_row = df_temp . iloc [ row_number ] transformed_msg = synonym_tokenize ( new_row [ 'message' ] ) transformed_msg = ' ' . join ( transformed_msg ) new_row [ 0 : ] [ 'message' ] = transformed_msg new_row [ 0 : ] [ 'id' ] = 'Artificial Data' new_data = new_data . append ( new_row ) print ( f"{column} ADDED: {new_data.shape[0]}" ) new_dataframe = new_dataframe . append ( new_data ) dataframe = dataframe . append ( new_dataframe ) return dataframe
Oversample the minor categories using synonyms.
Oversample the minor categories using synonyms .
"""
    Oversample the minor categories using synonyms.

    :param dataframe: pandas format dataframe to be oversampled
    :return: oversampled dataframe
    """ # Selecting class slice # Filtering data
None
dataframe
pandas format dataframe to be oversampled
pandas format dataframe to be oversampled