utils4text.py
def remove_toxicity(df):
    """
    Use parallel computing. Consider only one post at each time.
    In each post, detect the toxicity and remove the following dialog turns.

    Arg:
        df: A given conversation dataframe from a certain subreddit.
    Return:
        df_clean: A cleaner version of the conversation dataframe with no toxic words.
    """
    # initialize clean conversation dataframe
    df_clean = pd.DataFrame(columns = ['conversation id', 'subreddit', 'post title', 'author', 'dialog turn', 'text'])
    
    # consider each post
    df_post = df.reset_index().drop('index', axis = 1)
    clean_row_list = []

    # iterate over each comment from df_link_id
    for i, row in df_post.iterrows():
        if predict_prob([row['text']])[0] > 0.95 and row['dialog turn'] > 1:
            break
        else:
            clean_row_list.append({'conversation id': row['conversation id'], 'subreddit': row['subreddit'],
                                   'post title': row['post title'], 'author': row['author'],
                                   'dialog turn': row['dialog turn'], 'text': row['text']})
        
    df_clean = df_clean.append(pd.DataFrame(clean_row_list))

    return df_clean

    Use parallel computing. Consider only one post at each time.
    In each post, detect the toxicity and remove the following dialog turns.

    Arg:
        df: A given conversation dataframe from a certain subreddit.
    Return:
        df_clean: A cleaner version of the conversation dataframe with no toxic words.
    
Use parallel computing. Consider only one post at each time.
In each post, detect the toxicity and remove the following dialog turns.

A given conversation dataframe from a certain subreddit.
Return:
df_clean: A cleaner version of the conversation dataframe with no toxic words.
Use parallel computing . Consider only one post at each time . In each post detect the toxicity and remove the following dialog turns . A given conversation dataframe from a certain subreddit . Return : df_clean : A cleaner version of the conversation dataframe with no toxic words .
def remove_toxicity(df):
    df_clean = pd.DataFrame(columns = ['conversation id', 'subreddit', 'post title', 'author', 'dialog turn', 'text'])
    df_post = df.reset_index().drop('index', axis = 1)
    clean_row_list = []
    for i, row in df_post.iterrows():
        if predict_prob([row['text']])[0] > 0.95 and row['dialog turn'] > 1:
            break
        else:
            clean_row_list.append({'conversation id': row['conversation id'], 'subreddit': row['subreddit'],
                                   'post title': row['post title'], 'author': row['author'],
                                   'dialog turn': row['dialog turn'], 'text': row['text']})
    df_clean = df_clean.append(pd.DataFrame(clean_row_list))
    return df_clean
def remove_toxicity ( df ) : df_clean = pd . DataFrame ( columns = [ 'conversation id' , 'subreddit' , 'post title' , 'author' , 'dialog turn' , 'text' ] ) df_post = df . reset_index ( ) . drop ( 'index' , axis = 1 ) clean_row_list = [ ] for i , row in df_post . iterrows ( ) : if predict_prob ( [ row [ 'text' ] ] ) [ 0 ] > 0.95 and row [ 'dialog turn' ] > 1 : break else : clean_row_list . append ( { 'conversation id' : row [ 'conversation id' ] , 'subreddit' : row [ 'subreddit' ] , 'post title' : row [ 'post title' ] , 'author' : row [ 'author' ] , 'dialog turn' : row [ 'dialog turn' ] , 'text' : row [ 'text' ] } ) df_clean = df_clean . append ( pd . DataFrame ( clean_row_list ) ) return df_clean
Use parallel computing.
Use parallel computing .
"""
    Use parallel computing. Consider only one post at each time.
    In each post, detect the toxicity and remove the following dialog turns.

    Arg:
        df: A given conversation dataframe from a certain subreddit.
    Return:
        df_clean: A cleaner version of the conversation dataframe with no toxic words.
    """ # initialize clean conversation dataframe # consider each post # iterate over each comment from df_link_id
df