def make_texts():
    return [" ".join(np.random.permutation(sents)) for sents in brown.sents()]

brown_df = pd.DataFrame({
    'text': make_texts() + make_texts() + make_texts() + make_texts()
})

brown_df.sample(3)

brown_df.shape

def to_lowercase(input_string):
    return input_string.lower()

def replace_digits_with_token(input_string):
    return re.sub(r"\b\d+\b","tok_num", input_string)

def get_text_length(input_string):
    return len(re.split(r"(?:\s+)|(?:,)|(?:\-)",input_string))

def process_df(df):
    
    output_df = df.copy()
    
    # replace weird double quotes with normal ones
    output_df['text']      = output_df['text'].apply(lambda text: text.replace("``",'"'))

    # text to lower case
    output_df['text']      = output_df['text'].apply(lambda text: text.lower())
    
    # replace number with a special token
    output_df['text']      = output_df['text'].apply(lambda text: re.sub(r"\b\d+\b","tok_num", text))
    
    # take out texts that are too large or too small
    output_df['num_words'] = output_df['text'].apply(lambda text: len(re.split(r"(?:\s+)|(?:,)|(?:\-)",text)))   
        
    indices_to_remove_too_large = output_df[output_df['num_words'] > 50]
    output_df.drop(indices_to_remove_too_large.index, inplace=True)
    
    indices_to_remove_too_small = output_df[output_df['num_words'] < 10]
    output_df.drop(indices_to_remove_too_small.index, inplace=True)    
    
    output_df.reset_index(drop=True, inplace=True)
    
    return output_df

%%time
processed_df = process_df(brown_df)

processed_df.head()

processed_df.shape

"""
Explanation: generate a dataset that 4 times as big as the brown corpus by generating random permutations
End of explanation
"""


NUM_CORES = 8
df_chunks = np.array_split(brown_df,NUM_CORES)

%%time

with multiprocessing.Pool(NUM_CORES) as pool:
    processed_df = pd.concat(pool.map(process_df, df_chunks), ignore_index=True)

processed_df.shape

"""
Explanation: parallel version
End of explanation
"""


