# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Preprocessing text for Deep Learning
#
# This notebooks performs basic preprocessing on the Quora datasets using the method(s) from `common.nlp.sequence_preprocessing`. The goal is to prepare the data for using sequence models. This means some information, such as punctuation and capitals, are removed. This information should be captured in another process if we want to use it.

# +
import sys
sys.path.append("../..")
import numpy as np
import pandas as pd

from common.nlp.sequence_preprocessing import WORD_MAP, preprocess_text_for_dl
# -

# load data with right data types (this is important for the IDs in particular)
dtypes = {"qid": str, "question_text": str, "target": int}
train = pd.read_csv("../data/train.csv", dtype=dtypes)
test = pd.read_csv("../data/test.csv", dtype=dtypes)

# ### Preprocessing steps
# The `preprocess_text_for_DL` method performs the following steps:
# 1. Convert text to lower case
# 2. Replace shorthand writing (such as `won't`) to their full form (`will not`)
# 3. Selectively remove, retain or ignore punctuations
# 4. Replace numbers with # (eg: 1 by #, 23 by ##, 1993 by ####)
#
# The `common.nlp.sequence_preprocessing` module contains a variable `WORD_MAP` that specifies the mapping to use in step 2. This mapping is used by default, but can be overridden in the function call.

WORD_MAP

# The punctuation that is removed is that from `string.punctuation`.

import string
string.punctuation

# Preprocessing can be done for a single dataset or more datasets at once (all positional arguments are assumed to be datasets).

# one dataset example
just_train = preprocess_text_for_dl(train, text_col="question_text", word_map=WORD_MAP, puncts_ignore='/-', puncts_retain='&')
just_train.head()

just_train['question_text'][7]

# multiple dataset example
cleaned_train, cleaned_test = preprocess_text_for_dl(train, test, puncts_ignore='/-', puncts_retain='&')
cleaned_train.shape, cleaned_test.shape


# ### Results
# Let's look at some (random) results.

def sample_result(idx=None):
    if idx is None:
        idx = np.random.randint(0, len(cleaned_train))
    print(idx)
    print(train["question_text"].iloc[idx])
    print(cleaned_train["question_text"].iloc[idx])


# run this cell to see a random preprocessed instance and its original
sample_result()

# #### Some examples we might need to deal with differently...

sample_result(1287932)

# Should hyphens be replaced with a space instead of removed? <b>(Handled!)</b>

sample_result(378162)

# With slashes, maybe we should just choose one of the options (e.g., the most common one in the corpus?) to maintain a logical sentence? <b>(Handled!)</b>

sample_result(1012331)

# Maybe we should replace $ with `dollar` and replace general abbreviations like y.o. (years old) with full words?
#
# Also, we probably have to replace digits.

sample_result(1068603)

# What about `'s` that indicates possesion? By removing the apostrofe, the word turns into plural form or results in a misspelling of a name like here.

sample_result(535806)

# Some weird characters are still there apparently.
#
# __For the biggest part it seems to work fine though.__

StarcoderdataJupyterScriptsDedupFiltered