# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + id="ZmZSC_TJS1ep" colab_type="code" outputId="c3c95b6b-3766-4622-cff8-d4636f3c3a9f" executionInfo={"status": "ok", "timestamp": 1581650314965, "user_tz": 420, "elapsed": 1330, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 50}
import numpy as np
import pandas as pd

import string
import re

import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

# + id="4iHQfGB_S1ey" colab_type="code" colab={}
text = "Yes, that is a duplicate Catalog category. The Catalog number is C1357-A."

spacetext = "Yes, that is a duplicate   Catalog \t category. The   Catalog    number is C1357-A.\n"

doc = """Python is an interpreted, high-level, general-purpose programming 
language. Created by <NAME> and first released in 1991, Python has a 
design philosophy that emphasizes code readability, notably using significant 
whitespace. It provides constructs that enable clear programming on both small 
and large scales.[26] Van Rossum led the language community until stepping 
down as leader in July 2018.[27][28] Python features a dynamic type system 
and automatic memory management. It supports multiple programming paradigms, 
including object-oriented, imperative, functional and procedural, and has a 
large and comprehensive standard library.[29] Python interpreters are 
available for many operating systems. CPython, the reference implementation of 
Python, is open source software[30] and has a community-based development 
model, as do nearly all of Python's other implementations. Python and CPython 
are managed by the non-profit Python Software Foundation."""

urltext = """The url for sklearn documentation is https://scikit-learn.org/stable/. 
You can learn more about pipelines by following these links: https://scikit-learn.org/stable/modules/compose.html
and https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline"""

corpus = ["This is a brown house. This house is big.",
          "This is a small house. This house has 1 bedroom.",
          "This dog is brown. This dog likes to play",
          "The dog is in the bedroom."]

# + id="GpHfwKusS1e6" colab_type="code" outputId="2da10d6e-c314-4880-f236-11817e4b130a" executionInfo={"status": "ok", "timestamp": 1581650314968, "user_tz": 420, "elapsed": 1311, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
text.lower()

# + id="7JABuTUTTC8k" colab_type="code" outputId="3874ed4a-ef24-48ab-fb59-7fa2695438c4" executionInfo={"status": "ok", "timestamp": 1581650314968, "user_tz": 420, "elapsed": 1291, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
text.replace("cat", "#")

# + id="EShR9x9pTRty" colab_type="code" outputId="aac66f00-8be1-4ba3-961f-141910b51570" executionInfo={"status": "ok", "timestamp": 1581650314969, "user_tz": 420, "elapsed": 1284, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"cat")
regex.sub('#', text)

# + id="cXqJD4kKUfsn" colab_type="code" outputId="a7e9e41c-aaa6-4efa-930c-8596c4d0c2ee" executionInfo={"status": "ok", "timestamp": 1581650314970, "user_tz": 420, "elapsed": 1275, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"cat", re.IGNORECASE)
regex.sub('#', text)

# + id="UVCZvCxBU25i" colab_type="code" outputId="e6d85411-5836-4959-99cc-d4321ff9a14e" executionInfo={"status": "ok", "timestamp": 1581650314970, "user_tz": 420, "elapsed": 1263, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"\bcat")
regex.sub('#', text)

# + id="zWHAyamTWPvr" colab_type="code" outputId="664b074a-00bd-4c84-e7b4-1d314158eaaa" executionInfo={"status": "ok", "timestamp": 1581650314971, "user_tz": 420, "elapsed": 1253, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"[cat]", re.IGNORECASE)
regex.sub('#', text)

# + id="ZtV4GkkvXcV5" colab_type="code" outputId="cbecdaf1-0daa-40ef-fa7f-50e1dc695de2" executionInfo={"status": "ok", "timestamp": 1581650314971, "user_tz": 420, "elapsed": 1241, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"[a-z]", re.IGNORECASE)
regex.sub('#', text)

# + id="9wGC_JOaX3PO" colab_type="code" outputId="4ece9ae1-76be-43c1-a1d9-6d5bee2eb150" executionInfo={"status": "ok", "timestamp": 1581650314972, "user_tz": 420, "elapsed": 1234, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"[0-9]")
regex.sub('#', text)

# + id="jegnBI9yX-Q0" colab_type="code" outputId="53aaaf2f-c20a-4299-fbee-f7de617a71e3" executionInfo={"status": "ok", "timestamp": 1581650314972, "user_tz": 420, "elapsed": 1223, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"\d")
regex.sub('#', text)

# + id="rF4rLDNWYpcu" colab_type="code" outputId="468b7ebd-7f65-4ed0-f801-df39473ed0fd" executionInfo={"status": "ok", "timestamp": 1581650315265, "user_tz": 420, "elapsed": 1507, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"\d[0-9]")
regex.sub('#', text)

# + id="z10Al9H1Yr3y" colab_type="code" outputId="1e4bcaf1-5989-4b5e-dc24-eb6732500084" executionInfo={"status": "ok", "timestamp": 1581650315266, "user_tz": 420, "elapsed": 1496, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 67}
print(spacetext)
regex = re.compile(r"\s+")
regex.sub(' ', spacetext)

# + id="UfNVyZbXZ6Pv" colab_type="code" outputId="090572c3-c75e-48e0-8c20-ef995db5644e" executionInfo={"status": "ok", "timestamp": 1581650315267, "user_tz": 420, "elapsed": 1485, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 50}
t = "Yes, that is a dupli+cate   Catalog	cate+gory. The   Catalog    number is C1357-A"
print(t)
regex = re.compile(r"[\s+]")
regex.sub('#', t)

# + id="vUqMjqeNaKdo" colab_type="code" outputId="04e61284-7e4e-4ee1-8891-611a5fcff349" executionInfo={"status": "ok", "timestamp": 1581650315267, "user_tz": 420, "elapsed": 1475, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"\b\w\w+\b")   #two or more characters
regex.sub('#', text)

# + id="P-Z9w5XEa-im" colab_type="code" outputId="7cac8339-b57d-4953-c9c6-24e85efd3b82" executionInfo={"status": "ok", "timestamp": 1581650315268, "user_tz": 420, "elapsed": 1467, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 33}
regex = re.compile(r"\b\w{2,}\b")   #two or more characters
regex.sub('#', text)

# + id="OkF7maFjbZPl" colab_type="code" outputId="56ef5e14-f6bc-4c01-e409-74eaed65c519" executionInfo={"status": "ok", "timestamp": 1581650315268, "user_tz": 420, "elapsed": 1460, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 201}
regex = re.compile(r"\b\w{2,}\b")   #two or more characters
regex.findall(text)

# + id="SmkJTam6b3BO" colab_type="code" outputId="423cb3ac-4ca4-4748-8041-e1c08815452c" executionInfo={"status": "ok", "timestamp": 1581650315270, "user_tz": 420, "elapsed": 1455, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 117}
print(urltext)
regex = re.compile(r"\bhttps?://\S+\b")   #\S is any non space character. you can also use r"\bhttps?\S\b"
regex.findall(urltext)

# + id="cxScqlQqickY" colab_type="code" outputId="4fe250b1-6774-4757-c3bb-176bdf2f7bda" executionInfo={"status": "ok", "timestamp": 1581650315270, "user_tz": 420, "elapsed": 1448, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 50}
print(text)
regex_punct = r"[%s]"%string.punctuation
regex = re.compile(regex_punct)
regex.sub("#", text)

# + id="lwzrrTRljJJM" colab_type="code" outputId="2ee0655e-baa9-4e7f-883e-3361fa12e0c2" executionInfo={"status": "ok", "timestamp": 1581650315271, "user_tz": 420, "elapsed": 1441, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 87}
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
print(stopwords.words('english'))

# + id="Oum_Vbfjj1gE" colab_type="code" outputId="81d977bd-73f9-46f8-95d3-ebcbef22ae03" executionInfo={"status": "ok", "timestamp": 1581650315271, "user_tz": 420, "elapsed": 1433, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 50}
print(text)
sw = stopwords.words('english')
word_list = text.split()
new_word_list = [ word for word in word_list if word.lower() not in sw]
new_text = " ".join(new_word_list)
print(new_text)


# + id="9_BKItS1l6-w" colab_type="code" colab={}
def preprocess(txt):
  t1 = txt.lower()
  t2 = re.sub(r"[0-9]","", t1)
  t3 = re.sub(r"[%s]"%string.punctuation, "", t2)
  t4 = re.sub(r"\s+", " ", t3)
  sw = stopwords.words('english')
  t5 = " ".join(word for word in t4.split() if word not in sw)
  t6 = " ".join(re.findall(r"\b\w\w\w+\b", t5))
  return t6


# + id="FFQD5MeWqTNu" colab_type="code" outputId="d00ed414-a4c2-4f1d-ec10-e9e1c54f7fec" executionInfo={"status": "ok", "timestamp": 1581650315652, "user_tz": 420, "elapsed": 1770, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 100}
new_corpus = pd.Series([doc for doc in corpus]).apply(preprocess)
new_corpus

# + id="ZrAYkTSFnZ87" colab_type="code" outputId="9b273912-ffda-4759-8188-5f026ec34b5e" executionInfo={"status": "ok", "timestamp": 1581650315272, "user_tz": 420, "elapsed": 1422, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 50}
t = preprocess(text)
print(text,"\n",t)

# + id="6WAch8wRrU4k" colab_type="code" outputId="a318ee5c-65e2-4d4c-eb4b-7560cfcfd852" executionInfo={"status": "ok", "timestamp": 1581650315273, "user_tz": 420, "elapsed": 1415, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 84}
corpus1 = list(map(preprocess, corpus))
corpus1

# + id="0I3c87g-sIyR" colab_type="code" outputId="25b30016-314d-4f8d-eceb-94ab9eea8d25" executionInfo={"status": "ok", "timestamp": 1581650330042, "user_tz": 420, "elapsed": 558, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 84}
corpus2 = [c.split() for c in corpus1]
corpus2

# + id="bi1uNHUusYu-" colab_type="code" outputId="9a77ddc7-5f61-4ade-9d94-7671e1c4c9e8" executionInfo={"status": "ok", "timestamp": 1581650416363, "user_tz": 420, "elapsed": 551, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 284}
corpus3  = [pd.Series(w) for w in corpus2]
corpus3

# + id="nMxi8kBZsYxz" colab_type="code" outputId="30208d8a-ef9a-48fd-afa7-0dec8949359b" executionInfo={"status": "ok", "timestamp": 1581650592160, "user_tz": 420, "elapsed": 2001, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 234}
freq_dist = [w.value_counts() for w in corpus3]
freq_dist

# + id="Ur3t-R2esY0Z" colab_type="code" outputId="19588fac-8a56-4221-924f-155c04bda106" executionInfo={"status": "ok", "timestamp": 1581650768724, "user_tz": 420, "elapsed": 510, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 166}
df = pd.DataFrame(freq_dist)
df

# + id="LDa3HwgpsY23" colab_type="code" outputId="ee15236f-9473-4d7d-9e31-a61c72e9733c" executionInfo={"status": "ok", "timestamp": 1581650813237, "user_tz": 420, "elapsed": 552, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "08629147319952024849"}} colab={"base_uri": "https://localhost:8080/", "height": 166}
df = df.fillna('0')
df

# + id="n9XsMi8HsY5j" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered