# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python (moneyball)
#     language: python
#     name: moneyball
# ---

# # Vectorizing Raw Data: Count Vectorization

# ### Count vectorization 
#
# Creates a document-term matrix where the entry of each cell will be a count of the number of times that word occurred in that document.

# ### Read in text

# +
import pandas as pd
import re
import string
import nltk
pd.set_option('display.max_colwidth', 100)

stopwords = nltk.corpus.stopwords.words('english')
ps = nltk.PorterStemmer() # faster, but not as good

data = pd.read_csv("SMSSpamCollection.tsv", sep='\t')
data.columns = ['label', 'body_text']
# -

data.shape


# ### Create function to remove punctuation, tokenize, remove stopwords, and stem

def clean_text(text):
    text = "".join([word.lower() for word in text if word not in string.punctuation])
    tokens = re.split('\W+', text)
    text = [ps.stem(word) for word in tokens if word not in stopwords]
    return text


# ### Apply CountVectorizer

# +
from sklearn.feature_extraction.text import CountVectorizer

count_vect = CountVectorizer(analyzer=clean_text)
X_counts = count_vect.fit_transform(data['body_text'])
print(X_counts.shape)
print(count_vect.get_feature_names())
# -

# ### Apply CountVectorizer to smaller sample

# +
data_sample = data[0:20]

count_vect_sample = CountVectorizer(analyzer=clean_text)
X_counts_sample = count_vect_sample.fit_transform(data_sample['body_text'])
print(X_counts_sample.shape)
print(count_vect_sample.get_feature_names())
# -

# ### Vectorizers output sparse matrices
#
# _**Sparse Matrix**: A matrix in which most entries are 0. In the interest of efficient storage, a sparse matrix will be stored by only storing the locations of the non-zero elements._

X_counts_sample

X_counts_df = pd.DataFrame(X_counts_sample.toarray())
X_counts_df

X_counts_df.columns = count_vect_sample.get_feature_names()
X_counts_df





StarcoderdataJupyterScriptsDedupFiltered