import pandas as pd
import numpy as np
import scipy as sp
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
# from textblob import TextBlob, Word
from nltk.stem.snowball import SnowballStemmer
%matplotlib inline

df = pd.read_csv('https://github.com/albahnsen/PracticalMachineLearningClass/raw/master/datasets/mashable_texts.csv', index_col=0)

df.head()

"""
Explanation: 17 - Natural Language Processing
by Alejandro Correa Bahnsen and Jesus Solano
version 1.5, March 2019
Part of the class Practical Machine Learning
This notebook is licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License. Special thanks goes to Kevin Markham
What is NLP?

Using computers to process (analyze, understand, generate) natural human languages
Most knowledge created by humans is unstructured text, and we need a way to make sense of it
Build probabilistic model using data about a language

What are some of the higher level task areas?

Information retrieval: Find relevant results and similar results
Google


Information extraction: Structured information from unstructured documents
Events from Gmail


Machine translation: One language to another
Google Translate


Text simplification: Preserve the meaning of text, but simplify the grammar and vocabulary
Rewordify
Simple English Wikipedia


Predictive text input: Faster or easier typing
My application
A much better application


Sentiment analysis: Attitude of speaker
Hater News


Automatic summarization: Extractive or abstractive summarization
autotldr


Natural Language Generation: Generate text from data
How a computer describes a sports match
Publishers withdraw more than 120 gibberish papers


Speech recognition and generation: Speech-to-text, text-to-speech
Google's Web Speech API demo
Vocalware Text-to-Speech demo


Question answering: Determine the intent of the question, match query with knowledge base, evaluate hypotheses
How did supercomputer Watson beat Jeopardy champion Ken Jennings?
IBM's Watson Trivia Challenge
The AI Behind Watson



What are some of the lower level components?

Tokenization: breaking text into tokens (words, sentences, n-grams)
Stopword removal: a/an/the
Stemming and lemmatization: root word
TF-IDF: word importance
Part-of-speech tagging: noun/verb/adjective
Named entity recognition: person/organization/location
Spelling correction: "New Yrok City"
Word sense disambiguation: "buy a mouse"
Segmentation: "New York City subway"
Language detection: "translate this page"
Machine learning

Why is NLP hard?

Ambiguity:
Hospitals are Sued by 7 Foot Doctors
Juvenile Court to Try Shooting Defendant
Local High School Dropouts Cut in Half


Non-standard English: text messages
Idioms: "throw in the towel"
Newly coined words: "retweet"
Tricky entity names: "Where is A Bug's Life playing?"
World knowledge: "Mary and Sue are sisters", "Mary and Sue are mothers"

NLP requires an understanding of the language and the world.
Data
End of explanation
"""


y = df.shares
y.describe()

y = pd.cut(y, [0, 893, 1200, 2275, 63200], labels=[0, 1, 2, 3])

y.value_counts()

df['y'] = y

"""
Explanation: Tokenization

What: Separate text into units such as sentences or words
Why: Gives structure to previously unstructured text
Notes: Relatively easy with English language text, not easy with some languages

Create the target feature (number of shares)
End of explanation
"""


X = df.text

# use CountVectorizer to create document-term matrices from X
vect = CountVectorizer()
X_dtm = vect.fit_transform(X)

temp=X_dtm.todense()

vect.vocabulary_

# rows are documents, columns are terms (aka "tokens" or "features")
X_dtm.shape

# last 50 features
print(vect.get_feature_names()[-150:-100])

# show vectorizer options
vect

"""
Explanation: create document-term matrices
End of explanation
"""


vect = CountVectorizer(lowercase=False)
X_dtm = vect.fit_transform(X)
X_dtm.shape

X_dtm.todense()[0].argmax()

vect.get_feature_names()[8097]

"""
Explanation: lowercase: boolean, True by default
Convert all characters to lowercase before tokenizing.
End of explanation
"""


# include 1-grams and 2-grams
vect = CountVectorizer(ngram_range=(1, 4))
X_dtm = vect.fit_transform(X)
X_dtm.shape

# last 50 features
print(vect.get_feature_names()[-1000:-950])

"""
Explanation: ngram_range: tuple (min_n, max_n)
The lower and upper boundary of the range of n-values for different n-grams to be extracted. All values of n such that min_n <= n <= max_n will be used.
End of explanation
"""


# Default CountVectorizer
vect = CountVectorizer()
X_dtm = vect.fit_transform(X)

# use Naive Bayes to predict the star rating
nb = MultinomialNB()
pd.Series(cross_val_score(nb, X_dtm, y, cv=10)).describe()

# define a function that accepts a vectorizer and calculates the accuracy
def tokenize_test(vect):
    X_dtm = vect.fit_transform(X)
    print('Features: ', X_dtm.shape[1])
    nb = MultinomialNB()
    print(pd.Series(cross_val_score(nb, X_dtm, y, cv=10)).describe())

# include 1-grams and 2-grams
vect = CountVectorizer(ngram_range=(1, 2))
tokenize_test(vect)

"""
Explanation: Predict shares
End of explanation
"""


# remove English stop words
vect = CountVectorizer(stop_words='english')
tokenize_test(vect)

# set of stop words
print(vect.get_stop_words())

"""
Explanation: Stopword Removal

What: Remove common words that will likely appear in any text

Why: They don't tell you much about your text


stop_words: string {'english'}, list, or None (default)

If 'english', a built-in stop word list for English is used.
If a list, that list is assumed to contain stop words, all of which will be removed from the resulting tokens.
If None, no stop words will be used. max_df can be set to a value in the range [0.7, 1.0) to automatically detect and filter stop words based on intra corpus document frequency of terms.
End of explanation
"""


# remove English stop words and only keep 100 features
vect = CountVectorizer(stop_words='english', max_features=100)
tokenize_test(vect)

# all 100 features
print(vect.get_feature_names())

# include 1-grams and 2-grams, and limit the number of features
vect = CountVectorizer(ngram_range=(1, 2), max_features=1000)
tokenize_test(vect)

"""
Explanation: Other CountVectorizer Options

max_features: int or None, default=None
If not None, build a vocabulary that only consider the top max_features ordered by term frequency across the corpus.
End of explanation
"""


# include 1-grams and 2-grams, and only include terms that appear at least 2 times
vect = CountVectorizer(ngram_range=(1, 2), min_df=2)
tokenize_test(vect)

"""
Explanation: min_df: float in range [0.0, 1.0] or int, default=1
When building the vocabulary ignore terms that have a document frequency strictly lower than the given threshold. This value is also called cut-off in the literature. If float, the parameter represents a proportion of documents, integer absolute counts.
End of explanation
"""


# initialize stemmer
stemmer = SnowballStemmer('english')

# words

vect = CountVectorizer()
vect.fit(X)

words = list(vect.vocabulary_.keys())[:100]

# stem each word
print([stemmer.stem(word) for word in words])

"""
Explanation: Stemming and Lemmatization
Stemming:

What: Reduce a word to its base/stem/root form
Why: Often makes sense to treat related words the same way
Notes:
Uses a "simple" and fast rule-based approach
Stemmed words are usually not shown to users (used for analysis/indexing)
Some search engines treat words with the same stem as synonyms
End of explanation
"""


from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()

import nltk
nltk.download('wordnet')

# assume every word is a noun
print([wordnet_lemmatizer.lemmatize(word) for word in words])

# assume every word is a verb
print([wordnet_lemmatizer.lemmatize(word,pos='v') for word in words])

# define a function that accepts text and returns a list of lemmas
def split_into_lemmas(text):
    text = text.lower()
    words = text.split()
    return [wordnet_lemmatizer.lemmatize(word) for word in words]

# use split_into_lemmas as the feature extraction function (WARNING: SLOW!)
vect = CountVectorizer(analyzer=split_into_lemmas)
tokenize_test(vect)

"""
Explanation: Lemmatization

What: Derive the canonical form ('lemma') of a word
Why: Can be better than stemming
Notes: Uses a dictionary-based approach (slower than stemming)
End of explanation
"""


# example documents
simple_train = ['call you tonight', 'Call me a cab', 'please call me... PLEASE!']

# Term Frequency
vect = CountVectorizer()
tf = pd.DataFrame(vect.fit_transform(simple_train).toarray(), columns=vect.get_feature_names())
tf

# Document Frequency
vect = CountVectorizer(binary=True)
df_ = vect.fit_transform(simple_train).toarray().sum(axis=0)
pd.DataFrame(df_.reshape(1, 6), columns=vect.get_feature_names())

# Term Frequency-Inverse Document Frequency (simple version)
tf/df_

# TfidfVectorizer
vect = TfidfVectorizer()
pd.DataFrame(vect.fit_transform(simple_train).toarray(), columns=vect.get_feature_names())

"""
Explanation: Term Frequency-Inverse Document Frequency (TF-IDF)

What: Computes "relative frequency" that a word appears in a document compared to its frequency across all documents
Why: More useful than "term frequency" for identifying "important" words in each document (high frequency in that document, low frequency in other documents)
Notes: Used for search engine scoring, text summarization, document clustering
End of explanation
"""


# create a document-term matrix using TF-IDF
vect = TfidfVectorizer(stop_words='english')
dtm = vect.fit_transform(X)
features = vect.get_feature_names()
dtm.shape

# choose a random text
review_id = 40
review_text = X[review_id]
review_length = len(review_text)

# create a dictionary of words and their TF-IDF scores
word_scores = {}
for word in vect.vocabulary_.keys():
    word = word.lower()
    if word in features:
        word_scores[word] = dtm[review_id, features.index(word)]

# print words with the top 5 TF-IDF scores
print('TOP SCORING WORDS:')
top_scores = sorted(word_scores.items(), key=lambda x: x[1], reverse=True)[:5]
for word, score in top_scores:
    print(word)

# print 5 random words
print('\n' + 'RANDOM WORDS:')
random_words = np.random.choice(list(word_scores.keys()), size=5, replace=False)
for word in random_words:
    print(word)

"""
Explanation: More details: TF-IDF is about what matters
Using TF-IDF to Summarize a text
End of explanation
"""


