with open('book.txt', 'r') as file:
    text = file.readlines()

"""
Explanation: Get the text we want to process
End of explanation
"""


# using a list comprehension to simplify iterating over the the text structure
snippet = " ".join(block.strip() for block in text[175:200])

snippet

# alternative with for-loop
other_snippet = []
for block in text[175:200]:
    other_snippet.append(block.strip())
other_snippet = " ".join(other_snippet)

other_snippet

whole_text = " ".join(block.strip() for block in text)

whole_text[5000:7500]

"""
Explanation: Let's take a smaller chunk from the text:
End of explanation
"""


from nltk.tokenize import sent_tokenize, word_tokenize

str.split?

# you can try to separate sentences by splitting on punctuation
snippet.split('.')

# The sentence tokenizer has some clever tricks to do a better job
sent_tokenize(snippet)

# splitting a text into tokens based on white space
snippet.split()

words = word_tokenize(snippet)

# word tokenize treats punctuation as a token
words

# let's plot the frequency of occurrence of different words
nltk.FreqDist?

fdist = nltk.FreqDist(words)

fdist.plot(30)

"""
Explanation: Tokenizing text

Tokens are meaningful chunks of text:
Sentences
words
punctuation
numbers
?



We'll look at some tools in nltk to help break the raw text into sentences and word tokens.
End of explanation
"""


from nltk.corpus import stopwords

stops = stopwords.words('english')

stops

"""
Explanation: Stop words
Stop words are words that you want to filter out from your text for downstream analysis. They are typically very common words which don't contain much useful information for the task at hand. There is no universal set of stop words and some domain knowledge is helpful for deciding what you want to include when processing your text.
End of explanation
"""


filtered_words = [word.lower() for word in words if word.lower() not in stops]

filtered_words

filtered_fdist = nltk.FreqDist(filtered_words)

filtered_fdist.plot(30)

import string

string.punctuation

stops = stopwords.words('english') + list(string.punctuation)

stops

filtered_words = [word.lower() for word in words if word.lower() not in stops]

filtered_fdist2 = nltk.FreqDist(filtered_words)

filtered_fdist2.plot(30)

"""
Explanation: How would you create a list of tokens that doesn't include the stopwords?
End of explanation
"""


def process_text(text):
    # break text into word tokens
    tokens = word_tokenize(text)
    # remove stopwords
    filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
    # filter for short punctuation
    filtered_words = [w for w in filtered_words if (len(w) > 2)]
    return filtered_words

whole_text[:110]

len(whole_text)

%%time
clean_text = process_text(whole_text)

fdist_whole_text = nltk.FreqDist(clean_text)

fdist_whole_text.plot(25)

"""
Explanation: Make a function that can combine some of our pre-processing tasks to clean up the raw text:
End of explanation
"""


boring_words = ['sir', 'upon', 'said', 'one']
stops += boring_words

%%time
cleaned_text = process_text(whole_text)

fdist_whole_text['holmes']

fdist_whole_text['watson']

"""
Explanation: If some of those words don't seem important, we can add them to 'stops' and clean text again
End of explanation
"""


from nltk.stem import PorterStemmer

help(nltk.stem)

ps = PorterStemmer()

print(ps.stem('Happy'))
print(ps.stem('Happiness'))
print(ps.stem('Had'))

print(ps.stem('Fishing'))
print(ps.stem('Fish'))
print(ps.stem('Fisher'))
print(ps.stem('Fishes'))
print(ps.stem('Fished'))

words = process_text(snippet)

stemmed = [ps.stem(word) for word in words]

for w, stem in zip(words, stemmed):
    print('{} ---> {}'.format(w, stem))

"""
Explanation: Stemming
End of explanation
"""


def stem_process(text):
    # tokenize
    tokens = word_tokenize(text)
    # remove stops
    filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
    filtered_words = [w for w in filtered_words if (len(w) > 2)]
    # stem
    stemmed_words = [ps.stem(w) for w in filtered_words]
    return stemmed_words

%%time
stemmed = stem_process(whole_text)

stemmed

fdist_stems = nltk.FreqDist(stemmed)

fdist_stems.plot(30)

"""
Explanation: Let's make another function that stems the word tokens during the processing stage
End of explanation
"""


from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

lemmatizer.lemmatize?

print(lemmatizer.lemmatize('having'))
print(lemmatizer.lemmatize('have'))
print(lemmatizer.lemmatize('had'))
print()
print(lemmatizer.lemmatize('fishing'))
print(lemmatizer.lemmatize('fish'))
print(lemmatizer.lemmatize('fisher'))
print(lemmatizer.lemmatize('fishes'))
print(lemmatizer.lemmatize('fished'))
print()
print(lemmatizer.lemmatize('am'))
print(lemmatizer.lemmatize('is'))
print(lemmatizer.lemmatize('was'))

# including POS for the lemmatizer can improve its output
print(lemmatizer.lemmatize('having', pos='v'))
print(lemmatizer.lemmatize('have', pos='v'))
print(lemmatizer.lemmatize('had', pos='v'))
print()
print(lemmatizer.lemmatize('fishing', pos='v'))
print(lemmatizer.lemmatize('fish', pos='v'))
print(lemmatizer.lemmatize('fisher', pos='n'))
print(lemmatizer.lemmatize('fishes', pos='v'))
print(lemmatizer.lemmatize('fished', pos='v'))
print()
print(lemmatizer.lemmatize('am', pos='v'))
print(lemmatizer.lemmatize('is', pos='v'))
print(lemmatizer.lemmatize('was', pos='v'))

lemmatized = [lemmatizer.lemmatize(word) for word in words]

for w, lemma in zip(words, lemmatized):
    print('{} ---> {}'.format(w, lemma))

lemmatizer.lemmatize('running', pos='v')

def lemma_process(text):
    # tokenize
    tokens = word_tokenize(text)
    # remove stops
    filtered_words = [token.lower() for token in tokens if not token.lower() in stops]
    filtered_words = [w for w in filtered_words if (len(w) > 2)]
    # lemmatize
    lemmatized_words = [lemmatizer.lemmatize(w) for w in filtered_words]
    return lemmatized_words

%%time
lemma_text = lemma_process(whole_text)

lemma_fdist = nltk.FreqDist(lemma_text)

lemma_fdist.plot(30)

"""
Explanation: Lemmatizing
In English, for example, run, runs, ran and running are forms of the same lexeme, with run as the lemma. Lexeme, in this context, refers to the set of all the forms that have the same meaning, and lemma refers to the particular form that is chosen by convention to represent the lexeme.
End of explanation
"""


nltk.help.upenn_tagset()

snippet

nltk.pos_tag(word_tokenize(sent_tokenize(snippet)[1]))

def process_POS(text):
    sentences = sent_tokenize(text)
    tagged_words = []
    for sentence in sentences:
        words = word_tokenize(sentence)
        tagged = nltk.pos_tag(words)
        tagged_words.append(tagged)
    return tagged_words

tagged_sentences = process_POS(snippet)

tagged_sentences

sentences =[]
for sentence in tagged_sentences[:5]:
    print(sentence)
    lemmas = []
    for word, pos in sentence:
        if pos == 'VBP':
            lemmas.append(lemmatizer.lemmatize(word, 'v'))
        elif pos in ['NN', 'NNS']:
            lemmas.append(lemmatizer.lemmatize(word, 'n'))
        else:
            lemmas.append(lemmatizer.lemmatize(word))
    sentences.append(lemmas)

"""
Explanation: Part-of-speech (POS) tagging
The process of marking up a word in a text as corresponding to a particular part of speech, based on both its definition and its context.
POS tagging is tricky because some words can have more than one POS depending on the context.
"Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo."
End of explanation
"""


from nltk import ngrams
from collections import Counter

bigrams = Counter(ngrams(word_tokenize(whole_text), 2))

for phrase, freq in bigrams.most_common(30):
    print("{}\t{}".format(phrase, freq))

trigrams = Counter(ngrams(word_tokenize(whole_text), 3))

for phrase, freq in trigrams.most_common(30):
    print("{}\t{}".format(phrase, freq))

"""
Explanation: ngrams
End of explanation
"""


stemmed = stem_process(whole_text)

stemmed_bigrams = Counter(ngrams(stemmed, 2))

stemmed_bigrams.most_common(20)

stemmed_trigrams = Counter(ngrams(stemmed, 3))

stemmed_trigrams.most_common(20)

"""
Explanation: Stopwords and punctuation will have an effect on ngrams!
End of explanation
"""


from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
data_corpus = ["John likes to watch movies. Mary likes movies too.", 
"John also likes to watch football games."]
X = vectorizer.fit_transform(data_corpus) 
print(X.toarray())
print(vectorizer.get_feature_names())

from nltk.sentiment.vader import SentimentIntensityAnalyzer

vader = SentimentIntensityAnalyzer()

text = "I dont hate movies!"

vader.polarity_scores(text)

"""
Explanation: Bag of words (BOW) text representation for machine learning
End of explanation
"""


