%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

import nltk
from nltk import word_tokenize

text = word_tokenize("And now for something completely different")
nltk.pos_tag(text)

text = word_tokenize("They refuse to permit us to obtain the refuse permit")
nltk.pos_tag(text)

"""
Explanation: Chapter 5: Categorizing and Tagging Words
Notes from NLTK Book http://www.nltk.org/book/
The goal of this chapter is to answer the following questions:
* What are lexical categories and how are they used in natural language processing?
* What is a good Python data structure for storing words and their categories?
* How can we automatically tag each word of a text with its word class?
Using a Tagger
part-of-speech (POS) tagger
End of explanation
"""


nltk.help.upenn_tagset()

"""
Explanation: CC: coordinating conjunction
RB: adverb
IN: preposition
NN: noun
JJ: adjective
VBP: present tense verb

more details below
End of explanation
"""


text = nltk.Text(word.lower() for word in nltk.corpus.brown.words())
text.similar('woman')

"""
Explanation: text.similar(w) finds words that appear in similar contexts to w
from book: The text.similar() method takes a word w, finds all contexts w1w w2, then finds all words w' that appear in the same context, i.e. w1w'w2
words similar to 'woman' -- mostly nouns
End of explanation
"""


text.similar('bought')

"""
Explanation: words similar to 'bought' -- mostly verbs
End of explanation
"""


text.similar('over')

text.similar('the')

"""
Explanation: words similar to 'over' -- mostly prepositions
End of explanation
"""


tagged_token = nltk.tag.str2tuple('fly/NN')
tagged_token

sent = '''
The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/IN
other/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CC
Fulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPS
said/VBD ``/`` ARE/BER well/QL operated/VBN and/CC follow/VB generally/RB
accepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJT
interest/NN of/IN both/ABX governments/NNS ''/'' ./.
'''
[nltk.tag.str2tuple(t) for t in sent.split()]

"""
Explanation: Tagged Corpora
Representing Tagged Tokens
typically represented as a tuple: (token, tag)
Can be generated using str2tuple...
End of explanation
"""


nltk.corpus.brown.tagged_words()

nltk.corpus.brown.tagged_words(tagset='universal')

print(nltk.corpus.nps_chat.tagged_words())

nltk.corpus.conll2000.tagged_words()

nltk.corpus.treebank.tagged_words()

"""
Explanation: Reading Tagged Corpora
The/at Fulton/np-tl County/nn-tl Grand/jj-tl Jury/nn-tl said/vbd Friday/nr an/at investigation/nn of/in Atlanta's/np$ recent/jj primary/nn election/nn produced/vbd / no/at evidence/nn ''/'' that/cs any/dti irregularities/nns took/vbd place/nn ./.

if a corpus contains tagged text, the corpus interface will have a tagged_words() method
End of explanation
"""


nltk.corpus.brown.tagged_words(tagset='universal')

nltk.corpus.treebank.tagged_words(tagset='universal')

"""
Explanation: not all corpora have the same tag sets, so can force mapping to universal tagset
End of explanation
"""


nltk.corpus.sinica_treebank.tagged_words()

nltk.corpus.indian.tagged_words()

nltk.corpus.mac_morpho.tagged_words()

nltk.corpus.conll2002.tagged_words()

nltk.corpus.cess_cat.tagged_words()

"""
Explanation: Tagged corpora in other languages...
End of explanation
"""


brn = nltk.corpus.brown
brown_news_tagged = brn.tagged_words(categories='news', tagset='universal')
tag_fd = nltk.FreqDist(tag for (word, tag) in brown_news_tagged)
tag_fd.most_common()

plot = plt.figure(figsize=(18,10))
tag_fd.plot(cumulative=True)

tag_fd.plot()

"""
Explanation: A Universal Part-of-Speech tagset
Common POS tags
End of explanation
"""


word_tag_pairs = nltk.bigrams(brown_news_tagged)
list(word_tag_pairs)[:20]

word_tag_pairs = nltk.bigrams(brown_news_tagged) # generator needs to be redefined
noun_preceders = [a[1] for a, b in word_tag_pairs if b[1] == 'NOUN']
noun_preceders[:20]

fdist = nltk.FreqDist(noun_preceders)
[tag for tag, _ in fdist.most_common()]

fdist.plot(cumulative=True)

"""
Explanation: Nouns
what parts of speech appear before nouns?
End of explanation
"""


wsj = nltk.corpus.treebank.tagged_words(tagset='universal')
word_tag_fd = nltk.FreqDist(wsj)
[wt[0] for wt, _ in word_tag_fd.most_common() if wt[1] == 'VERB'][:25]

"""
Explanation: Verbs
what are the most common verbs in news text?
End of explanation
"""


cfd1 = nltk.ConditionalFreqDist(wsj)
cfd1['yield'].most_common()

cfd1['cut'].most_common()

"""
Explanation: words and tags are paired. treat the word as a condition and the tag as an event and create a cfd
lists the most common tags, given a word
End of explanation
"""


wsh = nltk.corpus.treebank.tagged_words()
cfd2 = nltk.ConditionalFreqDist((tag, word) for (word, tag) in wsj)
list(cfd2['ADJ'])[:25]

"""
Explanation: reverse the order... conditions: tags, events: words 
see the likely words, given a tag
End of explanation
"""


brown_learned_text = brn.words(categories='learned')
sorted(set(b for a, b in nltk.bigrams(brown_learned_text) if a == 'often'))

"""
Explanation: Adjectives and Adverbs
skipping
Unsimplified Tags
skipping
Exploring Tagged Corpora
Find words that follow 'often'
End of explanation
"""


brown_lrnd_tagged = brn.tagged_words(categories='learned', tagset='universal')
brown_lrnd_tagged

tags = [b[1] for a, b in nltk.bigrams(brown_lrnd_tagged) if a[0] == 'often']
tags[:5]

fd = nltk.FreqDist(tags)
fd.tabulate()

"""
Explanation: instead, use tagged_words() to look at POS tags
End of explanation
"""


def process(sentence):
    for (w1, t1), (w2, t2), (w3, t3) in nltk.trigrams(sentence):
        if (t1.startswith('V') and t2 == 'TO' and t3.startswith('V')):
            print(w1, w2, w3)
            
for tagged_sent in brn.tagged_sents()[:200]:
    process(tagged_sent)

"""
Explanation: POS that follow 'often' the most are verbs**

Find words that are parts of sequences of tags+words (e.g., Verb, to, Verb)
End of explanation
"""


brown_news_tagged = brn.tagged_words(categories='news', tagset='universal')
data = nltk.ConditionalFreqDist((word.lower(), tag)
                                for word, tag in brown_news_tagged)
for word in sorted(data.conditions()):
    if len(data[word]) > 3:
        tags = [tag for tag, _ in data[word].most_common()]
        print(word, ' '.join(tags))

"""
Explanation: Identifying words that have ambiguous POS tags
End of explanation
"""


from nltk.corpus import brown
brown_tagged_sents = brown.tagged_sents(categories='news')
brown_sents = brown.sents(categories='news')
brown_sents[:1]

"""
Explanation: Mapping Words to Properties Using Python Dictionaries
skipping this section on python dictionary usage
Automatic Tagging
Using tagged sentences instead of words
End of explanation
"""


raw = 'I do not like green eggs and ham, I do not like them Sam I am!'
tokens = word_tokenize(raw)
default_tagger = nltk.DefaultTagger('NN')
default_tagger.tag(tokens)

"""
Explanation: Example: create a default tagger that tags everything as NN
default taggers assign a tag to every single word
End of explanation
"""


default_tagger.evaluate(brown_tagged_sents)

"""
Explanation: this "tagger" performs poorly, of course
End of explanation
"""


patterns = [
    (r'.*ing$', 'VBG'),               # gerunds
    (r'.*ed$', 'VBD'),                # simple past
    (r'.*es$', 'VBZ'),                # 3rd singular present
    (r'.*ould$', 'MD'),               # modals
    (r'.*\'s$', 'NN$'),               # possessive nouns
    (r'.*s$', 'NNS'),                 # plural nouns
    (r'^-?[0-9]+(.[0-9]+)?$', 'CD'),  # cardinal numbers
    (r'.*', 'NN')                     # nouns (default)
]

brown_sents[3]

regexp_tagger = nltk.RegexpTagger(patterns)
regexp_tagger.tag(brown_sents[3])

regexp_tagger.evaluate(brown_tagged_sents)

"""
Explanation: Default taggers are still useful because after processing several thousand words of English text, most new words will be nouns

The Regular Expression Tagger
parts of speech expressed as regular expressions
End of explanation
"""


fd = nltk.FreqDist(brown.words(categories='news'))
cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories='news'))
most_freq_words = fd.most_common(100)
likely_tags = dict((word, cfd[word].max()) for word, _ in most_freq_words)
baseline_tagger = nltk.UnigramTagger(model=likely_tags)
baseline_tagger.evaluate(brown_tagged_sents)

"""
Explanation: The Lookup Tagger
find 100 most frequent words and store their most-likely tag; evaluate its performance against previously tagged data brown_tagged_sents
End of explanation
"""


sent = brown.sents(categories='news')[3]
sent

baseline_tagger.tag(sent)

"""
Explanation: ~46% of tags were correct, just based on the 100 most frequent words
use this "lookup tagger" to tag untagged text
End of explanation
"""


baseline_tagger = nltk.UnigramTagger(model=likely_tags, backoff=nltk.DefaultTagger('NN'))
baseline_tagger.evaluate(brown_tagged_sents)

"""
Explanation: lots of None tags. These are words that were not in the 100 most frequent words.
Instead, use backoff -- if a word is in the lookup tagger table, use that tag. Otherwise, use the default tag (noun / NN).
End of explanation
"""


def performance(cfd, wordlist):
    
    ''' return evaluated performance value (0 to 1) for input cfd, wordlist
    '''
    
    lt = dict((word, cfd[word].max()) for word in wordlist)
    baseline_tagger = nltk.UnigramTagger(model=lt, backoff=nltk.DefaultTagger('NN'))
    return baseline_tagger.evaluate(brown.tagged_sents(categories='news'))
    
word_freqs = nltk.FreqDist(brown.words(categories='news')).most_common()
words_by_freq = [w for w, _ in word_freqs]
words_by_freq[:25]

cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories='news'))
sizes = 2 ** np.arange(15)
sizes

perfs = [performance(cfd, words_by_freq[:size]) for size in sizes]
perfs

plt.figure(figsize=(12,8))
plt.plot(sizes, perfs, marker='o')
plt.title('Lookup Tagger Performance with Varying Model Size')
plt.xlabel('Model Size')
plt.ylabel('Performance')
plt.show()

"""
Explanation: better performance... 58% vs 46% without backoff!
a program to create and evaluate lookup taggers having a range of sizes:
End of explanation
"""


from nltk.corpus import brown
brown_tagged_sents = brown.tagged_sents(categories='news')
brown_sents = brown.sents(categories='news')
unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)
unigram_tagger.tag(brown_sents[2007])

"""
Explanation: N-Gram Tagging
Unigram Tagging
A unigram tagger behaves just like a lookup tagger, except there is a more convenient technique for setting it up, called training
Training a UnigramTagger by specifying tagged sentence data as a parameter
End of explanation
"""


split_size = int(len(brown_tagged_sents) * 0.9)
split_size

train_sents = brown_tagged_sents[:split_size]
test_sents = brown_tagged_sents[split_size:]
unigram_tagger = nltk.UnigramTagger(train_sents)
unigram_tagger.evaluate(test_sents)

"""
Explanation: Separating Training and Testing Data -- train/test split
End of explanation
"""


bigram_tagger = nltk.BigramTagger(train_sents)
bigram_tagger.tag(brown_sents[2007])

unseen_sent = brown_sents[4203]
bigram_tagger.tag(unseen_sent)

"""
Explanation: General N-Gram Tagging -- NgramTagger class
(using BigramTagger here)
End of explanation
"""


bigram_tagger.evaluate(test_sents)

"""
Explanation: Lots of None tags! The BigramTagger never saw a lot of word pairs during training
Accuracy score will be very poor:
End of explanation
"""


t0 = nltk.DefaultTagger('NN')
t1 = nltk.UnigramTagger(train_sents, backoff=t0)
t2 = nltk.BigramTagger(train_sents, backoff=t1)
t2.evaluate(test_sents)

t3 = nltk.TrigramTagger(train_sents, backoff=t2)
t3.evaluate(test_sents)

"""
Explanation: as N-Gram 'N' increases, data sparsity increases -- tradeoff between precision and recall
Combining Taggers
implementing multiple backoffs
Example from book:
1. try Bigram tagger
2. if None, try Unigram tagger
3. if None, use default tagger
End of explanation
"""


cfd = nltk.ConditionalFreqDist(((x[1], y[1], z[0]), z[1])
                                for sent in brown_tagged_sents
                                for x, y, z in nltk.trigrams(sent))
ambiguous_contexts = [c for c in cfd.conditions() if len(cfd[c]) > 1]
ambiguous_contexts[:10]

sum(cfd[c].N() for c in ambiguous_contexts) / cfd.N()

"""
Explanation: Tagging Unknown Words
dealing with out-of-vocabulary items
Limit vocabulary to most frequent n words; replace every other word with UNK
Storing Taggers
use pickle module to serialize/store taggers to save time after training
Performance Limitations
How many ambiguous parts-of-speech does a tagger encounter?
Consider Trigram tagger:
End of explanation
"""


test_tags = [tag for sent in brown.sents(categories='editorial')
                 for word, tag in t2.tag(sent)]
gold_tags = [tag for word, tag in brown.tagged_words(categories='editorial')]
nltk.ConfusionMatrix(gold_tags, test_tags)

print(nltk.ConfusionMatrix(gold_tags, test_tags))

"""
Explanation: ~5% of trigrams are ambiguous
Confusion Matrix -- charting gold standard tags against tagger-assigned tags
End of explanation
"""


