import nltk
from nltk.corpus import names
import random

name = [(n,'M') for n in names.words('male.txt')] + [(n,'F') for n in names.words('female.txt')]
random.shuffle(name)
name[:10]

"""
Explanation: Ch6 Learning to Classify Text
本章的目標是回答幾個問題:

如何辨識能用來分類的顯著特徵?
如何建立language model來自動完成文字處理作業?
從這些model中可以學到什麼?

Supervised Classification
Classification將輸入標上正確的class label。Supervised是指預先得到正確標記的training data。
例如我們想根據姓氏的結尾判斷是男或女：
End of explanation
"""


def gender_feature(name): return {'last_letter': name[-1]}
featuresets = [(gender_feature(n), g) for (n,g) in name]
train, test = featuresets[500:], featuresets[:500]
classifier = nltk.NaiveBayesClassifier.train(train)

classifier.classify({'last_letter': 'a'})

nltk.classify.accuracy(classifier, test)

classifier.show_most_informative_features()

"""
Explanation: 準備一個function，用來產生feature，這裡的feature是用最後的英文字母。
End of explanation
"""


def gender_feature(name): return {'last_letter': name[-1], 'first_letter': name[0]}
featuresets = [(gender_feature(n), g) for (n,g) in name]
train, test = featuresets[500:], featuresets[:500]
classifier = nltk.NaiveBayesClassifier.train(train)

nltk.classify.accuracy(classifier, test)

def gender_feature(name): return {'last_letter': name[-1], 'first_letter': name[0], 'len': len(name)}
featuresets = [(gender_feature(n), g) for (n,g) in name]
train, test = featuresets[500:], featuresets[:500]
classifier = nltk.NaiveBayesClassifier.train(train)

nltk.classify.accuracy(classifier, test)

classifier.show_most_informative_features()

"""
Explanation: 嘗試不同的feature，例如加入第一個字母，或加入姓名的長度。
End of explanation
"""


from nltk.corpus import movie_reviews

movie_reviews.categories()

movie_reviews.fileids('neg')[:5], movie_reviews.fileids('pos')[:5]

documents = [(list(movie_reviews.words(f)), c)
             for c in movie_reviews.categories() for f in movie_reviews.fileids(c)]
random.shuffle(documents)

# 選擇2000個字作為feature
all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())
word_features = [w for (w, c) in all_words.most_common()[:2000]]
word_features[:5]

def document_features(doc):
    document_words = set(doc)  # 自動排除重複字
    features = {w: w in document_words for w in word_features}
    return features

tmp = document_features(['the', 'she'])
[key for key in tmp if tmp[key]]

featuresets = [(document_features(d), c) for (d, c) in documents]
train, test = featuresets[100:], featuresets[:100]
classifier = nltk.NaiveBayesClassifier.train(train)
nltk.classify.accuracy(classifier, test)

classifier.show_most_informative_features()

"""
Explanation: Document Classification
movie_reviews裡面已經分好正面和負面兩種評語，我們的目標是根據字彙，預測評語是正面或負面。
End of explanation
"""


import nltk
from nltk.corpus import brown
fdist = nltk.FreqDist()
fdist.update([w[-1:] for w in brown.words()])
fdist.update([w[-2:] for w in brown.words()])
fdist.update([w[-3:] for w in brown.words()])

common_suf = [k for (k, c) in fdist.most_common()[:100]]
common_suf[:5]

def pos_features(word):
    features = {suffix:word.lower().endswith(suffix) for suffix in common_suf}
    return features

featuresets = [(pos_features(n), pos) for (n, pos) in brown.tagged_words(categories='news')]
size = int(len(featuresets) * 0.1)
train, test = featuresets[size:], featuresets[:size]
size

classifier = nltk.DecisionTreeClassifier.train(train)
nltk.classify.accuracy(classifier, test)

# DecisionTree可以列出結構
print classifier.pretty_format(depth=10)

"""
Explanation: Part-of-Speech Tagging
在Chapter 5有介紹過regular expression tagger，當時是用手動設定規則，這裡嘗試用classifier找出規則。
End of explanation
"""


def pos_features(sentence, i):
    features = {"suf_1": sentence[i][-1:], "suf_2": sentence[i][-2:], "suf_3": sentence[i][-3:]}
    if i == 0:
        features['prev'] = '*'
    else:
        features['prev'] = sentence[i-1]
    return features

pos_features(brown.sents()[0], 8)

brown.sents()[0][7:10]

tagged_sents = brown.tagged_sents(categories='news')
featuresets = []
for tagged_sent in tagged_sents:
    untagged_sent = nltk.tag.untag(tagged_sent)
    for i, (word, tag) in enumerate(tagged_sent):
        featuresets.append( (pos_features(untagged_sent, i), tag) )
size = int(len(featuresets) * 0.1)
train_set, test_set = featuresets[size:], featuresets[:size]
classifier = nltk.NaiveBayesClassifier.train(train_set)
nltk.classify.accuracy(classifier, test_set)

classifier.show_most_informative_features()

"""
Explanation: Exploiting Context
Context是指一個單字的前後文，例如單字fly可以作為名詞或動詞，如果前面是a或the，表示是名詞。
End of explanation
"""


def pos_features(sentence, i, history):
    features = {"suf_1": sentence[i][-1:], "suf_2": sentence[i][-2:], "suf_3": sentence[i][-3:]}
    if i == 0:
        features['prev-word'] = '*'
        features['prev-tag'] = '*'
    else:
        features['prev-word'] = sentence[i-1]
        features['prev-tag'] = history[i-1]
    return features

# 自己定義自己的tagger class
class ConsecutivePosTagger(nltk.TaggerI):
    def __init__(self, train_sents):
        train_set = []
        for tagged_sent in train_sents:
            untagged_sent = nltk.tag.untag(tagged_sent)
            history = []
            # 利用enumerate產生 (index, value)
            for i, (word, tag) in enumerate(tagged_sent):
                featureset = pos_features(untagged_sent, i, history)
                train_set.append( (featureset, tag) )
                history.append(tag)
        self.classifier = nltk.NaiveBayesClassifier.train(train_set)
 
    def tag(self, sentence):
        history = []
        for i, word in enumerate(sentence):
            featureset = pos_features(sentence, i, history)
            tag = self.classifier.classify(featureset)
            history.append(tag)
        return zip(sentence, history)

tagged_sents = brown.tagged_sents(categories='news')
size = int(len(tagged_sents) * 0.1)
train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]
tagger = ConsecutivePosTagger(train_sents)
tagger.evaluate(test_sents)

"""
Explanation: Sequence Classification
可以更進一步，將單字預測的詞性，用來預測下一個單字的詞性。
End of explanation
"""


import random
from nltk.corpus import brown
tagged_sents = list(brown.tagged_sents(categories='news'))
random.shuffle(tagged_sents)
# 用10%的資料作為test set
size = int(len(tagged_sents) * 0.1)
train_set, test_set = tagged_sents[size:], tagged_sents[:size]

"""
Explanation: Evaluation
產生test set的方法: 利用random.shuffle
End of explanation
"""


