from __future__ import division, unicode_literals
from ml.similarity import pre_process, TfidfCluster
from textblob import TextBlob as tb
import nltk, re, pprint
import nltk.chunk
from nltk.corpus import twitter_samples
from nltk import tag
from nltk.corpus import wordnet
from nltk.corpus.reader.wordnet import POS_LIST
import math
from stemming.porter2 import stem
from nltk import ngrams as _ngrams
import json
from nltk.corpus import stopwords

from linguistic import pattern


class TestNLTK(object):
    def setup(self):
        pass

    def teardown(self):
        pass

    def test_named_entity_extraction(self):
        data_problem = [
            """
                If you are navigating through the list of open tabs inside the All+Tabs panel and you wanna filter by a term you have to select the search text field first.
                It would be nice if any entered character is automatically routed to the search field and the filter gets applied.
            """,
            """
                In maximized mode there is something like 3 pixels padding on the right side of "All tabs" panel.
                It doesn't exist on the left side of panel and in not maximized mode.
            """,
            """
                When you have the All+Tabs panel open it would be great if you can press Cmd/Ctrl+F to focus the search text field. Right now the panel gets hidden and the Find toolbar is shown without focus.
                IMO using the command inside the All+Tabs panel would make more sense.
            """,
            """
                Steps to reproduce:
                Nothing... had multiple windows and tiles open... for about 4 hours

                Actual results:
                Crashed without warning
            """,
            """
                Firefox crashes at leat 6 times a day.  Installed latest version but still crashing.  Goes very slow before it crashes.
            """,
            """
                Steps to reproduce:
                W have installed Firefox 18 (as we did with all previous version) on Solaris 10 SPAC 64b

                Actual results:
                When we tried to start it form a console, it crashed with a message: Segmentation fault.
                And it produced a core dump

                Expected results:
                Firefox should have open correctly
            """,
            """
                screen shots: Aurora 7.0a2 top, FF 3.6.18 bottom

                User Agent: Mozilla/5.0 (X11; Linux i686 on x86_64; rv:7.0a2) Gecko/20110709 Firefox/7.0a2
                Build ID: 20110709042004

                Steps to reproduce:

                Start Aurora with a new Profile just to be sure
                Open Menu Preferences
                Select  Advanced | Encryption , View Certificates
                Select tab certificates, scroll down to Thawte
                Select first Thawte Certificate, click "view..." button

                (Actually the problem is not specific to this single certificate. I tried many more built-in root certificates and they all had the same issue.)



                Actual results:

                Certificate Viewer opens, it says "could not verify this certificate for unknown reasons"


                Expected results:

                Certificate Viewer opens, it should say "this certificates has been verified for the following uses" and a list of uses.
            """,
            """
                I agree with the utility of this feature it's just going to take some serious work. This is a pretty hacky area of the code that is screaming for a rewrite.

                I'll slate this for 1.1
            """
        ]

        data_analyses = [
            """"All Tabs" panel code removed in bug 670684""",
            """"All Tabs" panel code removed in bug 670684""",
            """"All Tabs" panel code removed in bug 670684""",
            """
                Please provide the crash ID from about:crashes.
                Are you able to reproduce it consistently?
            """,
            """Please post the related Report IDs from about:crashes.""",
            """
                Do you see crash report IDs in about:crashes?
            """,
            """
                I agree with the utility of this feature it's just going to take some serious work. This is a pretty hacky area of the code that is screaming for a rewrite.
                I'll slate this for 1.1
            """,
            """
                This is caused by bug 479393:
                We are expecting that CERT_VerifyCertificateNow will return SECSuccess if at least one of the given usages is valid. However, it actually returns SECFailure unless all of the given usages are valid. I remember that Kai's original patch was correctly ignoring the return value of CERT_VerifyCertificateNow, but he "corrected" it when I told him we should check the return value. My bad. :(

                The fix is simple: restore the error detection logic to the way it was done before:
            """
        ]

        # problems = data_problem[3:6]
        # analyses = data_analyses[3:6]
        problems = data_problem[0:3]
        analyses = data_analyses[0:3]
        lp = pattern.LinguisticPattern(problems, analyses)

        lp.named_entities(labels=['NP'])
        result = lp.find_ngrams_patterns()

        for p in result:
            print p


# from nltk.util import ngrams
# from nltk.corpus import gutenberg
#
# gut_ngrams = (ngram for sent in gutenberg.sents() for ngram in
#               ngrams(sent, 3, pad_left=True, pad_right=True, right_pad_symbol='EOS', left_pad_symbol="BOS"))
# freq_dist = nltk.FreqDist(gut_ngrams)
# kneser_ney = nltk.KneserNeyProbDist(freq_dist)
#
# prob_sum = 0
# for i in kneser_ney.samples():
#     if i[0] == "I" and i[1] == "confess":
#         prob_sum += kneser_ney.prob(i)
#         print "{0}:{1}".format(i, kneser_ney.prob(i))
# print prob_sum



# def find_ngram_to(n, entity, problem):
#
#     ngrams = _ngrams(problem.lower().split(), n)
#     result = [gram for gram in ngrams if entity in gram]
#     return result
#
#
#
#
# def find_ngrams(entity, problems):
#     result = {}
#     for x in xrange(5, 6):
#         result[x] = {}
#         for i, problem in enumerate(problems):
#             key = "p" + str(i + 1)
#             result[x][key] = find_ngram_to(x, entity, problem)
#         # break
#
#     return result
#
#
#
#
#
#
#
#
# def tf(word, blob):
#     return blob.words.count(word) / len(blob.words)
#
#
# def n_containing(word, bloblist):
#     return sum(1 for blob in bloblist if word in blob.words)
#
#
# def idf(word, bloblist):
#     y = (1 + n_containing(word, bloblist))
#     x = math.log(len(bloblist) / y)
#     return x
#
#
# def tfidf(word, blob, bloblist):
#     _tf = tf(word, blob)
#     _idf = idf(word, bloblist)
#
#     return _tf * _idf
#
#
# def tag_override(sent):
#     force_tags = {'crashes': 'VBZ'}
#     tagged_words = nltk.pos_tag(sent)
#     new_tagged_words = [(word, force_tags.get(word, tag)) for word, tag in tagged_words]
#     return new_tagged_words
#
#
# def get_pos(tag):
#     if tag:
#         s = str(tag).lower()[0]
#         if s in POS_LIST:
#             return s
#
#     return None
#
#
# def transform(tree):
#     return (tree.leaves(), tree.label())
#
#
# def disambiguity(word, tag):
#     words = wordnet.synsets(word, get_pos(tag))
#     if words:
#         value = words[0]
#         return value._lemma_names[0]
#     else:
#         return word

# clusters = [
#     {
#         "id": 1,
#         "problems": data_problem[0:3],
#         "analyses": data_analyses[0:3]
#         },
#     {
#         "id": 2,
#         "problems": data_problem[3:6],
#         "analyses": data_analyses[3:6]
#         },
#     {
#         "id": 3,
#         "problems": data_problem[6:7],
#         "analyses": data_analyses[6:7]
#         },
#     {
#         "id": 4,
#         "problems": data_problem[7:8],
#         "analyses": data_analyses[7:8]
#     },
# ]