sentence = """At eight o'clock on Thursday morning
... Arthur didn't feel very good."""

tokens = nltk.word_tokenize(sentence)

tokens

tagged = nltk.pos_tag(tokens)

tagged

"""
Explanation: NTLK Documentation
Understanding how NTLK works
End of explanation
"""


df = pd.read_csv('allPostText_test.csv')

df.info()

def vec(name):
    tokens = nltk.word_tokenize(name)
    tagged = nltk.pos_tag(tokens)
    return tagged

df['Text'].apply(vec).head(10)

df['Tags'] = df['Text'].apply(vec)

"""
Explanation: Look at this list of tags (https://pythonprogramming.net/natural-language-toolkit-nltk-part-speech-tagging)
So lets try to use this on out Daphne data
End of explanation
"""


def token(tags):
    mini_list = []
    for elem in tags:
        if elem[1] == 'NNP':
            mini_list.append(elem[0])
    return mini_list

df['People list'] = df['Tags'].apply(token)

df['People list'].head(10)

"""
Explanation: Only consider the ones, with NNP as second item
End of explanation
"""


# This looks promising

"""
Explanation: Not great, so lets look around for a better solution.
End of explanation
"""


for sent in nltk.sent_tokenize(df['Text'][1]):
    for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
        if hasattr(chunk, 'label'):
            print(chunk.label()+',', ' '.join(c[0] for c in chunk))

def peopled(elem):
    mini_list = []
    for sent in nltk.sent_tokenize(elem):
        for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
            if hasattr(chunk, 'label'):
                p = chunk.label(), ' '.join(c[0] for c in chunk)
                mini_list.append(p)
    return mini_list

df['people'] = df['Text'].apply(peopled)

"""
Explanation: sudo python -m nltk.downloader all if you have problems
Code stolen from Stackoverflow
End of explanation
"""


lst = list(df['people'])

lst = [x for x in lst if x !=[]]

flat_list = [item for sublist in lst for item in sublist]

name_list = []
for name in flat_list:
    if name[0] == 'PERSON':
        name_list.append(name[1])

pd.DataFrame(name_list)[0].value_counts()

"""
Explanation: Only getting the peolple
End of explanation
"""


