"""
Initialization
"""

'''
Standard modules
'''
import os
import collections
from pprint import pprint

'''
Analysis modules
'''
import pandas as pd
import nltk


'''
Custom modules
'''
import config
import utilities

'''
Misc
'''
nb_name = '20171002-daheng-select_daily_popular_topics'

news_period_title_docs_pkl = os.path.join(config.TMP_DIR, '{}-{}'.format(nb_name, 'news-period-title_docs.sr.pkl'))
news_title_docs_high_freq_words_df_pkl = os.path.join(config.TMP_DIR, '{}-{}'.format(nb_name, 'news-title_docs-high_freq_words.df.pkl'))

"""
Explanation: Select out daily popular topics
Objective: For each day, select out daily popular topic by analyzing high frequency terms in news titles of that day.
Last modified: 2017-10-15
Roadmap

Build news title docs for each day
Find out high frequency words in each news title doc
Write out to csv files for manually inspection
Filter out news related to manually selected topics and re-produce csv files

Steps
End of explanation
"""


if 1 == 1:
    '''
    Load in pickle for news data over selected period.
    '''
    news_period_df = pd.read_pickle(config.NEWS_PERIOD_DF_PKL)

"""
Print any single news title
"""
news_period_df.loc[3, 'news_title']

"""
Print complete news titles
"""
with pd.option_context('display.max_colwidth', 100):
    display(news_period_df[['news_collected_time', 'news_title']])

"""
Group news by day of news_collected_time and concatenate news_titles
"""
news_titles_sr = news_period_df.resample('D', on='news_collected_time')['news_title'].apply(lambda x: '\n'.join(x))

"""
Print any single news title doc
"""
print(news_titles_sr.iloc[0])
# news_titles_sr.loc['2015-01-01']

"""
Print all news title docs
"""
with pd.option_context('display.max_colwidth', 130):
    print(news_titles_sr)

"""
Make tmp sr pickle
"""
if 0 == 1:
    news_titles_sr.to_pickle(news_period_title_docs_pkl)

"""
Explanation: Build news title docs for each day
End of explanation
"""


"""
Load tmp sr pickle for news title docs
"""
if 1 == 1:
    news_titles_sr = pd.read_pickle(news_period_title_docs_pkl)

test_str = news_titles_sr.iloc[0]

print(test_str)

"""
Check stop-words
"""
stop_words = nltk.corpus.stopwords.words('english')
print(stop_words)
print('Count: {}'.format(len(stop_words)))

"""
Pre-processing steps
"""
if 1 == 1:
    tokens = nltk.word_tokenize(test_str)
    
    '''
    Regonize NE in tokens and preserve
    
    Note: performance of NE recognition depends on the pre-trained model provided in nltk package
    '''
    chunks = nltk.ne_chunk(nltk.pos_tag(tokens))
    # Each element of chunks is either a (word, pos) tuple or a Tree() containing the parts of the chunk
    tokens = [chunk[0] if isinstance(chunk, tuple) else ' '.join(node[0] for node in chunk) for chunk in chunks]
    
    '''
    Remove non-alphabetical tokens
    '''
    # tokens = [token for token in tokens if token.isalpha()]
    
    '''
    Remove stop-words
    '''
    tokens = [token for token in tokens if token.lower() not in stop_words]
    
    '''
    Remove misc single punctuation tokens
    '''
    misc_punc_lst = [",", ":", "'", "'s", ";", ".", "?", "(", ")", "..."]
    tokens = [token for token in tokens if token not in misc_punc_lst]
    
    print(tokens)

"""
Count token frequency and print
"""
if 1 == 1:
    token_counter = collections.Counter(tokens)
    print(str(token_counter.most_common(50)))

%%time
"""
Warp up previous pre-process steps in function and apply to all news title docs.
Put resluts in df.
"""
def count_high_freq_words(news_title_doc):
    """
    param new_title_doc: a string of news title doc
    return: a string of high frequency words in doc
    """
    tokens = nltk.word_tokenize(news_title_doc)
    
    chunks = nltk.ne_chunk(nltk.pos_tag(tokens))
    # Each element of chunks is either a (word, pos) tuple or a Tree() containing the parts of the chunk
    tokens = [chunk[0] if isinstance(chunk, tuple) else ' '.join(node[0] for node in chunk) for chunk in chunks]
    
    stop_words = nltk.corpus.stopwords.words('english')
    tokens = [token for token in tokens if token.lower() not in stop_words]
    
    misc_punc_lst = [",", ":", "'", "'s", ";", ".", "?", "(", ")", "..."]
    tokens = [token for token in tokens if token not in misc_punc_lst]
    
    token_counter = collections.Counter(tokens)
    return str(token_counter.most_common(50))

if 0 == 1:
    results_dict = {}
    
    for ind_val, sr_val in news_titles_sr.iteritems():
        results_dict[ind_val] = count_high_freq_words(sr_val)
        
    high_freq_words_sr = pd.Series(results_dict)

with pd.option_context('display.max_rows', 150, 'display.max_colwidth', 130):
    print(high_freq_words_sr)

"""
Make tmp df pickle
"""
if 0 == 1:
    news_title_docs_high_freq_words_df = pd.concat([news_titles_sr, high_freq_words_sr], axis=1)
    news_title_docs_high_freq_words_df.columns = ['news_title_doc', 'high_freq_words']

    news_title_docs_high_freq_words_df.to_pickle(news_title_docs_high_freq_words_df_pkl)

"""
Check results
"""
news_title_docs_high_freq_words_df = pd.read_pickle(news_title_docs_high_freq_words_df_pkl)
with pd.option_context('display.max_colwidth', 100):
    display(news_title_docs_high_freq_words_df)

with pd.option_context('display.max_colwidth', 130):
    print(news_title_docs_high_freq_words_df['high_freq_words'])

"""
Explanation: Find out high frequency word in each news title doc
End of explanation
"""


"""
Write out all news titles to csv file
"""
if 0 == 1:
    news_titles_csv_file = os.path.join(config.HR_DIR, 'news_titles.csv')
    news_period_df = pd.read_pickle(config.NEWS_PERIOD_DF_PKL)
    
    news_period_df.to_csv(path_or_buf=news_titles_csv_file, columns=['news_collected_time', 'news_title'], sep='\t', header=True, index=True)

"""
Write out news title high freq words to csv file
"""
if 0 == 1:
    news_title_high_freq_words_csv_file = os.path.join(config.HR_DIR, 'news_title_high_freq_words.csv')
    news_title_docs_high_freq_words_df = pd.read_pickle(news_title_docs_high_freq_words_df_pkl)
    
    news_title_docs_high_freq_words_df.to_csv(path_or_buf=news_title_high_freq_words_csv_file, columns=['high_freq_words'], sep='\t', header=True, index=True)

"""
Explanation: Write out to csv files for manually inspection
End of explanation
"""


%%time
"""
Iteration of removing news related to manually selected topics and compling new selected topics list.
"""
iter_num = 'v2'

filtered_news_titles_csv_file = os.path.join(config.HR_DIR, 'news_titles.{}.csv'.format(iter_num))
filtered_news_title_high_freq_words_csv_file = os.path.join(config.HR_DIR, 'news_title_high_freq_words.{}.csv'.format(iter_num))

if 0 == 1:
    '''
    Filter out news related to topics have been manually selected out 
    '''
    filtered_news_dct_lst = []
    
    news_period_df = pd.read_pickle(config.NEWS_PERIOD_DF_PKL)
    
    count = 0
    for row_ind, row in news_period_df.iterrows():
        news_dict = {'news_collected_time': row['news_collected_time'], 'news_title': row['news_title']}
        match_topic = False

        for topic_ind, topic in enumerate(config.MANUALLY_SELECTED_TOPICS_LST):
            if utilities.news_title_match(row['news_title'], topic['keywords_lst'], verbose=False):
                match_topic = True
                count += 1
                break
        
        if not match_topic:
            filtered_news_dct_lst.append(news_dict)
    
    print('{} out of {} news titles filtered out.'.format(count, news_period_df['news_id'].count()))
    filtered_news_df = pd.DataFrame(filtered_news_dct_lst)
    
    '''
    Group news by day of news_collected_time and concatenate news_titles
    '''
    news_titles_sr = filtered_news_df.resample('D', on='news_collected_time')['news_title'].apply(lambda x: '\n'.join(x))
    
    '''
    Re-counted high freq words file
    '''
    results_dict = {}
    
    for ind_val, sr_val in news_titles_sr.iteritems():
        results_dict[ind_val] = count_high_freq_words(sr_val)
        
    high_freq_words_sr = pd.Series(results_dict)
    
    news_title_docs_high_freq_words_df = pd.concat([news_titles_sr, high_freq_words_sr], axis=1)
    news_title_docs_high_freq_words_df.columns = ['news_title_doc', 'high_freq_words']
    
    """
    Write out to csv files
    """
    filtered_news_df.to_csv(path_or_buf=filtered_news_titles_csv_file, columns=['news_collected_time', 'news_title'], sep='\t', header=True, index=True)
    news_title_docs_high_freq_words_df.to_csv(path_or_buf=filtered_news_title_high_freq_words_csv_file, columns=['high_freq_words'], sep='\t', header=True, index=True)

"""
Explanation: Filter out news related to manually selected topics and re-produce csv files
End of explanation
"""


