import warnings
warnings.filterwarnings('ignore')
from pathlib import Path
import pickle
import re
from tqdm import tqdm
tqdm.pandas()
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
pd.options.display.max_columns = None
pd.options.display.max_colwidth = None
import nltk
from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.manifold import TSNE
import chart_studio.plotly as py
import plotly.graph_objects as go
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics import silhouette_score
from sklearn.decomposition import NMF, LatentDirichletAllocation
from sklearn.datasets import fetch_20newsgroups
from sklearn.pipeline import Pipeline
from sklearn import metrics
vectorizer = TfidfVectorizer()
from sklearn import decomposition
from sklearn.cluster import KMeans
from sklearn.decomposition import TruncatedSVD
from scipy.sparse import random as sparse_random
from collections import Counter
import random
import skfuzzy as fuzz
from fcmeans import FCM
import spacy
from spacy import displacy
nlp = spacy.load('en_core_web_sm')
from spacy.lang.en.stop_words import STOP_WORDS
stopwords = list(STOP_WORDS)
import functools

class nlp_pipeline():
    """
    the input dataframe is by default appid index
    df is the MERGED dataframe
    df_labels is the output of self.kmeans_cluster and saved with name 'predicted_kmeans_labels
    03/08/21
        After eyeballing self.eyeball_app_changed_niche_label(), I see that they do not ever change over time, the description.
        So I decide from now on to combine all panel's description together.

    03/09/21
        I found that those appids are with exactly the same text information in each panel (after deleting None and impute missing).
        So I conclude that the apps that changed from niche to broad type is completely due to the algorithm only.
        It is pointless to conduct panel specific text label prediction.
        From this particular method, I prove that apps' text labels are time INVARIANT.
        Now I should find a suitable panel regression model that incorporate time-invariant independent variables.

    04/02/21
        1.  I have deleted combined_panel, because I have checked that descriptions in every panel is the same,
            and it is time-invariant, so niche label will be generated by combining all panels.
        2.  I will delete consecutive_panel = True, and uses all panels' text columns in generating niche label,
            this is because I will be using difference-in-difference for pre and after covid, therefore I need to use all columns.
            Even if you are not using diff-in-diff, since niche label is time-invariant, so using all panels will not be any different
            from using only consecutive panels.
        3.  I have deleted functions that assign niche dummy or niche scale dummy, and descriptive stats functions
            relates to niche labels because they are moved to regression_analysis class
        4.  I will divide full sample into game and nongame, and do the NLP algorithm within each subsample.
            Note that game and nongame are time-invariant.
            Accordingly, I will delete the function that generate genreIdGame in regression_analysis class, so
            this dummy is only generated once in the entire code.
    """
    nlp_graph_path = Path(
        '/home/naixin/Insync/naixin88@sina.cn/OneDrive/__CODING__/PycharmProjects/GOOGLE_PLAY/___essay_1___/nlp/graphs')
    panel_path = Path(
        '/home/naixin/Insync/naixin88@sina.cn/OneDrive/_____GWU_ECON_PHD_____/___Dissertation___/____WEB_SCRAPER____/__PANELS__/___essay_1_panels___')
    tokenizer = nlp.Defaults.create_tokenizer(nlp)

    # after examining the optimal svd cluster graphs, I write dict here as class attributes for each dataset
    # ------------------------------------------------------------------------
    optimal_svd_components_201907 = {
        'full':
                       {'full': 1500},
        'minInstalls':
                       {'ImputedminInstalls_tier1': 1000,
                        'ImputedminInstalls_tier2': 1300,
                        'ImputedminInstalls_tier3': 1300},
        'genreId':
                       {'ART_AND_DESIGN': 150,
                        'COMICS': 50,
                        'PERSONALIZATION': 150,
                        'PHOTOGRAPHY': 150,
                        'AUTO_AND_VEHICLES': 150,
                        'GAME_ROLE_PLAYING': 200,
                        'GAME_ACTION': 50,
                        'GAME_RACING': 150,
                        'TRAVEL_AND_LOCAL': 150,
                        'GAME_ADVENTURE': 150,
                        'SOCIAL': 150,
                        'GAME_SIMULATION': 200,
                        'LIFESTYLE': 150,
                        'EDUCATION': 200,
                        'BEAUTY': 50,
                        'GAME_CASUAL': 200,
                        'BOOKS_AND_REFERENCE': 150,
                        'BUSINESS': 200,
                        'FINANCE': 200,
                        'GAME_STRATEGY': 150,
                        'SPORTS': 150,
                        'COMMUNICATION': 150,
                        'DATING': 50,
                        'ENTERTAINMENT': 150,
                        'GAME_BOARD': 150,
                        'EVENTS': 50,
                        'SHOPPING': 150,
                        'FOOD_AND_DRINK': 150,
                        'HEALTH_AND_FITNESS': 200,
                        'HOUSE_AND_HOME': 100,
                        'TOOLS': 150,
                        'LIBRARIES_AND_DEMO': 150,
                        'MAPS_AND_NAVIGATION': 150,
                        'MEDICAL': 200,
                        'MUSIC_AND_AUDIO': 150,
                        'NEWS_AND_MAGAZINES': 150,
                        'PARENTING': 150,
                        'GAME_PUZZLE': 250,
                        'VIDEO_PLAYERS': 150,
                        'PRODUCTIVITY': 150,
                        'WEATHER': 150,
                        'GAME_ARCADE': 150,
                        'GAME_CASINO': 50,
                        'GAME_CARD': 150,
                        'GAME_EDUCATIONAL': 150,
                        'GAME_MUSIC': 50,
                        'GAME_SPORTS': 150,
                        'GAME_TRIVIA': 100,
                        'GAME_WORD': 150},
        'developer':
                       {'top': 316,
                        'non-top': 1200}}
    # after examining the optimal km cluster graphs (both elbow and sihouette graphs),
    # I write dict here as class attributes for each dataset
    # ------------------------------------------------------------------------
    optimal_km_clusters_201907 = {
        'full':
            {'full': 250},
        'minInstalls':
            {'ImputedminInstalls_tier1': 240,
             'ImputedminInstalls_tier2': 300,
             'ImputedminInstalls_tier3': 80},
        'genreId':
            {'ART_AND_DESIGN': 8,
             'COMICS': 12,
             'PERSONALIZATION': 25,
             'PHOTOGRAPHY': 25,
             'AUTO_AND_VEHICLES': 30,
             'GAME_ROLE_PLAYING': 30,
             'GAME_ACTION': 4,
             'GAME_RACING': 30,
             'TRAVEL_AND_LOCAL': 30,
             'GAME_ADVENTURE': 40,
             'SOCIAL': 25,
             'GAME_SIMULATION': 35,
             'LIFESTYLE': 35,
             'EDUCATION': 35,
             'BEAUTY': 12,
             'GAME_CASUAL': 30,
             'BOOKS_AND_REFERENCE': 20,
             'BUSINESS': 40,
             'FINANCE': 35,
             'GAME_STRATEGY': 35,
             'SPORTS': 35,
             'COMMUNICATION': 35,
             'DATING': 16,
             'ENTERTAINMENT': 40,
             'GAME_BOARD': 15,
             'EVENTS': 10,
             'SHOPPING': 16,
             'FOOD_AND_DRINK': 20,
             'HEALTH_AND_FITNESS': 30,
             'HOUSE_AND_HOME': 14,
             'TOOLS': 30,
             'LIBRARIES_AND_DEMO': 13,
             'MAPS_AND_NAVIGATION': 30,
             'MEDICAL': 35,
             'MUSIC_AND_AUDIO': 30,
             'NEWS_AND_MAGAZINES': 20,
             'PARENTING': 10,
             'GAME_PUZZLE': 35,
             'VIDEO_PLAYERS': 30,
             'PRODUCTIVITY': 35,
             'WEATHER': 10,
             'GAME_ARCADE': 35,
             'GAME_CASINO': 4,
             'GAME_CARD': 17,
             'GAME_EDUCATIONAL': 40,
             'GAME_MUSIC': 17,
             'GAME_SPORTS': 29,
             'GAME_TRIVIA': 17,
             'GAME_WORD': 7},
        'developer':
            {'top': 60,
             'non-top': 200}}

    def __init__(self,
                 tcn,
                 initial_panel,
                 all_panels,
                 df=None,
                 sub_sample_text_cols=None,
                 tf_idf_matrices=None,
                 svd_matrices=None,
                 output_labels=None):
        self.tcn = tcn
        self.initial_panel = initial_panel
        self.all_panels = all_panels
        self.df = df
        self.ss_text_cols = sub_sample_text_cols
        self.tf_idf_matrices = tf_idf_matrices
        self.svd_matrices = svd_matrices
        self.output_labels = output_labels

    def open_divided_df(self):
        """
        The reason to use imputed missing dataframe instead of imputed and deleted missing is that everytime you delete different number of rows
        (depends on the definition of missig), but not every time you need to re-run text clustering (because it is time-consuming),
        so I will just use the FULL converted data and you could merged this predicted labels to imputed and deleted missing in combine_dataframes class.
        """
        f_name = self.initial_panel + '_imputed_deleted_subsamples.pickle'
        q = nlp_pipeline.panel_path / f_name
        with open(q, 'rb') as f:
            self.df = pickle.load(f)
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def generate_save_input_text_col(self):
        """
        Purpose of creating this cell is to avoid creating run_subsample switch in each function below.
        Everytime you run the full sample NLP, you run the sub samples NLP simultaneously, it would take longer, but anyways.
        """
        # --------------- full sample text cols ------------------------------------------------------
        full_sample_text_col = self.df[self.tcn + 'ModeClean'].copy(deep=True)
        # --------------- minInstalls text cols ------------------------------------------------------
        minInstall_tier1_text_col = self.df.loc[
            self.df['ImputedminInstalls_tier1'] == 1, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        minInstall_tier2_text_col = self.df.loc[
            self.df['ImputedminInstalls_tier2'] == 1, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        minInstall_tier3_text_col = self.df.loc[
            self.df['ImputedminInstalls_tier3'] == 1, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        # --------------- category text cols ---------------------------------------------------------
        app_categories = self.df['ImputedgenreId_Mode'].unique().tolist()
        app_cat_text_cols = dict.fromkeys(app_categories)
        for i in app_categories:
            app_cat_text_cols[i] = self.df.loc[self.df[i] == 1, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        # --------------- star developer text cols ---------------------------------------------------
        top_digital_firms_text_col = self.df.loc[
            self.df['top_digital_firms'] == 1, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        non_top_digital_firms_text_col = self.df.loc[
            self.df['top_digital_firms'] == 0, [self.tcn + 'ModeClean']].squeeze().copy(deep=True)
        # --------------- compile text cols ----------------------------------------------------------
        self.ss_text_cols = {'full': {'full': full_sample_text_col},
                             'minInstalls': {'ImputedminInstalls_tier1': minInstall_tier1_text_col,
                                             'ImputedminInstalls_tier2': minInstall_tier2_text_col,
                                             'ImputedminInstalls_tier3': minInstall_tier3_text_col},
                             'genreId': app_cat_text_cols,
                             'developer': {'top': top_digital_firms_text_col,
                                           'non-top': non_top_digital_firms_text_col}}
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def tf_idf_transformation(self):
        pipe = Pipeline(steps=[('tfidf',
                                TfidfVectorizer(
                                    stop_words='english',
                                    strip_accents='unicode',
                                    max_features=2000))])
        matrix_df_dict = dict.fromkeys(self.ss_text_cols.keys())
        for sample, content in matrix_df_dict.items():
            matrix_df_dict[sample] = dict.fromkeys(self.ss_text_cols[sample].keys())
        for sample, content in self.ss_text_cols.items():
            for ss_name, col in content.items():
                print('TF-IDF TRANSFORMATION')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                matrix = pipe.fit_transform(col)
                matrix_df = pd.DataFrame(matrix.toarray(),
                                         columns=pipe['tfidf'].get_feature_names())
                matrix_df['app_ids'] = col.index.tolist()
                matrix_df.set_index('app_ids', inplace=True)
                matrix_df_dict[sample][ss_name] = matrix_df
                print(matrix_df.shape)
        self.tf_idf_matrices = matrix_df_dict
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def find_optimal_svd_component_plot(self):
        """
        https://medium.com/swlh/truncated-singular-value-decomposition-svd-using-amazon-food-reviews-891d97af5d8d
        https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.TruncatedSVD.html
        """
        for sample, content in self.tf_idf_matrices.items():
            for ss_name, matrix in content.items():
                print('FIND OPTIMAL SVD COMPONENTS')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                n_comp = np.round(np.linspace(0, matrix.shape[1]-1, 20))
                n_comp = n_comp.astype(int)
                explained = []
                for x in tqdm(n_comp):
                    svd = TruncatedSVD(n_components=x)
                    svd.fit(matrix)
                    explained.append(svd.explained_variance_ratio_.sum())
                    print("Number of components = %r and explained variance = %r" % (x, svd.explained_variance_ratio_.sum()))
                fig, ax = plt.subplots()
                ax.plot(n_comp, explained)
                ax.grid()
                plt.xlabel('Number of components')
                plt.ylabel("Explained Variance")
                plt.title(self.initial_panel + sample + ss_name + " Plot of Number of components v/s explained variance")
                filename = self.initial_panel + '_' + sample + '_' + ss_name + '_optimal_svd_graph.png'
                fig.savefig(nlp_pipeline.nlp_graph_path / 'optimal_svd_comp' / filename, facecolor='white', dpi=300)
                plt.show()
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def truncate_svd(self, random_state):
        matrix_df_dict = dict.fromkeys(self.ss_text_cols.keys())
        for sample, content in matrix_df_dict.items():
            matrix_df_dict[sample] = dict.fromkeys(self.ss_text_cols[sample].keys())
        for sample, content in self.tf_idf_matrices.items():
            for ss_name, matrix in content.items():
                print('TRUNCATE SVD')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                svd = TruncatedSVD(n_components=nlp_pipeline.optimal_svd_components_201907[sample][ss_name],
                                   random_state=random_state)
                matrix_transformed = svd.fit_transform(matrix)
                print(matrix_transformed.shape)
                matrix_transformed_df = pd.DataFrame(matrix_transformed)  # do not need to assign column names because those do not correspond to each topic words (they have been transformed)
                matrix_transformed_df['app_ids'] = matrix.index.tolist()
                matrix_transformed_df.set_index('app_ids', inplace=True)
                matrix_df_dict[sample][ss_name] = matrix_transformed_df
        self.svd_matrices = matrix_df_dict
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def find_optimal_cluster_elbow(self):
        """
        https://blog.cambridgespark.com/how-to-determine-the-optimal-number-of-clusters-for-k-means-clustering-14f27070048f
        https://scikit-learn.org/stable/modules/clustering.html
        """
        for sample, content in self.svd_matrices.items():
            for ss_name, matrix in content.items():
                print('ELBOW METHOD TO FIND OPTIMAL KM CLUSTERS')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                # note here, for svd comps, the maximum components are the total number of features (columns)
                # here, the total number of clusters are the total number of apps (at extreme, 1 app per cluster)
                # starts from 1 because this is only within cluster sum of squared distances
                # the maximum number of clusters is controlled below 1/5 of total number of points because first it would
                # take extremely long time to compute without controlling for maximum number of clusters, second,
                # it is reasonable to assume that one would not want only 4 points in a cluster.
                n_cluster_list = np.round(np.linspace(1, matrix.shape[0] - 0.8 * matrix.shape[0], 10))
                n_cluster_list = n_cluster_list.astype(int)
                sum_of_squared_distances = []
                for k in tqdm(n_cluster_list):
                    km = KMeans(n_clusters=k)
                    km = km.fit(matrix)
                    sum_of_squared_distances.append(km.inertia_)
                fig, ax = plt.subplots()
                ax.plot(n_cluster_list, sum_of_squared_distances, 'bx-')
                ax.grid()
                plt.xlabel('k')
                plt.ylabel('Sum_of_squared_distances')
                plt.title(self.initial_panel + sample + ss_name + ' Elbow Method For Optimal k')
                filename = self.initial_panel + '_' + sample + '_' + ss_name + '_elbow_optimal_cluster.png'
                fig.savefig(nlp_pipeline.nlp_graph_path / 'optimal_clusters' / filename, facecolor='white', dpi=300)
                plt.show()
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def find_optimal_cluster_silhouette(self):
        """
        https://medium.com/analytics-vidhya/how-to-determine-the-optimal-k-for-k-means-708505d204eb
        https://medium.com/@kunal_gohrani/different-types-of-distance-metrics-used-in-machine-learning-e9928c5e26c7
        """
        for sample, content in self.svd_matrices.items():
            for ss_name, matrix in content.items():
                print('SILHOUETTE SCORE TO FIND OPTIMAL KM CLUSTERS')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                # starting from 2 because this score need to calculate between cluster estimators
                n_cluster_list = np.round(np.linspace(2, matrix.shape[0] - 0.8 * matrix.shape[0], 10))
                n_cluster_list = n_cluster_list.astype(int)
                silhouette_scores = []
                for k in tqdm(n_cluster_list):
                    km = KMeans(n_clusters=k)
                    km = km.fit(matrix)
                    labels = km.labels_
                    silhouette_scores.append(silhouette_score(matrix, labels, metric='cosine'))
                fig, ax = plt.subplots()
                ax.plot(n_cluster_list, silhouette_scores, 'bx-')
                ax.grid()
                plt.xlabel('k')
                plt.ylabel('silhouette_scores (cosine distance)')
                plt.title(self.initial_panel + sample + ss_name + ' Silhouette Scores For Optimal k')
                filename = self.initial_panel + '_' + sample + '_' + ss_name + '_silhouette_optimal_cluster.png'
                fig.savefig(nlp_pipeline.nlp_graph_path / 'optimal_clusters' / filename, facecolor='white', dpi=300)
                plt.show()
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)

    def kmeans_cluster(self,
                       init,
                       random_state):
        label_dict = dict.fromkeys(self.ss_text_cols.keys())
        for sample, content in label_dict.items():
            label_dict[sample] = dict.fromkeys(self.ss_text_cols[sample].keys())
        for sample, content in self.svd_matrices.items():
            for ss_name, matrix in content.items():
                print('KMEANS CLUSTER')
                print(self.initial_panel, ' -- ', sample, ' -- ', ss_name)
                kmeans = KMeans(n_clusters=nlp_pipeline.optimal_km_clusters_201907[sample][ss_name],
                                init=init,
                                random_state=random_state)
                y_kmeans = kmeans.fit_predict(matrix)  # put matrix_transformed_df here would generate same result as put matrix_transformed
                matrix[sample + '_' + ss_name + '_kmeans_labels'] = y_kmeans
                label_single = matrix[[sample + '_' + ss_name + '_kmeans_labels']]
                label_dict[sample][ss_name] = label_single
        self.output_labels = label_dict
        # --------------------------- save -------------------------------------------------
        # for this one, you do not need to run text cluster label every month when you scraped new data, because they would more or less stay the same
        filename = self.initial_panel + '_predicted_labels_dict.pickle'
        q = nlp_pipeline.panel_path / 'predicted_text_labels' / filename
        pickle.dump(self.output_labels, open(q, 'wb'))
        return nlp_pipeline(
                 tcn=self.tcn,
                 initial_panel=self.initial_panel,
                 all_panels=self.all_panels,
                 df=self.df,
                 sub_sample_text_cols=self.ss_text_cols,
                 tf_idf_matrices=self.tf_idf_matrices,
                 svd_matrices=self.svd_matrices,
                 output_labels=self.output_labels)
