# (C) Copyright IBM Corp. 2019, 2020.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from nltk.corpus import stopwords  
#####################################################################

##############################################################
import re 
import pandas as pd

#######################################################################
from sklearn.feature_extraction.text import CountVectorizer 
 

def order_ngram(ngram): 
    words = ngram.split()
    # sort the list.
    words.sort()
    return " ".join(words)

##############################################################
# returns clean string
def clean_text(text, custom_stop_words = []):
    
    REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
    BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_:]') 
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
    text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
    
    STOPWORDS = set(stopwords.words('english'))
    STOPWORDS = STOPWORDS | set(custom_stop_words)
    
    text = ' '.join(word for word in text.split() if word not in STOPWORDS) 
    
    return text
 
def get_frequent_words_bigrams(utterances, num_unigrams,num_bigrams,custom_stop_words):
    
    utterances = [clean_text(x,custom_stop_words) for x in utterances]
    
    if len(utterances)==0:
        print("Warning! List of utterances is empty")
        return {"name": "","children": []}
    
    unigrams_found = False
    bigrams_found = False
    
    for i in range(0, len(utterances)):
        current_length=len(utterances[i].split())
        if current_length>1:
            unigrams_found = True
            bigrams_found = True
            break
        elif current_length>0:
            unigrams_found = True
    
    if not unigrams_found:
        print("Warning! List of utterances is empty. Perhaps the documents only contain stop words")
        return {"name": "","children": []}
    
    cv_words = CountVectorizer(ngram_range=(1,1))
    
    cv_fit_words=cv_words.fit_transform(utterances)     
    word_list = cv_words.get_feature_names();    
    word_count_list  = cv_fit_words.toarray().sum(axis=0)    
    words_sorted = sorted(zip(word_count_list, word_list),reverse=True)
    words_sorted=words_sorted[0:min(len(words_sorted), num_unigrams)]
    
    if bigrams_found:
        cv_bigrams = CountVectorizer(ngram_range=(2,2))
        
        cv_fit_bigrams=cv_bigrams.fit_transform(utterances)     
        bigram_list = cv_bigrams.get_feature_names()
        
        num_overall_bigrams = len(bigram_list)
            
        bigram_count_list  = cv_fit_bigrams.toarray().sum(axis=0)    
        bigram_zip = sorted(zip(bigram_count_list, bigram_list),reverse=True)
        
        counter_bigrams=0
        counter_merged_bigrams=0
        bigram_sorted=[]
        bigram_final_list=[]
        
        # merge same order
        while counter_bigrams < num_overall_bigrams and counter_merged_bigrams < num_bigrams:
            current_bigram = bigram_zip[counter_bigrams][1] 
            current_count = bigram_zip[counter_bigrams][0] 
        
            S = order_ngram(current_bigram)
            if S in bigram_sorted:
                ind = bigram_sorted.index(S) 
                bigram_final_list[ind][0] += current_count
                 
            else:
                bigram_final_list.append([current_count, current_bigram])
                bigram_sorted.append(S)

                counter_merged_bigrams += 1
                
            counter_bigrams += 1    
        
    else:
        print("Warning! List of bigrams is empty.")

    children = []
    for i in range(0, min(len(words_sorted), num_unigrams)):
        children.append({"name": words_sorted[i][1], "value": int(words_sorted[i][0])})
    if bigrams_found:
        for i in range(0, min(len(bigram_final_list), num_bigrams)):
            children.append({"name": bigram_final_list[i][1], "value": int(bigram_final_list[i][0])})
    data = {"name": "","children": children}

    return data
 
##########################################################################################################
def get_df_frequencies(neg_dict, pos_dict, neg_list, pos_list, table_title1, table_title2):
    neg_list_neg_freq=[]
    neg_list_pos_freq=[]
    pos_list_neg_freq=[]
    pos_list_pos_freq=[]
     
    num_neg_features = len(neg_list)
    if num_neg_features==0:
        print("Warning! No significant keywords were found.")
        
    num_pos_features = len(pos_list)
    
    for i in range(0, num_neg_features):
        neg_list_neg_freq.append(neg_dict[neg_list[i]][0])              
        neg_list_pos_freq.append(neg_dict[neg_list[i]][2]) 
        
    for i in range(0, num_pos_features):        
        pos_list_pos_freq.append(pos_dict[pos_list[i]][0])              
        pos_list_neg_freq.append(pos_dict[pos_list[i]][2])
                
    if num_neg_features>num_pos_features:
        D = [None]*(num_neg_features-num_pos_features)
        pos_list.extend(D)
        pos_list_pos_freq.extend(D)
        pos_list_neg_freq.extend(D)
    
    if num_pos_features>num_neg_features:
        D = [None]*(num_pos_features-num_neg_features)
        neg_list.extend(D)
        neg_list_neg_freq.extend(D)
        neg_list_pos_freq.extend(D)
    
    df_frequencies = pd.DataFrame() 
    df_frequencies[table_title1] = neg_list 
    df_frequencies["Failure keywords: negative corpus frequencies, %"] = neg_list_neg_freq
    df_frequencies["Failure keywords: positive corpus frequencies, %"] = neg_list_pos_freq
    df_frequencies[table_title2] = pos_list
    df_frequencies["Success keywords: positive corpus frequencies, %"] = pos_list_pos_freq
    df_frequencies["Success keywords: negative corpus frequencies, %"] = pos_list_neg_freq

    return df_frequencies    


########################################################################
def get_features(text_negative, text_positive, num_features, n_range=(1,2)):    
    
    PRINT_MODE = False
    
    #no_shows=["account","number"]
    no_shows=[]
    
    ratio_parameter=2.0
    num_negative=len(text_negative)
    num_positive=len(text_positive)
     
    cv_negative = CountVectorizer(ngram_range=n_range)
    
    cv_fit=cv_negative.fit_transform(text_negative)     
    word_list_negative = cv_negative.get_feature_names();    
    count_list_negative = cv_fit.toarray().sum(axis=0)    
    negative_count_unigrams = sorted(zip(count_list_negative, word_list_negative),reverse=True)  
    dict_negative = {word_list_negative[i]: count_list_negative[i] for i in range(len(word_list_negative))} 
 
    cv_positive = CountVectorizer(ngram_range=n_range)
             
    cv_fit=cv_positive.fit_transform(text_positive)     
    word_list_positive = cv_positive.get_feature_names();    
    count_list_positive = cv_fit.toarray().sum(axis=0)    
    positive_count_unigrams = sorted(zip(count_list_positive, word_list_positive),reverse=True)  
    dict_positive = {word_list_positive[i]: count_list_positive[i] for i in range(len(word_list_positive))} 
    
    ##############################
    # perform frequence check, print warnings
    counter_neg_features=0
    counter=0
    neg_list_sorted=[]
    neg_list=[]
    neg_dict={}
    while counter_neg_features<num_features:
        current_neg_feature=negative_count_unigrams[counter][1] 
        current_neg_count=negative_count_unigrams[counter][0]
            
        get_neg_freq=current_neg_count / num_negative 
        
        if current_neg_feature in dict_positive.keys():
            current_pos_count = dict_positive[current_neg_feature]
            get_pos_freq = (current_pos_count + 0.0) / num_positive 
        else:
            current_pos_count = 0
            get_pos_freq = 1.0 / num_positive 
         
        if ratio_parameter*get_pos_freq>=get_neg_freq:
            if PRINT_MODE:
                print("Frequencies for negative feature "+current_neg_feature+" are not appropriate. Feature is omitted")
                print("Negative frequency: "+str(get_neg_freq))
                print("Positive frequency: "+str(get_pos_freq))
                
        elif current_neg_feature not in no_shows:
            S = order_ngram(current_neg_feature)
            if S in neg_list_sorted:
                ind = neg_list_sorted.index(S)
                prev = neg_list[ind] 
                neg_dict[prev][0] += current_neg_count
                neg_dict[prev][1] += current_pos_count
                neg_dict[prev][2] += ((current_neg_count + neg_dict[prev][0]) / max(current_pos_count + neg_dict[prev][1],1)) *\
                    (num_positive/num_negative)
            else:
                neg_list.append(current_neg_feature)
                neg_list_sorted.append(S)

                neg_dict[current_neg_feature] = [current_neg_count, current_pos_count, get_neg_freq / get_pos_freq]
                counter_neg_features+=1
        
        counter+=1 
        
        if counter==len(negative_count_unigrams):
            break
        
    counter_pos_features=0
    counter=0
    pos_list_sorted=[]
    pos_list=[]
    pos_dict={}
    while counter_pos_features<num_features:
        current_pos_feature = positive_count_unigrams[counter][1] 
        current_pos_count = positive_count_unigrams[counter][0]
        
        get_pos_freq=current_pos_count / num_positive
        
        if current_pos_feature in dict_negative.keys():
            current_neg_count = dict_negative[current_pos_feature]
            get_neg_freq = dict_negative[current_pos_feature]
        else:
            current_neg_count = 0
            get_neg_freq = 1.0 / num_negative         
        
        if get_pos_freq<=ratio_parameter*get_neg_freq:
            if PRINT_MODE:
                print("Frequencies for positive feature '"+current_pos_feature+"' are not appropriate. Feature is omitted.")
                print("Negative frequency: "+str(get_neg_freq))
                print("Positive frequency: "+str(get_pos_freq))
        elif current_pos_feature not in no_shows:
            S = order_ngram(current_pos_feature)
            if S in pos_list_sorted:
                ind = pos_list_sorted.index(S)
                prev = pos_list[ind] 
                pos_dict[prev][0] += current_pos_count
                pos_dict[prev][1] += current_neg_count
                pos_dict[prev][2] += ((current_pos_count + pos_dict[prev][0]) / max(current_neg_count + pos_dict[prev][1],1)) *\
                    (num_negative/num_positive )
            else:
                pos_list.append(current_pos_feature)
                pos_list_sorted.append(S)
                
                pos_dict[current_pos_feature] = [current_pos_count, current_neg_count, get_pos_freq / get_neg_freq]
                counter_pos_features+=1
        
        counter+=1    
        
        if counter==len(positive_count_unigrams):
            break  
    
    return neg_dict, pos_dict, neg_list, pos_list 



########################################################################
def keyword_table_detailed(text_negative, text_positive, num_keywords, custom_stop_words, title1="Title 1", title2="Title 2"):
    
    COEFF = [0.4, 0.4, 0.2]
    
    text_positive = [clean_text(x,custom_stop_words) for x in text_positive]
    text_negative = [clean_text(x,custom_stop_words) for x in text_negative]
    
    if len(text_positive)==0 or len(text_negative)==0:
        print("Warning! At least one of List of utterances is empty")
        neg_dict = {}
        pos_dict = {}
        neg_list = [] 
        pos_list = []
        
    else:        
        
        unigrams_found_negative = False
        bigrams_found_negative = False
        trigrams_found_negative = False
        
        for i in range(0, len(text_negative)):
            current_length=len(text_negative[i].split())
            if current_length>2:
                unigrams_found_negative = True
                bigrams_found_negative = True
                trigrams_found_negative = True
                break
            elif current_length>1:
                unigrams_found_negative = True
                bigrams_found_negative = True
            elif current_length>0:
                unigrams_found_negative = True
        
        ################################################################        
        unigrams_found_positive = False
        bigrams_found_positive = False
        trigrams_found_positive = False
        
        for i in range(0, len(text_positive)):
            current_length=len(text_positive[i].split())
            if current_length>2:
                unigrams_found_positive = True
                bigrams_found_positive = True
                trigrams_found_positive = True
                break
            elif current_length>1:
                unigrams_found_positive = True
                bigrams_found_positive = True
            elif current_length>0:
                unigrams_found_positive = True        
        
        ####################################################################
        unigrams_found = unigrams_found_negative and unigrams_found_positive
        bigrams_found = bigrams_found_negative and bigrams_found_positive
        trigrams_found = trigrams_found_negative and trigrams_found_positive
        
        if not unigrams_found:
            print("Warning! At least one of utterance lists is empty. Perhaps the documents only contain stop words")
            neg_dict = {}
            pos_dict = {}
            neg_list = [] 
            pos_list = []
        else:
            num_keywords1 = round(num_keywords*COEFF[0])
        
            neg_dict, pos_dict, neg_list, pos_list =  get_features(text_negative,text_positive,num_keywords1,n_range=(1,1))
            
            if not bigrams_found:
                print("Warning! No bigrams for at least one of utterance lists.")  
            else:
     
                num_keywords2 = round(num_keywords*COEFF[1])
            
                neg_dict2, pos_dict2, neg_list2, pos_list2 =  get_features(text_negative,text_positive,num_keywords2,n_range=(2,2))
                
                neg_list.extend(neg_list2)  
                pos_list.extend(pos_list2) 
                
                neg_dict.update(neg_dict2) 
                pos_dict.update(pos_dict2) 
                
                if not trigrams_found:
                    print("Warning! No trigrams for at least one of utterance lists.")    
                else:  
                    num_keywords3 = num_keywords - num_keywords1 - num_keywords2 
    
                    neg_dict3, pos_dict3, neg_list3, pos_list3 =  get_features(text_negative,text_positive,num_keywords3,n_range=(3,3))
     
                    neg_list.extend(neg_list3)                      
                    pos_list.extend(pos_list3)                     
                   
                    neg_dict.update(neg_dict3)                   
                    pos_dict.update(pos_dict3)
  
    ####################################################################################################   
    df_frequencies = get_df_frequencies(neg_dict, pos_dict, neg_list, pos_list, title1, title2) 
    
    df_frequencies.rename(columns={"Failure keywords: negative corpus frequencies, %": title1+": Frequency",\
                                   "Failure keywords: positive corpus frequencies, %": title1+": Power",\
                                   "Success keywords: positive corpus frequencies, %": title2+": Frequency",\
                                   "Success keywords: negative corpus frequencies, %": title2+": Power"}, inplace=True)
     
    return df_frequencies

################################################################################################
def get_data_for_comparison_visual(user_input_abandoned, user_input_completed, num_keywords, custom_stop_words=[]):
        
    df_keywords_detailed = keyword_table_detailed(user_input_abandoned, user_input_completed, num_keywords, custom_stop_words)
    
    #print(df_keywords_detailed.head(n=25))
    
    data={}
    data["name"]=""
    data["children"]=[]
    for i in range(0,len(df_keywords_detailed)):
        if  df_keywords_detailed.iloc[i,1] is None:
            data["children"].append({"name": df_keywords_detailed.iloc[i,0], "value": df_keywords_detailed.iloc[i,1]})
        else:
            data["children"].append({"name": df_keywords_detailed.iloc[i,0], "value": int(df_keywords_detailed.iloc[i,1])})
        
    return data  
pandas.DataFrame