"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Looking LSTM Detailly !
</h1>
</div>
"""
"""
![](https://www.csail.mit.edu/sites/default/files/2020-08/FedTech-DeepLearning.jpg)
"""
"""
### This time, I try Looking LSTM Detailly ( Especially Model's Weights )! 
"""
"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Import Libraries and Load Datasets
</h1>
</div>
"""
import numpy as np
import pandas as pd
from keras import models, layers
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt

import matplotlib.font_manager as fm
best_font = fm.FontProperties(fname='../input/staatfont/Staatliches-Regular.ttf')
"""
### I use bitcoin price dataset.
### LSTM Showing great accuracy in Time series predict!
"""
"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Data processing and make simple model
</h1>
</div>
"""
data =pd.read_csv("../input/bitcoin-historical-data/bitstampUSD_1-min_data_2012-01-01_to_2021-03-31.csv")
data = data[['Timestamp','Open']]
data['Timestamp'] = pd.to_datetime(data['Timestamp'],unit='s').dt.date
data = data.dropna()
data = data.groupby('Timestamp').mean()

data = data.reset_index(drop=True)
data.columns=['price']

scaler = MinMaxScaler()
data['price'] = scaler.fit_transform(np.array(data['price']).reshape(-1,1))

X = []
y = []
for i in range(len(data)-5):
    X.append(list(data.loc[i:i+4,"price"]))
    y.append(data.loc[i+5,"price"])
    
X = np.array(X)
y = np.array(y)
X = X.reshape(-1,5,1)

model = models.Sequential()
model.add(layers.LSTM(5,input_shape=X.shape[1:]))
model.add(layers.Dense(1))
model.compile(optimizer='adam',loss='MSE')
model.summary()
model.fit(X, y, epochs=5,verbose=2,validation_split=0.3)
"""
### I use only Price data and Windowing size is 5
### I just make simple model.
### If you want more great performance, Try increase Windowing size and NN
"""
"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Look at model's Weights
</h1>
</div>
"""
names = [weight.name for layer in model.layers for weight in layer.weights]
weights = model.get_weights()

np.set_printoptions(suppress=True)
for name, weight in zip(names, weights):
    print(name, weight.shape)
    print(weight)

    layer_type = name.split('/')[1]
    if layer_type == 'kernel:0':
        kernel_0 = weight
    if layer_type == 'recurrent_kernel:0':
        recurrent_kernel_0 = weight
    elif layer_type == 'bias:0':
        bias_0 = weight
    print()
"""
### The LSTM Model has 3 weights
### 1. kernel
### 2. recurrent_kernel
### 3. bias
"""
"""
![](https://www.oreilly.com/library/view/neural-networks-and/9781492037354/assets/mlst_1413.png)
"""
"""
### This image help to understand LSTM's contruction
"""
kernel_weights = weights[0]
recurrent_kernel_weights = weights[1]
bias = weights[2]

n = 1
units = 5  # LSTM layers

# (1, 20) embedding dims, units * 4
Wi = kernel_weights[:, 0:units]
Wf = kernel_weights[:, units:2 * units]
Wc = kernel_weights[:, 2 * units:3 * units]
Wo = kernel_weights[:, 3 * units:]

# (5, 20) units, units * 4
Ui = recurrent_kernel_weights[:, 0:units]
Uf = recurrent_kernel_weights[:, units:2 * units]
Uc = recurrent_kernel_weights[:, 2 * units:3 * units]
Uo = recurrent_kernel_weights[:, 3 * units:]

# (20,) units * 4
bi = bias[0:units]
bf = bias[units:2 * units]
bc = bias[2 * units:3 * units]
bo = bias[3 * units:]

def sigmoid(x):
    return 1 / (1 + np.exp(-x))
print('Wf: ',Wf,'\nWi: ',Wi,'\nWo: ',Wo,'\nWc: ',Wc,)

print('\nUf: ',Uf,'\nUi: ',Ui,'\nUo: ',Uo,'\nUc: ',Uc,)

print('\nbf: ',bf,'\nbi: ',bi,'\nbo: ',bo,'\nbc: ',bc,)
"""
![](https://wikimedia.org/api/rest_v1/media/math/render/svg/2db2cba6a0d878e13932fa27ce6f3fb71ad99cf1)
"""
"""
### Using weights, We can know Cell_state values(Ct) and Hidden_state(ht) values
"""
ht_1 = np.zeros(n * units).reshape(n, units)
Ct_1 = np.zeros(n * units).reshape(n, units)

results = []
for t in range(0, len(X[2000,:])):
    xt = np.array(X[2000,t])
    ft = sigmoid(np.dot(xt, Wf) + np.dot(ht_1, Uf) + bf)  # forget gate
    it = sigmoid(np.dot(xt, Wi) + np.dot(ht_1, Ui) + bi)  # input gate
    ot = sigmoid(np.dot(xt, Wo) + np.dot(ht_1, Uo) + bo)  # output gate
    Ct = ft * Ct_1 + it * np.tanh(np.dot(xt, Wc) + np.dot(ht_1, Uc) + bc)
    ht = ot * np.tanh(Ct)

    ht_1 = ht  # hidden state, previous memory state
    Ct_1 = Ct  # cell state, previous carry state

    results.append(ht)
    print(t,': ht', ht)
model.predict(X[2000:2001])
"""
### Using this calculation, We can get same values with model's predict
### last hidden_state value(ht[4]) == model.predict value
"""
"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Which column will have the biggest impact?
</h1>
</div>
"""
"""
### I want to know that Which columns have the biggest impact.
### So, I calculate this LSTM process
"""
ht_1 = np.zeros(n * units).reshape(n, units)
Ct_1 = np.zeros(n * units).reshape(n, units)

h_t_value = []

influence_h_t_value = []
for t in range(0, len(X[1000,:])):
    xt = np.array(X[1000,t])
    ft = sigmoid(np.dot(xt, Wf) + np.dot(ht_1, Uf) + bf)  # forget gate
    influence_ft = (np.dot(ht_1, Uf))/(np.dot(xt, Wf) + np.dot(ht_1, Uf) + bf) * ft

    it = sigmoid(np.dot(xt, Wi) + np.dot(ht_1, Ui) + bi)  # input gate
    influence_it = (np.dot(ht_1, Ui))/(np.dot(xt, Wi) + np.dot(ht_1, Ui) + bi) * it

    ot = sigmoid(np.dot(xt, Wo) + np.dot(ht_1, Uo) + bo)  # output gate
    influence_ot = np.dot(ht_1, Uo) / (np.dot(xt, Wo) + np.dot(ht_1, Uo) + bo) * ot

    gt =  np.tanh(np.dot(xt, Wc) + np.dot(ht_1, Uc) + bc)
    influence_gt =np.dot(ht_1, Uc) / (np.dot(xt, Wc) + np.dot(ht_1, Uc) + bc) * gt
    
    Ct = ft * Ct_1 + it * gt
    influence_ct = influence_ft * Ct_1 + influence_it * influence_gt
    ht = ot * np.tanh(Ct)
    influence_ht = influence_ot * (influence_ct/Ct) * ht
    
    influence_h_t_value.append(influence_ht)

    ht_1 = ht  # hidden state, previous memory state
    Ct_1 = Ct  # cell state, previous carry state
    
    h_t_value.append(ht)
    
influence_h_t_value.append(h_t_value[-1])
for i in range(len(influence_h_t_value)-1,0,-1):
    influence_h_t_value[i] = influence_h_t_value[i] - influence_h_t_value[i-1]
    
influence_h_t_value = influence_h_t_value[1:]
impact_columns = np.dot(influence_h_t_value,weights[3]) + (weights[4]/5)

for i in range(len(impact_columns)):
    print('columns_number : ',i, 'impact value : ', float(impact_columns[i]))
    
print('\nSum of value : ', float(sum(np.dot(influence_h_t_value,weights[3]) + (weights[4]/5))))

print('\nkeras model_predict : ', float(model.predict(X[1000:1001])))
"""
### I've used three days to do this. 😴
"""
"""
### Then, Why am i obsessed with the impact?
"""
"""
<div style="color:white;
           display:fill;
           border-radius:5px;
           background-color:blue;
           font-size:110%;
           font-family:Verdana;
           letter-spacing:0.5px">
<h1 style="text-align: center;
           padding: 10px;
              color:white">
Trying on NLP ! ( Review Data )
</h1>
</div>
"""
"""
### I just try that on NLP
"""
data = pd.read_csv('../input/womens-ecommerce-clothing-reviews/Womens Clothing E-Commerce Reviews.csv')
data = data[['Review Text','Rating']]
data = data.dropna()
positive = data[(data['Rating'] == 5)].sample(2370,random_state=100)
negative = data[(data['Rating'] == 2) | (data['Rating'] == 1)]
data = pd.concat([negative,positive])
data = data.reset_index(drop=True)

data['Rating'] = data['Rating'].apply(lambda x : 1 if x==5 else 0)
"""
### I used Womens Clothing E-Commerce Reviews dataset
"""
data.head(5)
data.tail(5)
"""
### Rating mean that 0 is Negative review, and 1 is Positive review
"""
import re
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer

def data_processing(text):
    return_arr = []
    text = re.sub(r"[^a-zA-Z]"," ",text)
    text = re.sub(r" {2,}"," ",text)
    text = text.lower()
    words = word_tokenize(text)
    s = PorterStemmer()
    stopword = stopwords.words('english')
    for t in words:
        if t not in stopword:
            return_arr.append(t)
            
    return_arr = [s.stem(w) for w in return_arr]
    return return_arr

data['processing'] = data['Review Text'].apply(data_processing)
"""
### I only use alphabet text, (delete special character or number)
### And, delete stopwords, changing analogous term.
"""
len_arr = []
for i in range(len(data)):
    len_arr.append(len(data.loc[i,'processing']))
    
import seaborn as sns
sns.histplot(len_arr)
def data_processing2(text):
    return_arr = []
    text = re.sub(r"[^a-z]"," ",text)
    text = re.sub(r" {2,}"," ",text)
    return text

negative_sentences = data[data['Rating'] ==0]['processing']
positive_sentences = data[data['Rating'] ==1]['processing']

negative_sentences = str(list(negative_sentences))
negative_sentences = data_processing2(negative_sentences)

negative_sentences = negative_sentences.split(' ')
negative_sentences = pd.DataFrame(negative_sentences)[0].value_counts()
negative_sentences = pd.DataFrame(negative_sentences)

positive_sentences = str(list(positive_sentences))
positive_sentences = data_processing2(positive_sentences)

positive_sentences = positive_sentences.split(' ')
positive_sentences = pd.DataFrame(positive_sentences)[0].value_counts()
positive_sentences = pd.DataFrame(positive_sentences)

con = pd.concat([negative_sentences,positive_sentences],axis=1)

con.columns = ['negative','positive']

con = con.dropna()

con['negative_value'] = con[['negative','positive']].apply(lambda x : x[0]/(x[0]+x[1]) *-1,axis=1)
con['positive_value'] = con[['negative','positive']].apply(lambda x : x[1]/(x[0]+x[1]),axis=1)

con['total_value'] = con['negative_value'] + con['positive_value']
con = con.reset_index()
con = con.drop(['negative','positive','negative_value','positive_value'],axis=1)
word_index =con['index'].to_list()
con = np.array(con)
"""
### And, I just use, Rate of Negative words, and Positive word 

### It mean

### Negative Rate : -1 * Negatvie Counts / (Negative Counts + Positive Counts)
### Positive Rate : 1 * Positive Counts / (Negative Counts + Positive Counts)

### After Sum Two : Negative Rate + Positive Rate
"""
con[:10]
from tqdm import tqdm

X = []
for sen in tqdm(data['processing']):
    word_arr =[]
    for word in sen:
        if word in word_index:
            word_arr.append(float(con[con[:,0] ==word,1]))
        else:
            word_arr.append(0)
    X.append(word_arr)
X = pd.DataFrame(X)
X = X.fillna(0)
X = X.loc[:,:50]
X= X.to_numpy()
y = np.array(data['Rating'])
X = X.reshape(4740,51,1)
X.shape, y.shape
from tensorflow.keras.layers import Embedding, Dense,LSTM
from tensorflow.keras.models import Sequential

model = Sequential()
model.add(LSTM(51,input_shape=X.shape[1:],activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
model.summary()
"""
### Just use simple LSTM Model
"""
model.fit(X,y, batch_size=128, epochs=15, validation_split=0.3)
names = [weight.name for layer in model.layers for weight in layer.weights]
weights = model.get_weights()

kernel_weights = weights[0]
recurrent_kernel_weights = weights[1]
bias = weights[2]

n = 1
units = 51  # LSTM layers

Wi = kernel_weights[:, 0:units]
Wf = kernel_weights[:, units:2 * units]
Wc = kernel_weights[:, 2 * units:3 * units]
Wo = kernel_weights[:, 3 * units:]

Ui = recurrent_kernel_weights[:, 0:units]
Uf = recurrent_kernel_weights[:, units:2 * units]
Uc = recurrent_kernel_weights[:, 2 * units:3 * units]
Uo = recurrent_kernel_weights[:, 3 * units:]

bi = bias[0:units]
bf = bias[units:2 * units]
bc = bias[2 * units:3 * units]
bo = bias[3 * units:]
def make_plot(number):
    ht_1 = np.zeros(n * units).reshape(n, units)
    Ct_1 = np.zeros(n * units).reshape(n, units)

    h_t_value = []

    influence_h_t_value = []
    for t in range(0, len(X[number,:])):
        xt = np.array(X[number,t])
        ft = sigmoid(np.dot(xt, Wf) + np.dot(ht_1, Uf) + bf)  # forget gate
        influence_ft = (np.dot(ht_1, Uf))/(np.dot(xt, Wf) + np.dot(ht_1, Uf) + bf) * ft

        it = sigmoid(np.dot(xt, Wi) + np.dot(ht_1, Ui) + bi)  # input gate
        influence_it = (np.dot(ht_1, Ui))/(np.dot(xt, Wi) + np.dot(ht_1, Ui) + bi) * it

        ot = sigmoid(np.dot(xt, Wo) + np.dot(ht_1, Uo) + bo)  # output gate
        influence_ot = np.dot(ht_1, Uo) / (np.dot(xt, Wo) + np.dot(ht_1, Uo) + bo) * ot

        gt =  np.tanh(np.dot(xt, Wc) + np.dot(ht_1, Uc) + bc)
        influence_gt =np.dot(ht_1, Uc) / (np.dot(xt, Wc) + np.dot(ht_1, Uc) + bc) * gt

        Ct = ft * Ct_1 + it * gt
        influence_ct = influence_ft * Ct_1 + influence_it * influence_gt
        ht = ot * np.tanh(Ct)
        influence_ht = influence_ot * (influence_ct/Ct) * ht

        influence_h_t_value.append(influence_ht)

        ht_1 = ht  # hidden state, previous memory state
        Ct_1 = Ct  # cell state, previous carry state

        h_t_value.append(ht)

    influence_h_t_value.append(h_t_value[-1])
    for i in range(len(influence_h_t_value)-1,0,-1):
        influence_h_t_value[i] = influence_h_t_value[i] - influence_h_t_value[i-1]

    influence_h_t_value = influence_h_t_value[1:]

    impact_columns = np.dot(influence_h_t_value,weights[3]) + (weights[4]/units)

    if model.predict(X[number:number+1]) > 0.5:
        b_color = 'lightgreen'
    else:
        b_color ='lightcyan'

    fig = plt.figure(figsize=(15,3),facecolor=b_color)

    for k in range(len(data.loc[number,'processing'])):
        s = data.loc[number,'processing'][k]
        va = round(float(impact_columns[k]),2)
        if va > 0.5:
            color ='green'
        elif va< -0.3:
            color ='blue'
        else:
            color ='black'

        if k < 17:
            plt.text(s=s, x=k*0.7, y=0,font=best_font,fontsize=20,color=color,va='center',ha='center')
            plt.text(s=va,x=k*0.7, y=-0.1,font=best_font,fontsize=20,color=color,va='center',ha='center')
        elif k < 34:
            plt.text(s=s, x=k*0.7 - 17*0.7, y=-0.2,font=best_font,fontsize=20,color=color,va='center',ha='center')
            plt.text(s=va,x=k*0.7- 17*0.7, y=-0.3,font=best_font,fontsize=20,color=color,va='center',ha='center')
        else:
            plt.text(s=s, x=k*0.7 - 34*0.7, y=-0.4,font=best_font,fontsize=20,color=color,va='center',ha='center')
            plt.text(s=va,x=k*0.7- 34*0.7, y=-0.5,font=best_font,fontsize=20,color=color,va='center',ha='center')

    plt.xlim(0,10)
    plt.ylim(-0.5,0.1)
    plt.axis('off')
    plt.show()
"""
### If influence value is more than 0.5 : Green color
### If influence value is lower than -0.5 : blue color

### The background color mean too
"""
"""
### ( I don't consider activation function like sigmoid )
"""
make_plot(489)
make_plot(2243)
make_plot(2378)
make_plot(2628)
make_plot(4560)
make_plot(4339)
"""
### I expect if you use word2vec or FastText. etc Can get more meaningful accuracy.
### If you have Good idea, feel free to review comments plz.
### I always welcom feedback
"""
"""
### reference
* http://docs.likejazz.com/lstm/
* http://colah.github.io/posts/2015-08-Understanding-LSTMs/
"""