# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="216E63EBD0DC4C26828C96B784C49B24" mdEditEnable=false
# # GRU

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="35A5BB52AD23476AB3234E38360D557E" mdEditEnable=false
# RNN存在的问题：梯度较容易出现衰减或爆炸（BPTT）  
# ⻔控循环神经⽹络：捕捉时间序列中时间步距离较⼤的依赖关系  
# **RNN**:  
#
#
# ![Image Name](https://cdn.kesci.com/upload/image/q5jjvcykud.png?imageView2/0/w/320/h/320)
#
#
# $$
# H_{t} = ϕ(X_{t}W_{xh} + H_{t-1}W_{hh} + b_{h})
# $$
# **GRU**:
#
#
# ![Image Name](https://cdn.kesci.com/upload/image/q5jk0q9suq.png?imageView2/0/w/640/h/640)
#
#
#
# $$
# R_{t} = σ(X_tW_{xr} + H_{t−1}W_{hr} + b_r)\\    
# Z_{t} = σ(X_tW_{xz} + H_{t−1}W_{hz} + b_z)\\  
# \widetilde{H}_t = tanh(X_tW_{xh} + (R_t ⊙H_{t−1})W_{hh} + b_h)\\
# H_t = Z_t⊙H_{t−1} + (1−Z_t)⊙\widetilde{H}_t
# $$
# • 重置⻔有助于捕捉时间序列⾥短期的依赖关系；  
# • 更新⻔有助于捕捉时间序列⾥⻓期的依赖关系。    
# ### 载入数据集

# + id="303B4A29B33E4646B59188EC4D57BA14" jupyter={} tags=[] slideshow={"slide_type": "slide"}
import os
os.listdir('/home/kesci/input')

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="954C3D57745047D587E4BFA23CE0BFA0"
import numpy as np
import torch
from torch import nn, optim
import torch.nn.functional as F



# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="64E106307FF64926859FD92A826422D6"
import sys
sys.path.append("../input/")
import d2l_jay9460 as d2l
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

(corpus_indices, char_to_idx, idx_to_char, vocab_size) = d2l.load_data_jay_lyrics()

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="CE7C3A96CFCC449A8D5F65577841999B" mdEditEnable=false
# ### 初始化参数

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="9FB61618A6DB406D843C7BCEDDA63EB1"
num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size
print('will use', device)

def get_params():  
    def _one(shape):
        ts = torch.tensor(np.random.normal(0, 0.01, size=shape), device=device, dtype=torch.float32) #正态分布
        return torch.nn.Parameter(ts, requires_grad=True)
    def _three():
        return (_one((num_inputs, num_hiddens)),
                _one((num_hiddens, num_hiddens)),
                torch.nn.Parameter(torch.zeros(num_hiddens, device=device, dtype=torch.float32), requires_grad=True))
     
    W_xz, W_hz, b_z = _three()  # 更新门参数
    W_xr, W_hr, b_r = _three()  # 重置门参数
    W_xh, W_hh, b_h = _three()  # 候选隐藏状态参数
    
    # 输出层参数
    W_hq = _one((num_hiddens, num_outputs))
    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device, dtype=torch.float32), requires_grad=True)
    return nn.ParameterList([W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q])

def init_gru_state(batch_size, num_hiddens, device):   #隐藏状态初始化
    return (torch.zeros((batch_size, num_hiddens), device=device), )


# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="1CD23FE7F21C4D5D82702BE6CACE3DBF" mdEditEnable=false
# ### GRU模型

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="F4D5CB26DCBE45E58DECB282979C85D0"
def gru(inputs, state, params):
    W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q = params
    H, = state
    outputs = []
    for X in inputs:
        Z = torch.sigmoid(torch.matmul(X, W_xz) + torch.matmul(H, W_hz) + b_z)
        R = torch.sigmoid(torch.matmul(X, W_xr) + torch.matmul(H, W_hr) + b_r)
        H_tilda = torch.tanh(torch.matmul(X, W_xh) + R * torch.matmul(H, W_hh) + b_h)
        H = Z * H + (1 - Z) * H_tilda
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H,)


# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="DC589ABFFE464B32BD3CADCE9C64DE87" mdEditEnable=false
# ### 训练模型

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="CF81DCA87B884C2E996632B5E49BB9D9"
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="4FA0FC8D41C04605BD1FCA235D00233B"
d2l.train_and_predict_rnn(gru, get_params, init_gru_state, num_hiddens,
                          vocab_size, device, corpus_indices, idx_to_char,
                          char_to_idx, False, num_epochs, num_steps, lr,
                          clipping_theta, batch_size, pred_period, pred_len,
                          prefixes)

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="32035E22A7B04D3A8F787C7CB15F366F" mdEditEnable=false
# ### 简洁实现

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="CAE438B5EC9A454B8C1AAAF4EFEF7548"
num_hiddens=256
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率
gru_layer = nn.GRU(input_size=vocab_size, hidden_size=num_hiddens)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="15C47CF30681418997AEDB9F08C15C9B"
# # LSTM

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="E7D0864A57DB432483AD1B0C89171313" mdEditEnable=false
# ** 长短期记忆long short-term memory **:  
# 遗忘门:控制上一时间步的记忆细胞 
# 输入门:控制当前时间步的输入  
# 输出门:控制从记忆细胞到隐藏状态  
# 记忆细胞：⼀种特殊的隐藏状态的信息的流动  
#
#
# ![Image Name](https://cdn.kesci.com/upload/image/q5jk2bnnej.png?imageView2/0/w/640/h/640)
#
# $$
# I_t = σ(X_tW_{xi} + H_{t−1}W_{hi} + b_i) \\
# F_t = σ(X_tW_{xf} + H_{t−1}W_{hf} + b_f)\\
# O_t = σ(X_tW_{xo} + H_{t−1}W_{ho} + b_o)\\
# \widetilde{C}_t = tanh(X_tW_{xc} + H_{t−1}W_{hc} + b_c)\\
# C_t = F_t ⊙C_{t−1} + I_t ⊙\widetilde{C}_t\\
# H_t = O_t⊙tanh(C_t)
# $$
#

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="33A4C20200044DFC80CD2FB42F2D8CA0" mdEditEnable=false
# ### 初始化参数

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="E296216AE65A40608DA0B7856091AA39"
num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size
print('will use', device)

def get_params():
    def _one(shape):
        ts = torch.tensor(np.random.normal(0, 0.01, size=shape), device=device, dtype=torch.float32)
        return torch.nn.Parameter(ts, requires_grad=True)
    def _three():
        return (_one((num_inputs, num_hiddens)),
                _one((num_hiddens, num_hiddens)),
                torch.nn.Parameter(torch.zeros(num_hiddens, device=device, dtype=torch.float32), requires_grad=True))
    
    W_xi, W_hi, b_i = _three()  # 输入门参数
    W_xf, W_hf, b_f = _three()  # 遗忘门参数
    W_xo, W_ho, b_o = _three()  # 输出门参数
    W_xc, W_hc, b_c = _three()  # 候选记忆细胞参数
    
    # 输出层参数
    W_hq = _one((num_hiddens, num_outputs))
    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device, dtype=torch.float32), requires_grad=True)
    return nn.ParameterList([W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc, b_c, W_hq, b_q])

def init_lstm_state(batch_size, num_hiddens, device):
    return (torch.zeros((batch_size, num_hiddens), device=device), 
            torch.zeros((batch_size, num_hiddens), device=device))


# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="91D90EADF26C40728E0CBB1849E3FCF3" mdEditEnable=false
# ### LSTM模型

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="27A0EBB56386481F895FA2FFB0E9180F"
def lstm(inputs, state, params):
    [W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc, b_c, W_hq, b_q] = params
    (H, C) = state
    outputs = []
    for X in inputs:
        I = torch.sigmoid(torch.matmul(X, W_xi) + torch.matmul(H, W_hi) + b_i)
        F = torch.sigmoid(torch.matmul(X, W_xf) + torch.matmul(H, W_hf) + b_f)
        O = torch.sigmoid(torch.matmul(X, W_xo) + torch.matmul(H, W_ho) + b_o)
        C_tilda = torch.tanh(torch.matmul(X, W_xc) + torch.matmul(H, W_hc) + b_c)
        C = F * C + I * C_tilda
        H = O * C.tanh()
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H, C)


# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="40C5775290DB48DB937B836471F2AC17" mdEditEnable=false
# ### 训练模型

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="DD60C69E13F84A17BA441F7E75984E2F"
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

d2l.train_and_predict_rnn(lstm, get_params, init_lstm_state, num_hiddens,
                          vocab_size, device, corpus_indices, idx_to_char,
                          char_to_idx, False, num_epochs, num_steps, lr,
                          clipping_theta, batch_size, pred_period, pred_len,
                          prefixes)

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="F6D1F7613F63401CBDD49FBD7E611B4A" mdEditEnable=false
# ### 简洁实现

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="8AC2F2314D8C4DA097334B838732F9D2"
num_hiddens=256
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率
lstm_layer = nn.LSTM(input_size=vocab_size, hidden_size=num_hiddens)
model = d2l.RNNModel(lstm_layer, vocab_size)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="A5D90CA5217B4DB380BEF1CD17338BB6" mdEditEnable=false
# # 深度循环神经网络  
#
# ![Image Name](https://cdn.kesci.com/upload/image/q5jk3z1hvz.png?imageView2/0/w/320/h/320)
#
#
# $$
# \boldsymbol{H}_t^{(1)} = \phi(\boldsymbol{X}_t \boldsymbol{W}_{xh}^{(1)} + \boldsymbol{H}_{t-1}^{(1)} \boldsymbol{W}_{hh}^{(1)} + \boldsymbol{b}_h^{(1)})\\
# \boldsymbol{H}_t^{(\ell)} = \phi(\boldsymbol{H}_t^{(\ell-1)} \boldsymbol{W}_{xh}^{(\ell)} + \boldsymbol{H}_{t-1}^{(\ell)} \boldsymbol{W}_{hh}^{(\ell)} + \boldsymbol{b}_h^{(\ell)})\\
# \boldsymbol{O}_t = \boldsymbol{H}_t^{(L)} \boldsymbol{W}_{hq} + \boldsymbol{b}_q
# $$

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="9E06D2B447EA485B88F31C9B57ED3A79"


num_hiddens=256
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率

gru_layer = nn.LSTM(input_size=vocab_size, hidden_size=num_hiddens,num_layers=2)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="E386044D687F4D5D862CE5F6E3E2C00A"
gru_layer = nn.LSTM(input_size=vocab_size, hidden_size=num_hiddens,num_layers=6)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

# + [markdown] jupyter={} tags=[] slideshow={"slide_type": "slide"} id="6F10BD29920D44F1808692B1AAED1FA7" mdEditEnable=false
# # 双向循环神经网络 
#
# ![Image Name](https://cdn.kesci.com/upload/image/q5j8hmgyrz.png?imageView2/0/w/320/h/320)
#
# $$ 
# \begin{aligned} \overrightarrow{\boldsymbol{H}}_t &= \phi(\boldsymbol{X}_t \boldsymbol{W}_{xh}^{(f)} + \overrightarrow{\boldsymbol{H}}_{t-1} \boldsymbol{W}_{hh}^{(f)} + \boldsymbol{b}_h^{(f)})\\
# \overleftarrow{\boldsymbol{H}}_t &= \phi(\boldsymbol{X}_t \boldsymbol{W}_{xh}^{(b)} + \overleftarrow{\boldsymbol{H}}_{t+1} \boldsymbol{W}_{hh}^{(b)} + \boldsymbol{b}_h^{(b)}) \end{aligned} $$
# $$
# \boldsymbol{H}_t=(\overrightarrow{\boldsymbol{H}}_{t}, \overleftarrow{\boldsymbol{H}}_t)
# $$
# $$
# \boldsymbol{O}_t = \boldsymbol{H}_t \boldsymbol{W}_{hq} + \boldsymbol{b}_q
# $$

# + jupyter={} tags=[] slideshow={"slide_type": "slide"} id="176C228203A040D38D18DC194099C9EA"
num_hiddens=128
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e-2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率

gru_layer = nn.GRU(input_size=vocab_size, hidden_size=num_hiddens,bidirectional=True)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

# + id="412543B3642948FA89C831603F7362BA" jupyter={} tags=[] slideshow={"slide_type": "slide"}


StarcoderdataJupyterScriptsDedupFiltered