class_docstr: ## Recurrent Highway Network Cell
  
  This implements equations $(6) - (9)$.
  
  $s_d^t = h_d^t \odot g_d^t + s_{d - 1}^t \odot c_d^t$
  
  where
  
  egin{align}
  h_0^t &=    anh(lin_{hx}(x) + lin_{hs}(s_D^{t-1})) \
  g_0^t &= \sigma(lin_{gx}(x) + lin_{gs}^1(s_D^{t-1})) \
  c_0^t &= \sigma(lin_{cx}(x) + lin_{cs}^1(s_D^{t-1}))
  \end{align}
  
  and for $0 < d < D$
  
  egin{align}
  h_d^t &=    anh(lin_{hs}^d(s_d^t)) \
  g_d^t &= \sigma(lin_{gs}^d(s_d^t)) \
  c_d^t &= \sigma(lin_{cs}^d(s_d^t))
  \end{align}
  
  $\odot$ stands for element-wise multiplication.
  
  Here we have made a couple of changes to notations from the paper.
  To avoid confusion with time, gate is represented with $g$,
  which was $t$ in the paper.
  To avoid confusion with multiple layers we use $d$ for depth and $D$ for
  total depth instead of $l$ and $L$ from the paper.
  
  We have also replaced the weight matrices and bias vectors from the equations with
  linear transforms, because that's how the implementation is going to look like.
  
  We implement weight tying, as described in paper, $c_d^t = 1 - g_d^t$.
class_docstr_tok: ["#","#","Recurrent","Highway","Network","Cell","This","implements","equations","$","(","6",")","-","(","9",")","$",".","$","s_d^t","=","h_d^t","\\odot","g_d^t","+","s_","{","d","-","1","}","^t","\\odot","c_d^t","$","where","\begin","{","align","}","h_0^t","&","=","anh","(","lin_","{","hx","}","(","x",")","+","lin_","{","hs","}","(","s_D^","{","t-1","}",")",")","\\","g_0^t","&","=","\\sigma","(","lin_","{","gx","}","(","x",")","+","lin_","{","gs","}","^1","(","s_D^","{","t-1","}",")",")","\\","c_0^t","&","=","\\sigma","(","lin_","{","cx","}","(","x",")","+","lin_","{","cs","}","^1","(","s_D^","{","t-1","}",")",")","\\end","{","align","}","and","for","$","0","<","d","<","D","$","\begin","{","align","}","h_d^t","&","=","anh","(","lin_","{","hs","}","^d","(","s_d^t",")",")","\\","g_d^t","&","=","\\sigma","(","lin_","{","gs","}","^d","(","s_d^t",")",")","\\","c_d^t","&","=","\\sigma","(","lin_","{","cs","}","^d","(","s_d^t",")",")","\\end","{","align","}","$","\\odot","$","stands","for","element-wise","multiplication",".","Here","we","have","made","a","couple","of","changes","to","notations","from","the","paper",".","To","avoid","confusion","with","time",",","gate","is","represented","with","$","g","$",",","which","was","$","t","$","in","the","paper",".","To","avoid","confusion","with","multiple","layers","we","use","$","d","$","for","depth","and","$","D","$","for","total","depth","instead","of","$","l","$","and","$","L","$","from","the","paper",".","We","have","also","replaced","the","weight","matrices","and","bias","vectors","from","the","equations","with","linear","transforms",",","because","that","'s","how","the","implementation","is","going","to","look","like",".","We","implement","weight","tying",",","as","described","in","paper",",","$","c_d^t","=","1","-","g_d^t","$","."]
docstr: `input_size` is the feature length of the input and `hidden_size` is
  the feature length of the cell.
  `depth` is $D$.
docstr_tok: ["`","input_size","`","is","the","feature","length","of","the","input","and","`","hidden_size","`","is","the","feature","length","of","the","cell",".","`","depth","`","is","$","D","$","."]
code: def __init__(self, input_size: int, hidden_size: int, depth: int):
      """
      `input_size` is the feature length of the input and `hidden_size` is
      the feature length of the cell.
      `depth` is $D$.
      """
      super().__init__()
  
      self.hidden_size = hidden_size
      self.depth = depth
      # We combine $lin_{hs}$ and $lin_{gs}$, with a single linear layer.
      # We can then split the results to get the $lin_{hs}$ and $lin_{gs}$ components.
      # This is the $lin_{hs}^d$ and $lin_{gs}^d$ for $0 \leq d < D$.
      self.hidden_lin = nn.ModuleList(
          [
              nn.Linear(hidden_size, 2 * hidden_size)
              for _ in range(depth)
          ]
      )
  
      # Similarly we combine $lin_{hx}$ and $lin_{gx}$.
      self.input_lin = nn.Linear(
          input_size, 2 * hidden_size, bias=False
      )
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/recurrent_highway_networks/__init__.py
imports: from typing import Optional
  import torch
  from torch import nn
  from labml_helpers.module import Module
label_desc: Use image node_id 1 to create a new RHNCell object from inherited base classes: Module with example: obj = RHNCell(input_size, hidden_size, depth)