class_docstr: ## Decoder module
  
  This consists of a LSTM
class_docstr_tok: ["#","#","Decoder","module","This","consists","of","a","LSTM"]
code: def forward(
      self,
      x: torch.Tensor,
      z: torch.Tensor,
      state: Optional[Tuple[torch.Tensor, torch.Tensor]],
  ):
      # Calculate the initial state
      if state is None:
          # $[h_0; c_0] = \tanh(W_{z}z + b_z)$
          h, c = torch.split(
              torch.tanh(self.init_state(z)), self.dec_hidden_size, 1
          )
          # `h` and `c` have shapes `[batch_size, lstm_size]`. We want to shape them
          # to `[1, batch_size, lstm_size]` because that's the shape used in LSTM.
          state = (
              h.unsqueeze(0).contiguous(),
              c.unsqueeze(0).contiguous(),
          )
  
      # Run the LSTM
      outputs, state = self.lstm(x, state)
  
      # Get $\log(q)$
      q_logits = self.q_log_softmax(self.q_head(outputs))
  
      # Get $(\hat{\Pi_i}, \mu_{x,i}, \mu_{y,i}, \hat{\sigma_{x,i}},
      # \hat{\sigma_{y,i}} \hat{\rho_{xy,i}})$.
      # `torch.split` splits the output into 6 tensors of size `self.n_distribution`
      # across dimension `2`.
      pi_logits, mu_x, mu_y, sigma_x, sigma_y, rho_xy = torch.split(
          self.mixtures(outputs), self.n_distributions, 2
      )
  
      # Create a bi-variate Gaussian mixture
      # $\Pi$ and
      # $\mathcal{N}(\mu_{x}, \mu_{y}, \sigma_{x}, \sigma_{y}, \rho_{xy})$
      # where
      # $$\sigma_{x,i} = \exp(\hat{\sigma_{x,i}}), \sigma_{y,i} = \exp(\hat{\sigma_{y,i}}),
      # \rho_{xy,i} = \tanh(\hat{\rho_{xy,i}})$$
      # and
      # $$\Pi_i = \operatorname{softmax}(\hat{\Pi})_i = \frac{\exp(\hat{\Pi_i})}{\sum_{j = 1}^3 \exp(\hat{\Pi_j})}$$
      #
      # $\Pi$ is the categorical probabilities of choosing the distribution out of the mixture
      # $\mathcal{N}(\mu_{x}, \mu_{y}, \sigma_{x}, \sigma_{y}, \rho_{xy})$.
      dist = BivariateGaussianMixture(
          pi_logits,
          mu_x,
          mu_y,
          torch.exp(sigma_x),
          torch.exp(sigma_y),
          torch.tanh(rho_xy),
      )
  
      #
      return dist, q_logits, state
  
imports: import math
  from typing import Optional, Tuple, Any
  import numpy
  import torch
  import torch.nn
  from matplotlib import pyplot
  from torch import optim
  from torch.utils.data import Dataset, DataLoader
  import einops
  from labml import lab, experiment, tracker, monit
  from labml_helpers.device import DeviceConfigs
  from labml_helpers.module import Module
  from labml_helpers.optimizer import OptimizerConfigs
  from labml_helpers.train_valid import TrainValidConfigs, hook_model_outputs, BatchIndex
label_desc: Use image node_id 2 for calling the DecoderRNN obj's underlying member method code with example usage: obj.forward(x, z, state) and returns: dist,  q_logits,  state