class_docstr: ## Autoregressive language model
class_docstr_tok: ["#","#","Autoregressive","language","model"]
docstr: * `n_tokens` is the vocabulary size
  * `d_model` is the size of the token embeddings
  * `dropout` is the dropout probability
  * `hour_glass` is the [hourglass model](index.html)
docstr_tok: ["*","`","n_tokens","`","is","the","vocabulary","size","*","`","d_model","`","is","the","size","of","the","token","embeddings","*","`","dropout","`","is","the","dropout","probability","*","`","hour_glass","`","is","the","[","hourglass","model","]","(","index.html",")"]
code: def __init__(
      self,
      n_tokens: int,
      d_model: int,
      dropout: float,
      hour_glass: HourGlass,
  ):
      """
      * `n_tokens` is the vocabulary size
      * `d_model` is the size of the token embeddings
      * `dropout` is the dropout probability
      * `hour_glass` is the [hourglass model](index.html)
      """
      super().__init__()
      # Token embeddings
      self.embedding = nn.Embedding(n_tokens, d_model)
      # [Fixed positional embeddings](../positional_encoding.html).
      #
      # 📝 The
      # [official paper implementation](https://github.com/google/trax/blob/master/trax/models/research/hourglass.py)
      # use [relative attention](../xl/relative_mha.html)
      self.pos_embedding = PositionalEncoding(d_model, dropout)
      # [hourglass model](index.html)
      self.hour_glass = hour_glass
      # To normalize the final embeddings
      self.norm = nn.LayerNorm([d_model])
      # Embedding size
      self.d_model = d_model
      # Final linear layer to predict the logits
      self.output = nn.Linear(d_model, n_tokens)
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/transformers/hour_glass/experiment.py
imports: import math
  from typing import List
  import torch
  from torch import nn
  from labml import experiment
  from labml.configs import option
  from labml_helpers.module import Module
  from labml_nn.experiments.nlp_autoregression import NLPAutoRegressionConfigs
  from labml_nn.transformers.hour_glass import HourGlass
  from labml_nn.transformers.positional_encoding import PositionalEncoding
label_desc: Use image node_id 1 to create a new AutoregressiveTransformer object from inherited base classes: Module with example: obj = AutoregressiveTransformer(n_tokens, d_model, dropout, hour_glass)