class_docstr: Openai transformer, as per https://blog.openai.com/language-unsupervised/.
  Default parameters are the ones for their pretrained model.
  
  Parameters
  ----------
  vocab_size: ``int`` (optional, default: 40990)
      The size of the vocabulary, including the positional encodings.
  n_ctx: ``int`` (optional, default: 512)
      The number of positional encodings.
  embedding_dim: ``int`` (optional, default: 768)
      The dimension of the output embeddings.
  num_heads: ``int`` (optional, default: 12)
      How many "heads" the attention has.
  num_layers: ``int`` (optional, default: 12)
      How many layers of "blocks" the transformer has.
  embedding_dropout_probability: ``float`` (optional, default: 0.1)
      Dropout for the embedding.
  attention_dropout_probability: ``float`` (optional, default: 0.1)
      Dropout for attention.
  residual_dropout_probability: ``float`` (optional, default: 0.1)
      Dropout for residual
  activation_function: ``str`` (optional, default: ``'gelu'``)
      Activation function for the multi-layer perceptron.
  model_path: ``str`` (optional, default: ``None``)
      A tar.gz file containing serialized model weights. If supplied,
      the weights will be loaded from that file.
  requires_grad: ``bool`` (optional, default: ``False``)
      If true, the transformer will be fine-tuneable.
class_docstr_tok: ["Openai","transformer",",","as","per","https",":","\/\/blog.openai.com\/language-unsupervised\/",".","Default","parameters","are","the","ones","for","their","pretrained","model",".","Parameters","--","--","--","--","--","vocab_size",":","``","int","``","(","optional",",","default",":","40990",")","The","size","of","the","vocabulary",",","including","the","positional","encodings",".","n_ctx",":","``","int","``","(","optional",",","default",":","512",")","The","number","of","positional","encodings",".","embedding_dim",":","``","int","``","(","optional",",","default",":","768",")","The","dimension","of","the","output","embeddings",".","num_heads",":","``","int","``","(","optional",",","default",":","12",")","How","many","``","heads","''","the","attention","has",".","num_layers",":","``","int","``","(","optional",",","default",":","12",")","How","many","layers","of","``","blocks","''","the","transformer","has",".","embedding_dropout_probability",":","``","float","``","(","optional",",","default",":","0.1",")","Dropout","for","the","embedding",".","attention_dropout_probability",":","``","float","``","(","optional",",","default",":","0.1",")","Dropout","for","attention",".","residual_dropout_probability",":","``","float","``","(","optional",",","default",":","0.1",")","Dropout","for","residual","activation_function",":","``","str","``","(","optional",",","default",":","``","'gelu","'","``",")","Activation","function","for","the","multi-layer","perceptron",".","model_path",":","``","str","``","(","optional",",","default",":","``","None","``",")","A","tar.gz","file","containing","serialized","model","weights",".","If","supplied",",","the","weights","will","be","loaded","from","that","file",".","requires_grad",":","``","bool","``","(","optional",",","default",":","``","False","``",")","If","true",",","the","transformer","will","be","fine-tuneable","."]
code: def __init__(
      self,
      vocab_size=40990,
      n_ctx=512,
      embedding_dim=768,
      num_heads=12,
      num_layers=12,
      embedding_dropout_probability=0.1,
      attention_dropout_probability=0.1,
      residual_dropout_probability=0.1,
      activation_function="gelu",
      model_path=None,
      requires_grad=False,
  ):
      super(OpenaiTransformer, self).__init__()
  
      config = TransformerConfig(
          embedding_dim,
          num_heads,
          embedding_dropout_probability,
          attention_dropout_probability,
          residual_dropout_probability,
          activation_function,
      )
  
      self.vocab_size = vocab_size
      self.n_ctx = n_ctx
  
      self.num_output_layers = 1 + num_layers
  
      self.embed = torch.nn.Embedding(vocab_size, embedding_dim)
      self.drop = torch.nn.Dropout(embedding_dropout_probability)
  
      block = Block(n_ctx, config, scale=True)
      self.h = torch.nn.ModuleList(
          [copy.deepcopy(block) for _ in range(num_layers)]
      )
      self.decoder = torch.nn.Linear(
          embedding_dim, vocab_size, bias=False
      )
      self.decoder.weight = self.embed.weight  # Tied weights
      # To reproduce the noise_shape parameter of TF implementation
  
      torch.nn.init.normal_(self.embed.weight, std=0.02)
  
      for parameter in self.parameters():
          parameter.requires_grad = requires_grad
  
      if model_path:
          self.load_weights(model_path)
  
imports: from __future__ import with_statement
  from __future__ import division
  from __future__ import absolute_import
  import copy
  import io
  import json
  import logging
  import math
  import re
  import tarfile
  import numpy
  import torch
  from torch.nn import Parameter
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.file_utils import cached_path
  from allennlp.common.from_params import FromParams
  from io import open
label_desc: Use image node_id 1 to create a new OpenaiTransformer object from inherited base classes: torch.nn, FromParams with example: obj = OpenaiTransformer(vocab_size, n_ctx, embedding_dim, num_heads, num_layers, embedding_dropout_probability, attention_dropout_probability, residual_dropout_probability, activation_function, model_path, requires_grad)