class_docstr: GPT Transformer Model
class_docstr_tok: ["GPT","Transformer","Model"]
docstr: input_embeds: [batch_size x seq_len x n_embed]
  attention_masks: [batch_size x seq_len], 0 don't attend, 1 attend
docstr_tok: ["input_embeds",":","[","batch_size","x","seq_len","x","n_embed","]","attention_masks",":","[","batch_size","x","seq_len","]",",","0","do","n't","attend",",","1","attend"]
code: def forward(
      self, input_embeds, attention_masks=None, return_attentions=False
  ):
      """
      input_embeds: [batch_size x seq_len x n_embed]
      attention_masks: [batch_size x seq_len], 0 don't attend, 1 attend
      """
      B, T, C = input_embeds.size()
      assert T <= self.block_size, (
          f"Cannot forward sequence of length {T}, "
          f"block size is only {self.block_size}"
      )
  
      if attention_masks is not None:
          _B, _T = attention_masks.size()
          assert _B == B and _T == T
          # We create a 3D attention mask from a 2D tensor mask.
          # Sizes are [batch_size, 1, 1, to_seq_len]
          # So we can broadcast to
          # [batch_size, num_heads, from_seq_length, to_seq_length]
          # this attention mask is more simple than the triangular
          # masking of causal attention used in OpenAI GPT, we just need
          # to prepare the broadcast dimension here.
          attention_masks = attention_masks[:, None, None, :]
  
          # Since attention_mask is 1.0 for positions we want to attend
          # and 0.0 for masked positions, this operation will create a
          # tensor which is 0.0 for positions we want to attend and -inf
          # for masked positions. Since we are adding it to the raw scores
          # before the softmax, this is effectively the same as removing
          # these entirely.
          attention_masks = attention_masks.to(dtype=input_embeds.dtype)
          attention_masks = (1.0 - attention_masks) * -1e9
  
      # forward the GPT model itself
      x = self.transformer.drop(input_embeds)
  
      atts = []
      for block in self.transformer.h:
          x, att = block(x, attention_masks=attention_masks)
          atts.append(att)
      x = self.transformer.ln_f(x)
  
      if return_attentions:
          return x, atts
      else:
          return x
  
imports: import math
  from dataclasses import dataclass
  from typing import Tuple
  import torch
  import torch.nn
  from torch.nn import functional
  from ray.rllib.utils.annotations import DeveloperAPI
  from ray.rllib.utils.deprecation import Deprecated
label_desc: Use image node_id 3 for calling the GPT obj's underlying member method code with example usage: obj.forward(input_embeds, attention_masks, return_attentions) and returns: x,  atts, x