docstr: If max_position_embeddings <= 0, there's no position embeddings
  If word_embe_proj_dim is not None (e.g., OPT-350m), we embed to that dimension
      the project up to embed_dim
docstr_tok: ["If","max_position_embeddings","<","=","0",",","there","'s","no","position","embeddings","If","word_embe_proj_dim","is","not","None","(","e.g.",",","OPT-350m",")",",","we","embed","to","that","dimension","the","project","up","to","embed_dim"]
code: def __init__(
      self,
      embed_dim,
      vocab_size,
      max_position_embeddings,
      padding_idx=None,
      word_embed_proj_dim=None,
      device=None,
      dtype=None,
  ):
      """
      If max_position_embeddings <= 0, there's no position embeddings
      If word_embe_proj_dim is not None (e.g., OPT-350m), we embed to that dimension
          the project up to embed_dim
      """
      factory_kwargs = {"device": device, "dtype": dtype}
      super().__init__()
      if word_embed_proj_dim is None:
          self.word_embeddings = nn.Embedding(
              vocab_size,
              embed_dim,
              padding_idx=padding_idx,
              **factory_kwargs,
          )
          self.project_in = None
      else:
          self.word_embeddings = nn.Embedding(
              vocab_size,
              word_embed_proj_dim,
              padding_idx=padding_idx,
              **factory_kwargs,
          )
          self.project_in = nn.Linear(
              word_embed_proj_dim,
              embed_dim,
              bias=False,
              **factory_kwargs,
          )
      self.max_position_embeddings = max_position_embeddings
      if self.max_position_embeddings > 0:
          self.position_embeddings = nn.Embedding(
              max_position_embeddings, embed_dim, **factory_kwargs
          )
  
imports: import torch
  import torch.nn
  from einops import rearrange
  from torch import Tensor
  from flash_attn.utils.distributed import all_reduce, reduce_scatter
label_desc: Use image node_id 1 to create a new GPT2Embeddings object from inherited base classes: nn with example: obj = GPT2Embeddings(embed_dim, vocab_size, max_position_embeddings, padding_idx, word_embed_proj_dim, device, dtype)