class_docstr: Gated positional self-attention. This is what makes a ConViT a ConViT.
  The "gating" refers to lambda, the parameter that controls the relative
  contribution (i.e. gating) of positional vs. content-based attention in
  each head.
class_docstr_tok: ["Gated","positional","self-attention",".","This","is","what","makes","a","ConViT","a","ConViT",".","The","``","gating","''","refers","to","lambda",",","the","parameter","that","controls","the","relative","contribution","(","i.e",".","gating",")","of","positional","vs.","content-based","attention","in","each","head","."]
code: def __init__(
      self,
      embed_dim,  # Hidden dimensionality
      num_heads=9,  # Number of attention heads. Should be a square number
      qkv_bias=False,  # Query, key, and value bias
      qk_scale=None,  # Scale query and key
      attention_dropout_rate=0.0,  # Dropout on attention
      proj_dropout_rate=0.0,  # Dropout on linear projection after
      # attention
      locality_strength=1.0,  # Determines how much the positional
      # attention is focused on the patch of maximal attention. "Alpha"
      # in the paper. Equivalent to the temperature of positional
      # attention softmax. When alpha is large, attention is focused
      # only on local patches; when alpha is small, the attention is
      # spread out into a larger area
      locality_dim=10,  # Dimensionality of position embeddings. "D_pos"
      # in the paper
      use_local_init=True,  # Local initialization. This is what makes
      # a convit convolutional.
  ):
      super().__init__()
      self.num_heads = num_heads
      self.dim = embed_dim
      head_dim = embed_dim // num_heads
      self.scale = qk_scale or head_dim**-0.5
  
      self.qk = nn.Linear(embed_dim, embed_dim * 2, bias=qkv_bias)
      self.v = nn.Linear(embed_dim, embed_dim, bias=qkv_bias)
  
      self.attn_drop = nn.Dropout(attention_dropout_rate)
      self.proj = nn.Linear(embed_dim, embed_dim)
      self.pos_proj = nn.Linear(3 * locality_dim, num_heads)
      self.proj_drop = nn.Dropout(proj_dropout_rate)
      self.locality_strength = locality_strength
      self.locality_dim = locality_dim
      self.alpha = nn.Parameter(torch.ones(1, self.num_heads, 1, 1))
      self.apply(self._init_weights)
      if use_local_init:
          self.init_local(locality_strength=locality_strength)
  
imports: import copy
  import logging
  import math
  import torch
  import torch.nn
  from vissl.models.model_helpers import DropPath, to_2tuple, trunc_normal_
  from vissl.models.trunks import register_model_trunk
  from vissl.utils.hydra_config import AttrDict
label_desc: Use image node_id 1 to create a new GPSA object from inherited base classes: nn with example: obj = GPSA(embed_dim, num_heads, qkv_bias, qk_scale, attention_dropout_rate, proj_dropout_rate, locality_strength, locality_dim, use_local_init)