docstr: Self-attention module with q,k,v from the same input.
  
  Parameters
  ----------
  d_hidden : int
      hidden dimension
  kernel_sizes: int
      kernel sizes of convolutions to generate queries and keys
  n_head : int, optional
      number of attention heads, by default 1
  bias : bool, optional
      add bias term in input and output projections, by default True
  bidirectional : bool, optional
      if False, add a mask to avoid backward attention, by default False
  dist_enc : Optional[str], optional
      add relative distance embeddings to dot-product attention, can be
          'add' (linearly combine key and dist),
          'dot' (dot product between key and dist),
          or None (disabled),
      by default None
  share_values : bool, optional
      if True, a value reprensentation is shared by all attention heads,
      by default False
      ref. https://arxiv.org/abs/1912.09363
  dropout : float, optional
      dropout rate, by default 0.0
  temperature : float, optional
      softmax temperature, by default 1.0
docstr_tok: ["Self-attention","module","with","q",",","k",",","v","from","the","same","input",".","Parameters","--","--","--","--","--","d_hidden",":","int","hidden","dimension","kernel_sizes",":","int","kernel","sizes","of","convolutions","to","generate","queries","and","keys","n_head",":","int",",","optional","number","of","attention","heads",",","by","default","1","bias",":","bool",",","optional","add","bias","term","in","input","and","output","projections",",","by","default","True","bidirectional",":","bool",",","optional","if","False",",","add","a","mask","to","avoid","backward","attention",",","by","default","False","dist_enc",":","Optional","[","str","]",",","optional","add","relative","distance","embeddings","to","dot-product","attention",",","can","be","'add","'","(","linearly","combine","key","and","dist",")",",","'dot","'","(","dot","product","between","key","and","dist",")",",","or","None","(","disabled",")",",","by","default","None","share_values",":","bool",",","optional","if","True",",","a","value","reprensentation","is","shared","by","all","attention","heads",",","by","default","False","ref",".","https",":","\/\/arxiv.org\/abs\/1912.09363","dropout",":","float",",","optional","dropout","rate",",","by","default","0.0","temperature",":","float",",","optional","softmax","temperature",",","by","default","1.0"]
code: def __init__(
      self,
      d_hidden: int,
      kernel_sizes: List[int],
      n_head: int = 1,
      bias: bool = True,
      bidirectional: bool = False,
      dist_enc: Optional[str] = None,
      share_values: bool = False,
      dropout: float = 0.0,
      temperature: float = 1.0,
      **kwargs,
  ):
      """
      Self-attention module with q,k,v from the same input.
  
      Parameters
      ----------
      d_hidden : int
          hidden dimension
      kernel_sizes: int
          kernel sizes of convolutions to generate queries and keys
      n_head : int, optional
          number of attention heads, by default 1
      bias : bool, optional
          add bias term in input and output projections, by default True
      bidirectional : bool, optional
          if False, add a mask to avoid backward attention, by default False
      dist_enc : Optional[str], optional
          add relative distance embeddings to dot-product attention, can be
              'add' (linearly combine key and dist),
              'dot' (dot product between key and dist),
              or None (disabled),
          by default None
      share_values : bool, optional
          if True, a value reprensentation is shared by all attention heads,
          by default False
          ref. https://arxiv.org/abs/1912.09363
      dropout : float, optional
          dropout rate, by default 0.0
      temperature : float, optional
          softmax temperature, by default 1.0
      """
      super().__init__(**kwargs)
      n_groups = len(kernel_sizes)
      assert (
          d_hidden % n_head == 0
      ), f"hidden dim {d_hidden} cannot be split into {n_head} heads."
      assert (
          d_hidden % n_groups == 0
      ), f"hidden dim {d_hidden} cannot be split into {n_groups} groups."
      assert (
          n_head % n_groups == 0
      ), f"num_heads {n_head} cannot be allocated for {n_groups} groups."
      self.d_hidden = d_hidden
      self.kernel_sizes = kernel_sizes
      self.n_groups = n_groups
      self.d_group = self.d_hidden // self.n_groups
      self.n_head = n_head
      self.d_head = self.d_hidden // self.n_head
      self.bias = bias
      self.dist_enc = dist_enc
      self.bidirectional = bidirectional
      self.share_values = share_values
      self.temperature = temperature
  
      with self.name_scope():
          self.qk_proj = HybridConcurrent(axis=-1, prefix="qk_proj_")
          for ksize in self.kernel_sizes:
              self.qk_proj.add(
                  CausalConv1D(
                      channels=self.d_group * 2,
                      kernel_size=ksize,
                      prefix=f"conv{ksize}_",
                  )
              )
          self.v_proj = nn.Dense(
              units=self.d_head if self.share_values else d_hidden,
              use_bias=bias,
              flatten=False,
              weight_initializer=init.Xavier(),
              prefix="v_proj_",
          )
          self.out_proj = nn.Dense(
              units=d_hidden,
              use_bias=bias,
              flatten=False,
              weight_initializer=init.Xavier(),
              prefix="out_proj_",
          )
  
          if self.dist_enc is not None:
              assert self.dist_enc in [
                  "dot",
                  "add",
              ], f"distance encoding type {self.dist_enc} is not supported"
              self.posemb = SinusoidalPositionalEmbedding(d_hidden)
              self.pos_proj = nn.Dense(
                  units=d_hidden,
                  use_bias=bias,
                  flatten=False,
                  weight_initializer=init.Xavier(),
                  prefix="pos_proj_",
              )
              if self.dist_enc == "add":
                  self._ctt_bias_weight = Parameter(
                      "_ctt_bias_weight",
                      shape=(1, n_head, 1, self.d_head),
                      init=init.Xavier(),
                  )
                  self._pos_bias_weight = Parameter(
                      "_pos_bias_weight",
                      shape=(1, n_head, 1, self.d_head),
                      init=init.Xavier(),
                  )
  
          self.dropout = nn.Dropout(dropout)
  
imports: import math
  from typing import List, Optional, Tuple
  from mxnet import init
  from mxnet.gluon import HybridBlock, Parameter, nn
  from mxnet.gluon.contrib.nn import HybridConcurrent
  from gluonts.core.component import validated
  from gluonts.mx import Tensor
label_desc: Use image node_id 1 to create a new SelfAttention object from inherited base classes: HybridBlock with example: obj = SelfAttention(d_hidden, kernel_sizes, n_head, bias, bidirectional, dist_enc, share_values, dropout, temperature)