class_docstr: This class implements multi-head HyperMixing.
  It is an implementation of the token-mixing component in HyperMixer, a linear
  time drop-in replacement for self-attention. In contrast to the original HyperMixer,
  this module supports multiple heads, which improves the expressiveness of the model
  while decreasing the number of parameters.
  
  Reference: https://arxiv.org/abs/2203.03691
  
  Arguments
  ----------
  input_output_dim : int
      number of features in keys, queries, and values
  hypernet_size : int
      determines the size of the hidden layer of the token-mixing MLP.
  tied : bool
      If True, then the generated weight matrices of the token-mixing MLP are tied.
  num_heads : int
      parallel token-mixing MLPs.
  fix_tm_hidden_size : bool
      If True, the hidden-layer size is equal to hypernet_size rather than hypernet_size / num_heads.
  max_length : int
      Maximum number of input tokens. Needed for generating sufficiently large position embeddings.
  
  Example
  -------
  >>> import torch
  >>> inputs = torch.rand([8, 60, 512])
  >>> net = HyperMixing(512, 2048, num_heads=8)
  >>> outputs, attn = net(inputs, inputs, inputs)
  >>> outputs.shape
  torch.Size([8, 60, 512])
class_docstr_tok: ["This","class","implements","multi-head","HyperMixing",".","It","is","an","implementation","of","the","token-mixing","component","in","HyperMixer",",","a","linear","time","drop-in","replacement","for","self-attention",".","In","contrast","to","the","original","HyperMixer",",","this","module","supports","multiple","heads",",","which","improves","the","expressiveness","of","the","model","while","decreasing","the","number","of","parameters",".","Reference",":","https",":","\/\/arxiv.org\/abs\/2203.03691","Arguments","--","--","--","--","--","input_output_dim",":","int","number","of","features","in","keys",",","queries",",","and","values","hypernet_size",":","int","determines","the","size","of","the","hidden","layer","of","the","token-mixing","MLP",".","tied",":","bool","If","True",",","then","the","generated","weight","matrices","of","the","token-mixing","MLP","are","tied",".","num_heads",":","int","parallel","token-mixing","MLPs",".","fix_tm_hidden_size",":","bool","If","True",",","the","hidden-layer","size","is","equal","to","hypernet_size","rather","than","hypernet_size","\/","num_heads",".","max_length",":","int","Maximum","number","of","input","tokens",".","Needed","for","generating","sufficiently","large","position","embeddings",".","Example","--","--","--","-",">",">",">","import","torch",">",">",">","inputs","=","torch.rand","(","[","8",",","60",",","512","]",")",">",">",">","net","=","HyperMixing","(","512",",","2048",",","num_heads=8",")",">",">",">","outputs",",","attn","=","net","(","inputs",",","inputs",",","inputs",")",">",">",">","outputs.shape","torch.Size","(","[","8",",","60",",","512","]",")"]
code: def __init__(
      self,
      input_output_dim: int,
      hypernet_size: int,
      tied: bool = False,
      num_heads: int = 1,
      fix_tm_hidden_size=False,
      max_length=3000,
  ) -> None:
      super().__init__()
      self.input_output_dim = input_output_dim
      self.hyper = HyperNetwork(
          input_output_dim,
          hypernet_size,
          tied=tied,
          num_heads=num_heads,
          keep_output_size=fix_tm_hidden_size,
      )
      self.activation = nn.GELU()
      self.layer_norm = nn.LayerNorm(input_output_dim)
      self.num_heads = num_heads
  
      from speechbrain.lobes.models.transformer.Transformer import (
          PositionalEncoding,
      )
  
      # add pos encoding
      self.positional_encoding = PositionalEncoding(
          input_output_dim, max_length
      )
  
file_path: speechbrain/speechbrain/lobes/models/transformer/hypermixing.py
imports: from typing import Optional
  import math
  import torch
  from torch import nn
label_desc: Use image node_id 1 to create a new HyperMixing object from inherited base classes: nn with example: obj = HyperMixing(input_output_dim, hypernet_size, tied, num_heads, fix_tm_hidden_size, max_length)