docstr: A general constructor for neural modules fusing two equal sized tensors
  in forward. Following options are supported:
  
  "add" / "max" / "min" / "avg"             : respective operations on the two halves.
  "concat"                                  : NOOP.
  "concat_linear_{dim_mult}_{drop_rate}"    : MLP to fuse with hidden dim "dim_mult"
                                              (optional, def 1.) higher than input dim
                                              with optional dropout "drop_rate" (def: 0.)
  "ln+concat_linear_{dim_mult}_{drop_rate}" : perform MLP after layernorm on the input.
docstr_tok: ["A","general","constructor","for","neural","modules","fusing","two","equal","sized","tensors","in","forward",".","Following","options","are","supported",":","''","add","''","\/","``","max","''","\/","``","min","''","\/","``","avg","''",":","respective","operations","on","the","two","halves",".","``","concat","''",":","NOOP",".","``","concat_linear_","{","dim_mult","}","_","{","drop_rate","}","''",":","MLP","to","fuse","with","hidden","dim","``","dim_mult","''","(","optional",",","def","1",".",")","higher","than","input","dim","with","optional","dropout","``","drop_rate","''","(","def",":","0",".",")","``","ln+concat_linear_","{","dim_mult","}","_","{","drop_rate","}","''",":","perform","MLP","after","layernorm","on","the","input","."]
code: def __init__(self, mode, dim=None, kernel=3, padding=1):
      """
      A general constructor for neural modules fusing two equal sized tensors
      in forward. Following options are supported:
  
      "add" / "max" / "min" / "avg"             : respective operations on the two halves.
      "concat"                                  : NOOP.
      "concat_linear_{dim_mult}_{drop_rate}"    : MLP to fuse with hidden dim "dim_mult"
                                                  (optional, def 1.) higher than input dim
                                                  with optional dropout "drop_rate" (def: 0.)
      "ln+concat_linear_{dim_mult}_{drop_rate}" : perform MLP after layernorm on the input.
  
      """
      super().__init__()
      self.mode = mode
      if mode == "add":
          self.fuse_fn = lambda x: torch.stack(
              torch.chunk(x, 2, dim=2)
          ).sum(dim=0)
      elif mode == "max":
          self.fuse_fn = (
              lambda x: torch.stack(torch.chunk(x, 2, dim=2))
              .max(dim=0)
              .values
          )
      elif mode == "min":
          self.fuse_fn = (
              lambda x: torch.stack(torch.chunk(x, 2, dim=2))
              .min(dim=0)
              .values
          )
      elif mode == "avg":
          self.fuse_fn = lambda x: torch.stack(
              torch.chunk(x, 2, dim=2)
          ).mean(dim=0)
      elif mode == "concat":
          # x itself is the channel concat version
          self.fuse_fn = lambda x: x
      elif "concat_linear" in mode:
          if len(mode.split("_")) == 2:
              dim_mult = 1.0
              drop_rate = 0.0
          elif len(mode.split("_")) == 3:
              dim_mult = float(mode.split("_")[-1])
              drop_rate = 0.0
  
          elif len(mode.split("_")) == 4:
              dim_mult = float(mode.split("_")[-2])
              drop_rate = float(mode.split("_")[-1])
          else:
              raise NotImplementedError
  
          if mode.split("+")[0] == "ln":
              self.fuse_fn = nn.Sequential(
                  nn.LayerNorm(dim),
                  Mlp(
                      in_features=dim,
                      hidden_features=int(dim * dim_mult),
                      act_layer=nn.GELU,
                      out_features=dim,
                      drop_rate=drop_rate,
                  ),
              )
          else:
              self.fuse_fn = Mlp(
                  in_features=dim,
                  hidden_features=int(dim * dim_mult),
                  act_layer=nn.GELU,
                  out_features=dim,
                  drop_rate=drop_rate,
              )
  
      else:
          raise NotImplementedError
  
file_path: SlowFast/slowfast/models/common.py
imports: import torch
  import torch.nn
label_desc: Use image node_id 1 to create a new TwoStreamFusion object from inherited base classes: nn with example: obj = TwoStreamFusion(mode, dim, kernel, padding)