class_docstr: This is a PyTorch implementation of the paper Disentangled Representation Learning for Text-Video Retrieval.
  Args:
      base_encoder (`str`):
          CLIP encoder backbone. default: `clip_vit_b32`
      agg_module (`str`):
          Feature aggregation module for video. default: `seqTransf`, choices=[`ndone`, `seqLSTM`, `seqTransf`]
      interaction (`str`):
          Interaction type for retrieval. default: `wti`.
      wti_arch (`int)`:
          Select a architecture for weight branch. default: 2.
      cdcr (`int`):
          Channel decorrelation regularization. default: 3.
      cdcr_alpha1 (`float`):
          Coefficient 1 for channel decorrelation regularization. default: 1.0.
      cdcr_alpha2 (`float`):
          Coefficient 2 for channel decorrelation regularization. default: 0.06.
      cdcr_lambda (`float`):
          Coefficient for channel decorrelation regularization. default: 0.001.
      cross_num_hidden_layers (`int`):
          Number of hidden layers for cross transformer interaction.
class_docstr_tok: ["This","is","a","PyTorch","implementation","of","the","paper","Disentangled","Representation","Learning","for","Text-Video","Retrieval",".","Args",":","base_encoder","(","`","str","`",")",":","CLIP","encoder","backbone",".","default",":","`","clip_vit_b32","`","agg_module","(","`","str","`",")",":","Feature","aggregation","module","for","video",".","default",":","`","seqTransf","`",",","choices=","[","`","ndone","`",",","`","seqLSTM","`",",","`","seqTransf","`","]","interaction","(","`","str","`",")",":","Interaction","type","for","retrieval",".","default",":","`","wti","`",".","wti_arch","(","`","int",")","`",":","Select","a","architecture","for","weight","branch",".","default",":","2.","cdcr","(","`","int","`",")",":","Channel","decorrelation","regularization",".","default",":","3.","cdcr_alpha1","(","`","float","`",")",":","Coefficient","1","for","channel","decorrelation","regularization",".","default",":","1.0.","cdcr_alpha2","(","`","float","`",")",":","Coefficient","2","for","channel","decorrelation","regularization",".","default",":","0.06.","cdcr_lambda","(","`","float","`",")",":","Coefficient","for","channel","decorrelation","regularization",".","default",":","0.001.","cross_num_hidden_layers","(","`","int","`",")",":","Number","of","hidden","layers","for","cross","transformer","interaction","."]
code: def __init__(
      self,
      base_encoder: str = "clip_vit_b32",
      agg_module: str = "seqTransf",
      interaction: str = "wti",
      wti_arch: int = 2,
      cdcr: int = 3,
      cdcr_alpha1: float = 1.0,
      cdcr_alpha2: float = 0.06,
      cdcr_lambda: float = 0.001,
      cross_num_hidden_layers: Optional[int] = None,
      backbone_pretrained: bool = False,
  ):
      super().__init__()
      self.base_encoder = base_encoder
      self.agg_module = agg_module
      self.interaction = interaction
      self.wti_arch = wti_arch
      self.cdcr = cdcr
      self.cdcr_alpha1 = cdcr_alpha1
      self.cdcr_alpha2 = cdcr_alpha2
      self.cdcr_lambda = cdcr_lambda
  
      self.agg_module = agg_module
      backbone = base_encoder
  
      self.clip = clip.create_model(
          model_name=backbone,
          pretrained=backbone_pretrained,
          jit=False,
          clip4clip=True,
      )
  
      state_dict = self.clip.state_dict()
      context_length = state_dict["positional_embedding"].shape[0]
      transformer_width = state_dict["ln_final.weight"].shape[0]
      transformer_heads = transformer_width // 64
  
      if torch.cuda.is_available():
          convert_weights(self.clip)  # fp16
  
      cross_config = SimpleNamespace(
          **{
              "hidden_dropout_prob": 0.1,
              "hidden_size": 512,
              "max_position_embeddings": 128,
              "num_attention_heads": 8,
              "num_hidden_layers": 4,
              "vocab_size": 512,
          }
      )
      cross_config.max_position_embeddings = context_length
      cross_config.hidden_size = transformer_width
      self.cross_config = cross_config
      if self.interaction == "xti":
          if cross_num_hidden_layers is not None:
              setattr(
                  cross_config,
                  "num_hidden_layers",
                  cross_num_hidden_layers,
              )
  
          self.cross = CrossModel(cross_config)
          self.similarity_dense = nn.Linear(cross_config.hidden_size, 1)
      elif self.interaction == "mlp":
          self.similarity_dense = nn.Sequential(
              nn.Linear(transformer_width * 2, transformer_width),
              nn.ReLU(inplace=True),
              nn.Linear(transformer_width, 1),
          )
      elif self.interaction == "wti":
          if self.wti_arch == 1:
              self.text_weight_fc = nn.Linear(transformer_width, 1)
              self.video_weight_fc = nn.Linear(transformer_width, 1)
          elif self.wti_arch == 2:
              self.text_weight_fc = nn.Sequential(
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, 1),
              )
              self.video_weight_fc = nn.Sequential(
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, 1),
              )
          elif self.wti_arch == 3:
              self.text_weight_fc = nn.Sequential(
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, 1),
              )
              self.video_weight_fc = nn.Sequential(
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, transformer_width),
                  nn.ReLU(inplace=True),
                  nn.Linear(transformer_width, 1),
              )
  
      if self.agg_module in ["seqLSTM", "seqTransf"]:
          self.frame_position_embeddings = nn.Embedding(
              cross_config.max_position_embeddings,
              cross_config.hidden_size,
          )
          if self.agg_module == "seqTransf":
              self.transformerClip = TransformerClip(
                  width=transformer_width,  # pylint: disable=invalid-name
                  layers=cross_config.num_hidden_layers,
                  heads=transformer_heads,
              )
          if self.agg_module == "seqLSTM":
              self.lstm_visual = nn.LSTM(
                  input_size=cross_config.hidden_size,
                  hidden_size=cross_config.hidden_size,
                  batch_first=True,
                  bidirectional=False,
                  num_layers=1,
              )
  
      self.loss_fct = CrossEn()
      self.apply(
          self.init_weights
      )  # random init must before loading pretrain
  
      # ===> Initialization trick [HARdd COddE]
      new_state_dict = OrderedDict()
      if self.interaction == "xti":
          contain_cross = False
          for key in state_dict.keys():
              if key.find("cross.transformer") > -1:
                  contain_cross = True
                  break
          if contain_cross is False:
              for key, val in state_dict.items():
                  if key == "positional_embedding":
                      new_state_dict[
                          "cross.embeddings.position_embeddings.weight"
                      ] = val.clone()
                      continue
                  if key.find("transformer.resblocks") == 0:
                      num_layer = int(key.split(".")[2])
  
                      # cut from beginning
                      if num_layer < cross_config.num_hidden_layers:
                          new_state_dict["cross." + key] = val.clone()
                          continue
  
      if self.agg_module in ["seqLSTM", "seqTransf"]:
          contain_frame_position = False
          for key in state_dict.keys():
              if key.find("frame_position_embeddings") > -1:
                  contain_frame_position = True
                  break
          if contain_frame_position is False:
              for key, val in state_dict.items():
                  if key == "positional_embedding":
                      new_state_dict[
                          "frame_position_embeddings.weight"
                      ] = val.clone()
                      continue
                  if (
                      self.agg_module in ["seqTransf"]
                      and key.find("transformer.resblocks") == 0
                  ):
                      num_layer = int(key.split(".")[2])
                      # cut from beginning
                      if num_layer < cross_config.num_hidden_layers:
                          new_state_dict[
                              key.replace(
                                  "transformer.", "transformerClip."
                              )
                          ] = val.clone()
                          continue
  
      self.load_state_dict(new_state_dict, strict=False)
  
imports: from collections import OrderedDict
  from typing import Optional
  from types import SimpleNamespace
  import torch
  import logging
  from torch import nn
  from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence
  from towhee.models import clip
  from towhee.models.drl.until_module import convert_weights
  from towhee.models.drl.module_cross import CrossModel, Transformer
  from towhee.models.drl.until_module import LayerNorm, AllGather, AllGather2, CrossEn
label_desc: Use image node_id 1 to create a new DRL object from inherited base classes: nn with example: obj = DRL(base_encoder, agg_module, interaction, wti_arch, cdcr, cdcr_alpha1, cdcr_alpha2, cdcr_lambda, cross_num_hidden_layers, backbone_pretrained)