class_docstr: CoCa model.
  
  Args:
      dim (`int`):
          model dimension
      img_encoder (`nn.Module`):
          vision transformer - image encoder, returning image embeddings as (batch, seq, dim).
      image_dim (`int`):
          image embedding dimension, if not the same as model dimensions.
      num_tokens (`int`):
          number of text tokens.
      unimodal_depth (`int`):
          depth of the unimodal transformer.
      multimodal_depth (`int`):
          depth of the multimodal transformer.
      dim_head (`int`):
          dimension per attention head.
      heads (`int`):
          number of attention heads.
      caption_loss_weight (`float`):
          weight on the autoregressive caption loss.
      contrastive_loss_weight (`float`):
          weight on the contrastive loss between image and text CLS embeddings.
class_docstr_tok: ["CoCa","model",".","Args",":","dim","(","`","int","`",")",":","model","dimension","img_encoder","(","`","nn.Module","`",")",":","vision","transformer","-","image","encoder",",","returning","image","embeddings","as","(","batch",",","seq",",","dim",")",".","image_dim","(","`","int","`",")",":","image","embedding","dimension",",","if","not","the","same","as","model","dimensions",".","num_tokens","(","`","int","`",")",":","number","of","text","tokens",".","unimodal_depth","(","`","int","`",")",":","depth","of","the","unimodal","transformer",".","multimodal_depth","(","`","int","`",")",":","depth","of","the","multimodal","transformer",".","dim_head","(","`","int","`",")",":","dimension","per","attention","head",".","heads","(","`","int","`",")",":","number","of","attention","heads",".","caption_loss_weight","(","`","float","`",")",":","weight","on","the","autoregressive","caption","loss",".","contrastive_loss_weight","(","`","float","`",")",":","weight","on","the","contrastive","loss","between","image","and","text","CLS","embeddings","."]
code: def embed_text(self, text):
      batch, _ = text.shape[0], text.device
  
      seq = text.shape[1]
  
      text_tokens = self.token_emb(text)
  
      # append text cls tokens
  
      text_cls_tokens = repeat(
          self.text_cls_token, "d -> b 1 d", b=batch
      )
      text_tokens = torch.cat((text_tokens, text_cls_tokens), dim=-2)
  
      # create specific mask for text cls token at the end
      # to prevent it from attending to padding
  
      cls_mask = rearrange(text != self.pad_id, "b j -> b 1 j")
      attn_mask = F.pad(cls_mask, (0, 1, seq, 0), value=True)
  
      # go through unimodal layers
  
      for attn_ff in self.unimodal_layers:
          text_tokens = attn_ff(text_tokens, attn_mask=attn_mask)
  
      # get text cls token
  
      text_tokens, text_cls_tokens = (
          text_tokens[:, :-1],
          text_tokens[:, -1],
      )
      text_embeds = self.text_cls_norm(text_cls_tokens)
      return text_embeds, text_tokens
  
file_path: towhee/towhee/models/coca/coca.py
imports: import torch
  from torch import einsum, nn
  import torch.nn.functional
  from einops import rearrange, repeat
  from towhee.models.layers.cross_attention import LayerNorm, Residual, RotaryEmbedding, apply_rotary_pos_emb, CrossAttention
  from towhee.models.layers.activations.swiglu import SwiGLU
label_desc: Use image node_id 2 for calling the CoCa obj's underlying member method code with example usage: obj.embed_text(text) and returns: text_embeds,  text_tokens