class_docstr: ### Cross Attention Layer
  
  This falls-back to self-attention when conditional embeddings are not specified.
class_docstr_tok: ["#","#","#","Cross","Attention","Layer","This","falls-back","to","self-attention","when","conditional","embeddings","are","not","specified","."]
docstr: :param d_model: is the input embedding size
  :param n_heads: is the number of attention heads
  :param d_head: is the size of a attention head
  :param d_cond: is the size of the conditional embeddings
  :param is_inplace: specifies whether to perform the attention softmax computation inplace to
      save memory
docstr_tok: [":","param","d_model",":","is","the","input","embedding","size",":","param","n_heads",":","is","the","number","of","attention","heads",":","param","d_head",":","is","the","size","of","a","attention","head",":","param","d_cond",":","is","the","size","of","the","conditional","embeddings",":","param","is_inplace",":","specifies","whether","to","perform","the","attention","softmax","computation","inplace","to","save","memory"]
code: def __init__(
      self,
      d_model: int,
      d_cond: int,
      n_heads: int,
      d_head: int,
      is_inplace: bool = True,
  ):
      """
      :param d_model: is the input embedding size
      :param n_heads: is the number of attention heads
      :param d_head: is the size of a attention head
      :param d_cond: is the size of the conditional embeddings
      :param is_inplace: specifies whether to perform the attention softmax computation inplace to
          save memory
      """
      super().__init__()
  
      self.is_inplace = is_inplace
      self.n_heads = n_heads
      self.d_head = d_head
  
      # Attention scaling factor
      self.scale = d_head**-0.5
  
      # Query, key and value mappings
      d_attn = d_head * n_heads
      self.to_q = nn.Linear(d_model, d_attn, bias=False)
      self.to_k = nn.Linear(d_cond, d_attn, bias=False)
      self.to_v = nn.Linear(d_cond, d_attn, bias=False)
  
      # Final linear layer
      self.to_out = nn.Sequential(nn.Linear(d_attn, d_model))
  
      # Setup [flash attention](https://github.com/HazyResearch/flash-attention).
      # Flash attention is only used if it's installed
      # and `CrossAttention.use_flash_attention` is set to `True`.
      try:
          # You can install flash attention by cloning their Github repo,
          # [https://github.com/HazyResearch/flash-attention](https://github.com/HazyResearch/flash-attention)
          # and then running `python setup.py install`
          from flash_attn.flash_attention import FlashAttention
  
          self.flash = FlashAttention()
          # Set the scale for scaled dot-product attention.
          self.flash.softmax_scale = self.scale
      # Set to `None` if it's not installed
      except ImportError:
          self.flash = None
  
imports: from typing import Optional
  import torch
  import torch.nn.functional
  from torch import nn
label_desc: Use image node_id 1 to create a new CrossAttention object from inherited base classes: nn with example: obj = CrossAttention(d_model, d_cond, n_heads, d_head, is_inplace)