class_docstr: Implement the scaled dot product attention with softmax.
  Arguments
  ---------
      softmax_scale: The temperature to use for the softmax attention.
                    (default: 1/sqrt(d_keys) where d_keys is computed at
                    runtime)
      attention_dropout: The dropout rate to apply to the attention
                         (default: 0.0)
class_docstr_tok: ["Implement","the","scaled","dot","product","attention","with","softmax",".","Arguments","--","--","--","--","-","softmax_scale",":","The","temperature","to","use","for","the","softmax","attention",".","(","default",":","1\/sqrt","(","d_keys",")","where","d_keys","is","computed","at","runtime",")","attention_dropout",":","The","dropout","rate","to","apply","to","the","attention","(","default",":","0.0",")"]
code: def __init__(
      self,
      causal=False,
      softmax_scale=None,
      attention_dropout=0.0,
      alibi_slopes=None,
      deterministic=False,
  ):
      super().__init__()
      assert (
          flash_attn_varlen_qkvpacked_func is not None
      ), "FlashAttention is not installed"
      assert (
          flash_attn_qkvpacked_func is not None
      ), "FlashAttention is not installed"
      self.causal = causal
      self.softmax_scale = softmax_scale
      self.drop = nn.Dropout(attention_dropout)
      self.register_buffer(
          "alibi_slopes", alibi_slopes, persistent=False
      )
      self.deterministic = deterministic
  
imports: import math
  from functools import partial
  import torch
  import torch.nn
  from einops import rearrange, repeat
  from flash_attn.utils.distributed import get_dim_for_local_rank
label_desc: Use image node_id 1 to create a new FlashSelfAttention object from inherited base classes: nn with example: obj = FlashSelfAttention(causal, softmax_scale, attention_dropout, alibi_slopes, deterministic)