code: def test_inproj(
      proj_bias: bool, same_sizes: bool, same_settings: bool
  ):
      test_config = {
          "name": "scaled_dot_product",
          "dropout": 0.1,
          "causal": False,
          "seq_len": SEQ,
          "window_size": SEQ // 8 + 1,
          "num_heads": 1,
          "dim_head": MODEL,
      }
  
      attention = build_attention(test_config)
  
      # Construct the initial projection, test different options
      in_params = InputProjectionConfig(MODEL, MODEL, proj_bias)
  
      if same_settings:
          in_proj = InputProjection(in_params, None, None)
          out_features = MODEL
      else:
          out_features = MODEL if same_sizes else MODEL // 2
          in_params_flip = InputProjectionConfig(
              MODEL, out_features, proj_bias
          )
          in_proj = InputProjection(
              in_params_flip,  # Q proj
              in_params_flip,  # K proj
              in_params,  # V proj
          )
  
      # build a multi head dispatch to test this attention mechanism
      multi_head = MultiHeadDispatch(
          seq_len=SEQ,
          dim_model=MODEL,
          residual_dropout=0.1,
          num_heads=1,
          attention=attention,
          in_proj_container=in_proj,
          dim_key=out_features,
          dim_value=MODEL,
      )
  
      # Check kqv are not flipped
      # this will not catch all issues, but would catch a V being misplaced
      # make k and q complimentary, so that QKt is all zero and attention is uniform
  
      q = torch.cat(
          (
              torch.rand((1, MODEL // 2)),
              torch.zeros((1, MODEL // 2)),
          ),
          dim=1,
      ).expand((BATCH, SEQ, MODEL))
  
      k = torch.cat(
          (
              torch.zeros((1, MODEL // 2)),
              torch.rand((1, MODEL // 2)),
          ),
          dim=1,
      ).expand((BATCH, SEQ, MODEL))
      v = torch.rand(BATCH, SEQ, MODEL)
  
      # just check that a FW does not assert out
      _ = multi_head(query=q, key=k, value=v)
  
file_path: xformers/tests/test_attentions.py
imports: import math
  from typing import Tuple
  import pytest
  import torch
  from xformers.components import InputProjection, InputProjectionConfig, MultiHeadDispatch
  from xformers.components.attention import _DENSITY_THRESHOLD, ATTENTION_REGISTRY, build_attention
label_desc: Use image node_id 5 for calling a global function with example usage: test_inproj(proj_bias, same_sizes, same_settings) without return types