code: def _split_heads(
      fused_qkv: torch.Tensor, num_heads: int, head_dim: int
  ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
      """
      Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory
      storage as `fused_qkv`
  
      Args:
          fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim]
  
      Returns:
          query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
          value: [batch_size, seq_length, num_heads, head_dim]
      """
      batch_size, seq_length, three_times_hidden_size = fused_qkv.shape
      fused_qkv = fused_qkv.view(
          batch_size, seq_length, num_heads, 3 * head_dim
      )
      query_layer, key_layer, value_layer = fused_qkv.split(
          head_dim, dim=-1
      )
  
      query_layer = query_layer.transpose(1, 2).reshape(
          batch_size * num_heads, seq_length, head_dim
      )
      key_layer = key_layer.permute(0, 2, 3, 1).reshape(
          batch_size * num_heads, head_dim, seq_length
      )
      value_layer = value_layer.transpose(1, 2).reshape(
          batch_size * num_heads, seq_length, head_dim
      )
  
      return query_layer, key_layer, value_layer
  
imports: import math
  import os
  import warnings
  from typing import Optional, Tuple, Union
  import torch
  import torch.distributed
  import torch.utils.checkpoint
  from torch import nn
  from torch.nn import LayerNorm
  from torch.nn import functional
  from transformers.modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions
  from transformers import BloomConfig, PreTrainedModel
  from text_generation_server.utils.layers import TensorParallelColumnLinear, TensorParallelEmbedding, TensorParallelRowLinear, TensorParallelHead
label_desc: Use image node_id 5 for calling a global function with example usage: _split_heads(fused_qkv, num_heads, head_dim) and returns: query_layer,  key_layer,  value_layer