docstr: Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size,
  num_heads, ...]))
docstr_tok: ["Standardizes","the","format","of","the","cache","so","as","to","match","most","implementations",",","i.e",".","to","tuple","(","tuple","(","[","batch_size",",","num_heads",",","...","]",")",")"]
code: def _convert_to_standard_cache(
      past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]],
      batch_size: int,
  ) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]:
      """
      Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size,
      num_heads, ...]))
      """
      batch_size_times_num_heads, head_dim, seq_length = past_key_value[
          0
      ][0].shape
      num_heads = batch_size_times_num_heads // batch_size
      # key: [batch_size * num_heads, head_dim, seq_length] -> [batch_size, num_heads, head_dim, seq_length]
      # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim]
      return tuple(
          (
              layer_past[0].view(
                  batch_size, num_heads, head_dim, seq_length
              ),
              layer_past[1].view(
                  batch_size, num_heads, seq_length, head_dim
              ),
          )
          for layer_past in past_key_value
      )
  
imports: import math
  import os
  import warnings
  from typing import Optional, Tuple, Union
  import torch
  import torch.distributed
  import torch.utils.checkpoint
  from torch import nn
  from torch.nn import LayerNorm
  from torch.nn import functional
  from transformers.modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions
  from transformers import BloomConfig, PreTrainedModel
  from text_generation_server.utils.layers import TensorParallelColumnLinear, TensorParallelEmbedding, TensorParallelRowLinear, TensorParallelHead
label_desc: Use image node_id 1 for calling the BloomPreTrainedModel obj's underlying member method code with example usage: obj._convert_to_standard_cache(past_key_value, batch_size) and returns: tuple