




"""simple docstring"""

import argparse
import os

import torch

from diffusers import (
    CMStochasticIterativeScheduler,
    ConsistencyModelPipeline,
    UNetaDModel,
)


a         =      {
    '''sample_size''': 32,
    '''in_channels''': 3,
    '''out_channels''': 3,
    '''layers_per_block''': 2,
    '''num_class_embeds''': 1_000,
    '''block_out_channels''': [32, 64],
    '''attention_head_dim''': 8,
    '''down_block_types''': [
        '''ResnetDownsampleBlock2D''',
        '''AttnDownBlock2D''',
    ],
    '''up_block_types''': [
        '''AttnUpBlock2D''',
        '''ResnetUpsampleBlock2D''',
    ],
    '''resnet_time_scale_shift''': '''scale_shift''',
    '''upsample_type''': '''resnet''',
    '''downsample_type''': '''resnet''',
}

a         =      {
    '''sample_size''': 64,
    '''in_channels''': 3,
    '''out_channels''': 3,
    '''layers_per_block''': 3,
    '''num_class_embeds''': 1_000,
    '''block_out_channels''': [192, 192 * 2, 192 * 3, 192 * 4],
    '''attention_head_dim''': 64,
    '''down_block_types''': [
        '''ResnetDownsampleBlock2D''',
        '''AttnDownBlock2D''',
        '''AttnDownBlock2D''',
        '''AttnDownBlock2D''',
    ],
    '''up_block_types''': [
        '''AttnUpBlock2D''',
        '''AttnUpBlock2D''',
        '''AttnUpBlock2D''',
        '''ResnetUpsampleBlock2D''',
    ],
    '''resnet_time_scale_shift''': '''scale_shift''',
    '''upsample_type''': '''resnet''',
    '''downsample_type''': '''resnet''',
}

a         =      {
    '''sample_size''': 256,
    '''in_channels''': 3,
    '''out_channels''': 3,
    '''layers_per_block''': 2,
    '''num_class_embeds''': None,
    '''block_out_channels''': [256, 256, 256 * 2, 256 * 2, 256 * 4, 256 * 4],
    '''attention_head_dim''': 64,
    '''down_block_types''': [
        '''ResnetDownsampleBlock2D''',
        '''ResnetDownsampleBlock2D''',
        '''ResnetDownsampleBlock2D''',
        '''AttnDownBlock2D''',
        '''AttnDownBlock2D''',
        '''AttnDownBlock2D''',
    ],
    '''up_block_types''': [
        '''AttnUpBlock2D''',
        '''AttnUpBlock2D''',
        '''AttnUpBlock2D''',
        '''ResnetUpsampleBlock2D''',
        '''ResnetUpsampleBlock2D''',
        '''ResnetUpsampleBlock2D''',
    ],
    '''resnet_time_scale_shift''': '''default''',
    '''upsample_type''': '''resnet''',
    '''downsample_type''': '''resnet''',
}

a         =      {
    '''num_train_timesteps''': 40,
    '''sigma_min''': 0.0_0_2,
    '''sigma_max''': 80.0,
}

a         =      {
    '''num_train_timesteps''': 201,
    '''sigma_min''': 0.0_0_2,
    '''sigma_max''': 80.0,
}

a         =      {
    '''num_train_timesteps''': 151,
    '''sigma_min''': 0.0_0_2,
    '''sigma_max''': 80.0,
}






def 						_snake_case					( _snake_case							: Dict							)   -> int:






      '''simple docstring'''


      if isinstance(_snake_case    ,  _snake_case							):
            return v
      if v.lower() in ("yes", "true", "t", "y", "1"):
            return True
      elif v.lower() in ("no", "false", "f", "n", "0"):
            return False
      else:
            raise argparse.ArgumentTypeError('boolean value expected'							)






def 						_snake_case					( _snake_case							: Optional[int]    ,  _snake_case							: Tuple    ,  _snake_case							: Dict    ,  _snake_case							: Dict    ,  _snake_case							: Optional[Any]=False							)   -> List[str]:






      '''simple docstring'''


      _A        =       checkpoint[F'''{old_prefix}.in_layers.0.weight''']
      _A        =       checkpoint[F'''{old_prefix}.in_layers.0.bias''']
      _A        =       checkpoint[F'''{old_prefix}.in_layers.2.weight''']
      _A        =       checkpoint[F'''{old_prefix}.in_layers.2.bias''']
      _A        =       checkpoint[F'''{old_prefix}.emb_layers.1.weight''']
      _A        =       checkpoint[F'''{old_prefix}.emb_layers.1.bias''']
      _A        =       checkpoint[F'''{old_prefix}.out_layers.0.weight''']
      _A        =       checkpoint[F'''{old_prefix}.out_layers.0.bias''']
      _A        =       checkpoint[F'''{old_prefix}.out_layers.3.weight''']
      _A        =       checkpoint[F'''{old_prefix}.out_layers.3.bias''']

      if has_skip:
            _A        =       checkpoint[F'''{old_prefix}.skip_connection.weight''']
            _A        =       checkpoint[F'''{old_prefix}.skip_connection.bias''']

      return new_checkpoint






def 						_snake_case					( _snake_case							: List[Any]    ,  _snake_case							: int    ,  _snake_case							: Optional[int]    ,  _snake_case							: List[Any]    ,  _snake_case							: int=None							)   -> Optional[int]:






      '''simple docstring'''


      _A       ,   _A       ,   _A        =       checkpoint[F'''{old_prefix}.qkv.weight'''].chunk(3    ,  dim=0							)
      _A       ,   _A       ,   _A        =       checkpoint[F'''{old_prefix}.qkv.bias'''].chunk(3    ,  dim=0							)

      _A        =       checkpoint[F'''{old_prefix}.norm.weight''']
      _A        =       checkpoint[F'''{old_prefix}.norm.bias''']

      _A        =       weight_q.squeeze(-1							).squeeze(-1							)
      _A        =       bias_q.squeeze(-1							).squeeze(-1							)
      _A        =       weight_k.squeeze(-1							).squeeze(-1							)
      _A        =       bias_k.squeeze(-1							).squeeze(-1							)
      _A        =       weight_v.squeeze(-1							).squeeze(-1							)
      _A        =       bias_v.squeeze(-1							).squeeze(-1							)

      _A        =       (
          checkpoint[F'''{old_prefix}.proj_out.weight'''].squeeze(-1							).squeeze(-1							)
      )
      _A        =       checkpoint[F'''{old_prefix}.proj_out.bias'''].squeeze(-1							).squeeze(-1							)

      return new_checkpoint






def 						_snake_case					( _snake_case							: str    ,  _snake_case							: Any							)   -> str:






      '''simple docstring'''


      _A        =       torch.load(_snake_case    ,  map_location='cpu'							)
      _A        =       {}

      _A        =       checkpoint['time_embed.0.weight']
      _A        =       checkpoint['time_embed.0.bias']
      _A        =       checkpoint['time_embed.2.weight']
      _A        =       checkpoint['time_embed.2.bias']

      if unet_config["num_class_embeds"] is not None:
            _A        =       checkpoint['label_emb.weight']

      _A        =       checkpoint['input_blocks.0.0.weight']
      _A        =       checkpoint['input_blocks.0.0.bias']

      _A        =       unet_config['down_block_types']
      _A        =       unet_config['layers_per_block']
      _A        =       unet_config['attention_head_dim']
      _A        =       unet_config['block_out_channels']
      _A        =       1
      _A        =       channels_list[0]

      for i, layer_type in enumerate(_snake_case							):
            _A        =       channels_list[i]
            _A        =       current_channels != prev_channels
            if layer_type == "ResnetDownsampleBlock2D":
                  for j in range(_snake_case							):
                        _A        =       F'''down_blocks.{i}.resnets.{j}'''
                        _A        =       F'''input_blocks.{current_layer}.0'''
                        _A        =       True if j == 0 and downsample_block_has_skip else False
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  has_skip=_snake_case							)
                        current_layer += 1

            elif layer_type == "AttnDownBlock2D":
                  for j in range(_snake_case							):
                        _A        =       F'''down_blocks.{i}.resnets.{j}'''
                        _A        =       F'''input_blocks.{current_layer}.0'''
                        _A        =       True if j == 0 and downsample_block_has_skip else False
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  has_skip=_snake_case							)
                        _A        =       F'''down_blocks.{i}.attentions.{j}'''
                        _A        =       F'''input_blocks.{current_layer}.1'''
                        _A        =       convert_attention(
                            _snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
                        current_layer += 1

            if i != len(_snake_case							) - 1:
                  _A        =       F'''down_blocks.{i}.downsamplers.0'''
                  _A        =       F'''input_blocks.{current_layer}.0'''
                  _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
                  current_layer += 1

            _A        =       current_channels

      # hardcoded the mid-block for now
      _A        =       'mid_block.resnets.0'
      _A        =       'middle_block.0'
      _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
      _A        =       'mid_block.attentions.0'
      _A        =       'middle_block.1'
      _A        =       convert_attention(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
      _A        =       'mid_block.resnets.1'
      _A        =       'middle_block.2'
      _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)

      _A        =       0
      _A        =       unet_config['up_block_types']

      for i, layer_type in enumerate(_snake_case							):
            if layer_type == "ResnetUpsampleBlock2D":
                  for j in range(layers_per_block + 1							):
                        _A        =       F'''up_blocks.{i}.resnets.{j}'''
                        _A        =       F'''output_blocks.{current_layer}.0'''
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  has_skip=_snake_case							)
                        current_layer += 1

                  if i != len(_snake_case							) - 1:
                        _A        =       F'''up_blocks.{i}.upsamplers.0'''
                        _A        =       F'''output_blocks.{current_layer-1}.1'''
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
            elif layer_type == "AttnUpBlock2D":
                  for j in range(layers_per_block + 1							):
                        _A        =       F'''up_blocks.{i}.resnets.{j}'''
                        _A        =       F'''output_blocks.{current_layer}.0'''
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  has_skip=_snake_case							)
                        _A        =       F'''up_blocks.{i}.attentions.{j}'''
                        _A        =       F'''output_blocks.{current_layer}.1'''
                        _A        =       convert_attention(
                            _snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)
                        current_layer += 1

                  if i != len(_snake_case							) - 1:
                        _A        =       F'''up_blocks.{i}.upsamplers.0'''
                        _A        =       F'''output_blocks.{current_layer-1}.2'''
                        _A        =       convert_resnet(_snake_case    ,  _snake_case    ,  _snake_case    ,  _snake_case							)

      _A        =       checkpoint['out.0.weight']
      _A        =       checkpoint['out.0.bias']
      _A        =       checkpoint['out.2.weight']
      _A        =       checkpoint['out.2.bias']

      return new_checkpoint


if __name__ == "__main__":
   a         =      argparse.ArgumentParser()

   parser.add_argument('''--unet_path''', default=None, type=str, required=True, help='''Path to the unet.pt to convert.''')
   parser.add_argument(
       '''--dump_path''', default=None, type=str, required=True, help='''Path to output the converted UNet model.'''
   )
   parser.add_argument('''--class_cond''', default=True, type=str, help='''Whether the model is class-conditional.''')

   a         =      parser.parse_args()
   a         =      strabool(args.class_cond)

   a         =      os.path.basename(args.unet_path)
   print(F'''Checkpoint: {ckpt_name}''')

   # Get U-Net config
   if "imagenet64" in ckpt_name:
      a         =      IMAGENET_64_UNET_CONFIG
   elif "256" in ckpt_name and (("bedroom" in ckpt_name) or ("cat" in ckpt_name)):
      a         =      LSUN_256_UNET_CONFIG
   elif "test" in ckpt_name:
      a         =      TEST_UNET_CONFIG
   else:
      raise ValueError(F'''Checkpoint type {ckpt_name} is not currently supported.''')

   if not args.class_cond:
      a         =      None

   a         =      con_pt_to_diffuser(args.unet_path, unet_config)

   a         =      UNetaDModel(**unet_config)
   image_unet.load_state_dict(converted_unet_ckpt)

   # Get scheduler config
   if "cd" in ckpt_name or "test" in ckpt_name:
      a         =      CD_SCHEDULER_CONFIG
   elif "ct" in ckpt_name and "imagenet64" in ckpt_name:
      a         =      CT_IMAGENET_64_SCHEDULER_CONFIG
   elif "ct" in ckpt_name and "256" in ckpt_name and (("bedroom" in ckpt_name) or ("cat" in ckpt_name)):
      a         =      CT_LSUN_256_SCHEDULER_CONFIG
   else:
      raise ValueError(F'''Checkpoint type {ckpt_name} is not currently supported.''')

   a         =      CMStochasticIterativeScheduler(**scheduler_config)

   a         =      ConsistencyModelPipeline(unet=image_unet, scheduler=cm_scheduler)
   consistency_model.save_pretrained(args.dump_path)







"""simple docstring"""

import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
       import tensorflow as tf
       from tensorflow.python.framework.errors_impl import ResourceExhaustedError

       from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
       import pyanvml.pyanvml as nvml

a         =      logging.get_logger(__name__)






def 						_snake_case					( _snake_case							: bool    ,  _snake_case							: bool							)   -> Tuple:






      '''simple docstring'''


      def run_func(_snake_case							: Any							):
            @wraps(_snake_case							)
            def run_in_eager_mode(*_snake_case							: List[str]    ,  **_snake_case							: Tuple							):
                  return func(*_snake_case    ,  **_snake_case							)

            @wraps(_snake_case							)
            @tf.function(experimental_compile=_snake_case							)
            def run_in_graph_mode(*_snake_case							: Dict    ,  **_snake_case							: Tuple							):
                  return func(*_snake_case    ,  **_snake_case							)

            if do_eager_mode is True:
                  if use_xla is not False:
                        raise ValueError(
                            'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'							)
                  return run_in_eager_mode
            else:
                  return run_in_graph_mode

      return run_func






def 						_snake_case					( _snake_case							: int    ,  _snake_case							: int    ,  _snake_case							: int							)   -> ["tf.Tensor"]:






      '''simple docstring'''


      _A        =       random.Random()
      _A        =       [rng.randint(0    ,  vocab_size - 1							) for i in range(batch_size * sequence_length							)]
      return tf.constant(_snake_case    ,  shape=(batch_size, sequence_length)    ,  dtype=tf.intaa							)







class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''
 UpperCAmelCase   :					TensorFlowBenchmarkArguments
 UpperCAmelCase   :					PretrainedConfig
 UpperCAmelCase   :					str				  =     "TensorFlow"

 @property
 def       lowerCAmelCase_							(			self	:	str		):
       return tf.__version__

 def       lowerCAmelCase_							(			self	:	List[str]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       # initialize GPU on separate process
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_inference_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_speed(_inference		)

 def       lowerCAmelCase_							(			self	:	Optional[Any]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_train_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_speed(_train		)

 def       lowerCAmelCase_							(			self	:	Dict   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       # initialize GPU on separate process
       if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]   ,							_UpperCAmelCase		)
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_inference_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_memory(_inference		)

 def       lowerCAmelCase_							(			self	:	int   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]   ,							_UpperCAmelCase		)
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)

       _A        =       self._prepare_train_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_memory(_train		)

 def       lowerCAmelCase_							(			self	:	List[Any]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.config_dict[model_name]

       if self.args.fpaa:
             raise NotImplementedError('Mixed precision is currently not supported.'		)

       _A        =       (
           hasattr(_UpperCAmelCase   ,							'architectures'		)
           and isinstance(config.architectures   ,							_UpperCAmelCase		)
           and len(config.architectures		) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   _A        =       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _A        =       __import__('transformers'   ,							fromlist=[model_class]		)
                   _A        =       getattr(_UpperCAmelCase   ,							_UpperCAmelCase		)
                   _A        =       model_cls(_UpperCAmelCase		)
             except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
       else:
             _A        =       TF_MODEL_MAPPING[config.__class__](_UpperCAmelCase		)

       # encoder-decoder has vocab size saved differently
       _A        =       config.vocab_size if hasattr(_UpperCAmelCase   ,							'vocab_size'		) else config.encoder.vocab_size
       _A        =       random_input_ids(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_decoder_forward():
             return model(_UpperCAmelCase   ,							decoder_input_ids=_UpperCAmelCase   ,							training=_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_forward():
             return model(_UpperCAmelCase   ,							training=_UpperCAmelCase		)

       _A        =       encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

       return _inference

 def       lowerCAmelCase_							(			self	:	str   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.config_dict[model_name]

       if self.args.eager_mode is not False:
             raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'		)

       if self.args.fpaa:
             raise NotImplementedError('Mixed precision is currently not supported.'		)

       _A        =       (
           hasattr(_UpperCAmelCase   ,							'architectures'		)
           and isinstance(config.architectures   ,							_UpperCAmelCase		)
           and len(config.architectures		) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   _A        =       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _A        =       __import__('transformers'   ,							fromlist=[model_class]		)
                   _A        =       getattr(_UpperCAmelCase   ,							_UpperCAmelCase		)
                   _A        =       model_cls(_UpperCAmelCase		)
             except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
       else:
             _A        =       TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_UpperCAmelCase		)

       # encoder-decoder has vocab size saved differently
       _A        =       config.vocab_size if hasattr(_UpperCAmelCase   ,							'vocab_size'		) else config.encoder.vocab_size
       _A        =       random_input_ids(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_decoder_train():
             _A        =       model(_UpperCAmelCase   ,							decoder_input_ids=_UpperCAmelCase   ,							labels=_UpperCAmelCase   ,							training=_UpperCAmelCase		)[0]
             _A        =       tf.gradients(_UpperCAmelCase   ,							model.trainable_variables		)
             return gradients

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_train():
             _A        =       model(_UpperCAmelCase   ,							labels=_UpperCAmelCase   ,							training=_UpperCAmelCase		)[0]
             _A        =       tf.gradients(_UpperCAmelCase   ,							model.trainable_variables		)
             return gradients

       _A        =       encoder_decoder_train if config.is_encoder_decoder else encoder_train

       return _train

 def       lowerCAmelCase_							(			self	:	Tuple   ,							_UpperCAmelCase	:	int		):
       with self.args.strategy.scope():
             try:
                   if self.args.is_tpu or self.args.use_xla:
                         # run additional 10 times to stabilize compilation for tpu
                         logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'		)
                         timeit.repeat(_UpperCAmelCase   ,							repeat=1   ,							number=5		)

                   # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                   _A        =       timeit.repeat(
                       _UpperCAmelCase   ,							repeat=self.args.repeat   ,							number=10   ,							)

                   return min(_UpperCAmelCase		) / 10.0
             except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)



 def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	Callable[[], None]		):
       logger.info(
           'Note that TensorFlow allocates more memory than '
           'it might need to speed up computation. '
           'The memory reported here corresponds to the memory '
           'reported by `nvidia-smi`, which can vary depending '
           'on total available memory on the GPU that is used.'		)
       with self.args.strategy.scope():
             try:
                   if self.args.trace_memory_line_by_line:
                         if not self.args.eager_mode:
                               raise ValueError(
                                   '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
                                   ' consumption line by line.'		)
                         _A        =       start_memory_tracing('transformers'		)

                   if self.args.is_tpu:
                         # tpu
                         raise NotImplementedError(
                             'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
                             ' with `args.memory=False`'		)
                   elif self.args.is_gpu:
                         # gpu
                         if not is_pyanvml_available():
                               logger.warning(
                                   'py3nvml not installed, we won\'t log GPU memory usage. '
                                   'Install py3nvml (pip install py3nvml) to log information about GPU.'		)
                               _A        =       'N/A'
                         else:
                               logger.info(
                                   'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
                                   ' running on the same GPU.'		)
                               # init nvml
                               nvml.nvmlInit()
                               func()
                               _A        =       nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx		)
                               _A        =       nvml.nvmlDeviceGetMemoryInfo(_UpperCAmelCase		)
                               _A        =       meminfo.used
                               _A        =       Memory(_UpperCAmelCase		)
                               # shutdown nvml
                               nvml.nvmlShutdown()
                   else:
                         # cpu
                         if self.args.trace_memory_line_by_line:
                               logger.info(
                                   'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
                                   ' TensorFlow.'		)
                               _A        =       None
                         else:
                               _A        =       measure_peak_memory_cpu(_UpperCAmelCase		)
                               _A        =       Memory(_UpperCAmelCase		) if isinstance(_UpperCAmelCase   ,							_UpperCAmelCase		) else memory_bytes
                   if self.args.trace_memory_line_by_line:
                         _A        =       stop_memory_tracing(_UpperCAmelCase		)
                         if memory is None:
                               _A        =       summary.total
                   else:
                         _A        =       None

                   return memory, summary
             except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)
                   return "N/A", None

