


'''simple docstring'''



import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
							import tensorflow as tf
							from tensorflow.python.framework.errors_impl import ResourceExhaustedError

							from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
							import pyanvml.pyanvml as nvml

__lowercase     :						Optional[int]	=     logging.get_logger(__name__)






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       bool						,				_SCREAMING_SNAKE_CASE				:       bool       ):







					def run_func(_SCREAMING_SNAKE_CASE				:       Optional[Any]       ):
										@wraps(UpperCamelCase__       )
										def run_in_eager_mode(*_SCREAMING_SNAKE_CASE				:       int						,				**_SCREAMING_SNAKE_CASE				:       Dict       ):
															return func(*UpperCamelCase__						,				**UpperCamelCase__       )

										@wraps(UpperCamelCase__       )
										@tf.function(experimental_compile=UpperCamelCase__       )
										def run_in_graph_mode(*_SCREAMING_SNAKE_CASE				:       List[Any]						,				**_SCREAMING_SNAKE_CASE				:       Tuple       ):
															return func(*UpperCamelCase__						,				**UpperCamelCase__       )

										if do_eager_mode is True:
															if use_xla is not False:
																				raise ValueError(
																				    'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'       )
															return run_in_eager_mode
										else:
															return run_in_graph_mode

					return run_func






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int						,				_SCREAMING_SNAKE_CASE				:       int						,				_SCREAMING_SNAKE_CASE				:       int       ):
					__a      :	Tuple    =  random.Random()
					__a      :	Any    =  [rng.randint(0						,				vocab_size - 1       ) for i in range(batch_size * sequence_length       )]
					return tf.constant(UpperCamelCase__						,				shape=(batch_size, sequence_length)						,				dtype=tf.intaa       )





class 							__UpperCamelCase    (		lowerCAmelCase_ ):
	A_           =   42
	A_           =   42
	A_           =   "TensorFlow"




	@property
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						return tf.__version__




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						__a      :	Tuple    =  self.args.strategy
						if strategy is None:
											raise ValueError('A device strategy has to be initialized before using TensorFlow.'							)
						__a      :	Union[str, Any]    =  self._prepare_inference_func(_a						,    _a						,    _a							)
						return self._measure_speed(_inference							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						__a      :	int    =  self.args.strategy
						if strategy is None:
											raise ValueError('A device strategy has to be initialized before using TensorFlow.'							)
						__a      :	List[Any]    =  self._prepare_train_func(_a						,    _a						,    _a							)
						return self._measure_speed(_train							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						if self.args.is_gpu:
											tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,    _a							)
						__a      :	Any    =  self.args.strategy
						if strategy is None:
											raise ValueError('A device strategy has to be initialized before using TensorFlow.'							)
						__a      :	Optional[Any]    =  self._prepare_inference_func(_a						,    _a						,    _a							)
						return self._measure_memory(_inference							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						if self.args.is_gpu:
											tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,    _a							)
						__a      :	str    =  self.args.strategy
						if strategy is None:
											raise ValueError('A device strategy has to be initialized before using TensorFlow.'							)

						__a      :	Optional[int]    =  self._prepare_train_func(_a						,    _a						,    _a							)
						return self._measure_memory(_train							)




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						__a      :	str    =  self.config_dict[model_name]

						if self.args.fpaa:
											raise NotImplementedError('Mixed precision is currently not supported.'							)

						__a      :	Any    =  (
						    hasattr(_a						,    'architectures'							)
						    and isinstance(config.architectures						,    _a							)
						    and len(config.architectures							) > 0
						)
						if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
																__a      :	str    =  'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
																__a      :	int    =  __import__('transformers'						,    fromlist=[model_class]							)
																__a      :	Optional[int]    =  getattr(_a						,    _a							)
																__a      :	Dict    =  model_cls(_a							)
											except ImportError:
																raise ImportError(
																    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'							)
						else:
											__a      :	List[Any]    =  TF_MODEL_MAPPING[config.__class__](_a							)

						# encoder-decoder has vocab size saved differently
						__a      :	str    =  config.vocab_size if hasattr(_a						,    'vocab_size'							) else config.encoder.vocab_size
						__a      :	Any    =  random_input_ids(_a						,    _a						,    _a							)

						@run_with_tf_optimizations(self.args.eager_mode						,    self.args.use_xla							)
						def encoder_decoder_forward():
											return model(_a						,    decoder_input_ids=_a						,    training=_a							)

						@run_with_tf_optimizations(self.args.eager_mode						,    self.args.use_xla							)
						def encoder_forward():
											return model(_a						,    training=_a							)

						__a      :	Any    =  encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

						return _inference




	def 				__UpperCAmelCase							(       self						,    __a						,    __a						,    __a							):

						'''simple docstring'''





						__a      :	int    =  self.config_dict[model_name]

						if self.args.eager_mode is not False:
											raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'							)

						if self.args.fpaa:
											raise NotImplementedError('Mixed precision is currently not supported.'							)

						__a      :	Any    =  (
						    hasattr(_a						,    'architectures'							)
						    and isinstance(config.architectures						,    _a							)
						    and len(config.architectures							) > 0
						)
						if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
																__a      :	int    =  'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
																__a      :	int    =  __import__('transformers'						,    fromlist=[model_class]							)
																__a      :	Union[str, Any]    =  getattr(_a						,    _a							)
																__a      :	List[str]    =  model_cls(_a							)
											except ImportError:
																raise ImportError(
																    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
																    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'							)
						else:
											__a      :	Optional[int]    =  TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_a							)

						# encoder-decoder has vocab size saved differently
						__a      :	Any    =  config.vocab_size if hasattr(_a						,    'vocab_size'							) else config.encoder.vocab_size
						__a      :	Optional[Any]    =  random_input_ids(_a						,    _a						,    _a							)

						@run_with_tf_optimizations(self.args.eager_mode						,    self.args.use_xla							)
						def encoder_decoder_train():
											__a      :	Tuple    =  model(_a						,    decoder_input_ids=_a						,    labels=_a						,    training=_a							)[0]
											__a      :	List[str]    =  tf.gradients(_a						,    model.trainable_variables							)
											return gradients

						@run_with_tf_optimizations(self.args.eager_mode						,    self.args.use_xla							)
						def encoder_train():
											__a      :	Optional[Any]    =  model(_a						,    labels=_a						,    training=_a							)[0]
											__a      :	int    =  tf.gradients(_a						,    model.trainable_variables							)
											return gradients

						__a      :	Any    =  encoder_decoder_train if config.is_encoder_decoder else encoder_train

						return _train




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						with self.args.strategy.scope():
											try:
																if self.args.is_tpu or self.args.use_xla:
																					# run additional 10 times to stabilize compilation for tpu
																					logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'							)
																					timeit.repeat(_a						,    repeat=1						,    number=5							)

																# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																__a      :	Union[str, Any]    =  timeit.repeat(
																    _a						,    repeat=self.args.repeat						,    number=10						,    )

																return min(_a							) / 10.0
											except ResourceExhaustedError as e:
																self.print_fn(f"""Doesn't fit on GPU. {e}"""							)





	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						logger.info(
						    'Note that TensorFlow allocates more memory than '
						    'it might need to speed up computation. '
						    'The memory reported here corresponds to the memory '
						    'reported by `nvidia-smi`, which can vary depending '
						    'on total available memory on the GPU that is used.'							)
						with self.args.strategy.scope():
											try:
																if self.args.trace_memory_line_by_line:
																					if not self.args.eager_mode:
																										raise ValueError(
																										    '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
																										    ' consumption line by line.'							)
																					__a      :	Dict    =  start_memory_tracing('transformers'							)

																if self.args.is_tpu:
																					# tpu
																					raise NotImplementedError(
																					    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
																					    ' with `args.memory=False`'							)
																elif self.args.is_gpu:
																					# gpu
																					if not is_pyanvml_available():
																										logger.warning(
																										    'py3nvml not installed, we won\'t log GPU memory usage. '
																										    'Install py3nvml (pip install py3nvml) to log information about GPU.'							)
																										__a      :	List[str]    =  'N/A'
																					else:
																										logger.info(
																										    'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
																										    ' running on the same GPU.'							)
																										# init nvml
																										nvml.nvmlInit()
																										func()
																										__a      :	List[str]    =  nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx							)
																										__a      :	Any    =  nvml.nvmlDeviceGetMemoryInfo(_a							)
																										__a      :	Dict    =  meminfo.used
																										__a      :	Optional[Any]    =  Memory(_a							)
																										# shutdown nvml
																										nvml.nvmlShutdown()
																else:
																					# cpu
																					if self.args.trace_memory_line_by_line:
																										logger.info(
																										    'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
																										    ' TensorFlow.'							)
																										__a      :	Optional[int]    =  None
																					else:
																										__a      :	int    =  measure_peak_memory_cpu(_a							)
																										__a      :	Dict    =  Memory(_a							) if isinstance(_a						,    _a							) else memory_bytes
																if self.args.trace_memory_line_by_line:
																					__a      :	Tuple    =  stop_memory_tracing(_a							)
																					if memory is None:
																										__a      :	Tuple    =  summary.total
																else:
																					__a      :	List[Any]    =  None

																return memory, summary
											except ResourceExhaustedError as e:
																self.print_fn(f"""Doesn't fit on GPU. {e}"""							)
																return "N/A", None







'''simple docstring'''



from .imports import is_rich_available


if is_rich_available():
	from rich.traceback import install

	install(show_locals=False)

else:
	raise ModuleNotFoundError('To use the rich extension, install rich with `pip install rich`')



