


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
	import tensorflow as tf
	from tensorflow.python.framework.errors_impl import ResourceExhaustedError

	from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
	import pyanvml.pyanvml as nvml

A			:						int        =  logging.get_logger(__name__)






def 					UpperCamelCase   (      __magic_name__	:				bool       ,       __magic_name__	:				bool	)  ->    Optional[int]:
		"""simple docstring"""




		def run_func(__magic_name__	:				Dict	):
				@wraps(__magic_name__	)
				def run_in_eager_mode(*__magic_name__	:				str       ,       **__magic_name__	:				Union[str, Any]	):
						return func(*__magic_name__       ,       **__magic_name__	)

				@wraps(__magic_name__	)
				@tf.function(experimental_compile=__magic_name__	)
				def run_in_graph_mode(*__magic_name__	:				Optional[Any]       ,       **__magic_name__	:				Tuple	):
						return func(*__magic_name__       ,       **__magic_name__	)

				if do_eager_mode is True:
						if use_xla is not False:
								raise ValueError(
								    """Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."""	)
						return run_in_eager_mode
				else:
						return run_in_graph_mode

		return run_func






def 					UpperCamelCase   (      __magic_name__	:				int       ,       __magic_name__	:				int       ,       __magic_name__	:				int	)  ->    ["tf.Tensor"]:
		"""simple docstring"""




		lowercase__	     =						random.Random()
		lowercase__	     =						[rng.randint(0       ,       vocab_size - 1	) for i in range(batch_size * sequence_length	)]
		return tf.constant(__magic_name__       ,       shape=(batch_size, sequence_length)       ,       dtype=tf.intaa	)







class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					42
					A__     =					42
					A__     =					"TensorFlow"

					@property
					def     lowerCamelCase__	(self       :		Dict						)		->  List[Any]:




							"""simple docstring"""


							return tf.__version__

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  float:




							"""simple docstring"""


							lowercase__	     =						self.args.strategy
							if strategy is None:
									raise ValueError("""A device strategy has to be initialized before using TensorFlow."""						)
							lowercase__	     =						self._prepare_inference_func(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)
							return self._measure_speed(_inference						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  float:




							"""simple docstring"""


							lowercase__	     =						self.args.strategy
							if strategy is None:
									raise ValueError("""A device strategy has to be initialized before using TensorFlow."""						)
							lowercase__	     =						self._prepare_train_func(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)
							return self._measure_speed(_train						)

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  [Memory, Optional[MemorySummary]]:




							"""simple docstring"""


							if self.args.is_gpu:
									tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx] ,   _UpperCAmelCase						)
							lowercase__	     =						self.args.strategy
							if strategy is None:
									raise ValueError("""A device strategy has to be initialized before using TensorFlow."""						)
							lowercase__	     =						self._prepare_inference_func(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)
							return self._measure_memory(_inference						)

					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  [Memory, Optional[MemorySummary]]:




							"""simple docstring"""


							if self.args.is_gpu:
									tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx] ,   _UpperCAmelCase						)
							lowercase__	     =						self.args.strategy
							if strategy is None:
									raise ValueError("""A device strategy has to be initialized before using TensorFlow."""						)

							lowercase__	     =						self._prepare_train_func(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)
							return self._measure_memory(_train						)

					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  Callable[[], None]:




							"""simple docstring"""


							lowercase__	     =						self.config_dict[model_name]

							if self.args.fpaa:
									raise NotImplementedError("""Mixed precision is currently not supported."""						)

							lowercase__	     =						(
							    hasattr(_UpperCAmelCase ,   """architectures"""						)
							    and isinstance(config.architectures ,   _UpperCAmelCase						)
							    and len(config.architectures						) > 0
							)
							if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
											lowercase__	     =						"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
											lowercase__	     =						__import__("""transformers""" ,   fromlist=[model_class]						)
											lowercase__	     =						getattr(_UpperCAmelCase ,   _UpperCAmelCase						)
											lowercase__	     =						model_cls(_UpperCAmelCase						)
									except ImportError:
											raise ImportError(
											    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
											    """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""						)
							else:
									lowercase__	     =						TF_MODEL_MAPPING[config.__class__](_UpperCAmelCase						)

							# encoder-decoder has vocab size saved differently
							lowercase__	     =						config.vocab_size if hasattr(_UpperCAmelCase ,   """vocab_size"""						) else config.encoder.vocab_size
							lowercase__	     =						random_input_ids(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)

							@run_with_tf_optimizations(self.args.eager_mode ,   self.args.use_xla						)
							def encoder_decoder_forward():
									return model(_UpperCAmelCase ,   decoder_input_ids=_UpperCAmelCase ,   training=_UpperCAmelCase						)

							@run_with_tf_optimizations(self.args.eager_mode ,   self.args.use_xla						)
							def encoder_forward():
									return model(_UpperCAmelCase ,   training=_UpperCAmelCase						)

							lowercase__	     =						encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

							return _inference

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		int						)		->  Callable[[], None]:




							"""simple docstring"""


							lowercase__	     =						self.config_dict[model_name]

							if self.args.eager_mode is not False:
									raise ValueError("""Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."""						)

							if self.args.fpaa:
									raise NotImplementedError("""Mixed precision is currently not supported."""						)

							lowercase__	     =						(
							    hasattr(_UpperCAmelCase ,   """architectures"""						)
							    and isinstance(config.architectures ,   _UpperCAmelCase						)
							    and len(config.architectures						) > 0
							)
							if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
											lowercase__	     =						"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
											lowercase__	     =						__import__("""transformers""" ,   fromlist=[model_class]						)
											lowercase__	     =						getattr(_UpperCAmelCase ,   _UpperCAmelCase						)
											lowercase__	     =						model_cls(_UpperCAmelCase						)
									except ImportError:
											raise ImportError(
											    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
											    """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""						)
							else:
									lowercase__	     =						TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_UpperCAmelCase						)

							# encoder-decoder has vocab size saved differently
							lowercase__	     =						config.vocab_size if hasattr(_UpperCAmelCase ,   """vocab_size"""						) else config.encoder.vocab_size
							lowercase__	     =						random_input_ids(_UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase						)

							@run_with_tf_optimizations(self.args.eager_mode ,   self.args.use_xla						)
							def encoder_decoder_train():
									lowercase__	     =						model(_UpperCAmelCase ,   decoder_input_ids=_UpperCAmelCase ,   labels=_UpperCAmelCase ,   training=_UpperCAmelCase						)[0]
									lowercase__	     =						tf.gradients(_UpperCAmelCase ,   model.trainable_variables						)
									return gradients

							@run_with_tf_optimizations(self.args.eager_mode ,   self.args.use_xla						)
							def encoder_train():
									lowercase__	     =						model(_UpperCAmelCase ,   labels=_UpperCAmelCase ,   training=_UpperCAmelCase						)[0]
									lowercase__	     =						tf.gradients(_UpperCAmelCase ,   model.trainable_variables						)
									return gradients

							lowercase__	     =						encoder_decoder_train if config.is_encoder_decoder else encoder_train

							return _train

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Dict						)		->  float:




							"""simple docstring"""


							with self.args.strategy.scope():
									try:
											if self.args.is_tpu or self.args.use_xla:
													# run additional 10 times to stabilize compilation for tpu
													logger.info("""Do inference on TPU. Running model 5 times to stabilize compilation"""						)
													timeit.repeat(_UpperCAmelCase ,   repeat=1 ,   number=5						)

											# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
											lowercase__	     =						timeit.repeat(
											    _UpperCAmelCase ,   repeat=self.args.repeat ,   number=10 ,   )

											return min(_UpperCAmelCase						) / 10.0
									except ResourceExhaustedError as e:
											self.print_fn(f'''Doesn\'t fit on GPU. {e}'''						)


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Callable[[], None]						)		->  [Memory, MemorySummary]:




							"""simple docstring"""


							logger.info(
							    """Note that TensorFlow allocates more memory than """
							    """it might need to speed up computation. """
							    """The memory reported here corresponds to the memory """
							    """reported by `nvidia-smi`, which can vary depending """
							    """on total available memory on the GPU that is used."""						)
							with self.args.strategy.scope():
									try:
											if self.args.trace_memory_line_by_line:
													if not self.args.eager_mode:
															raise ValueError(
															    """`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"""
															    """ consumption line by line."""						)
													lowercase__	     =						start_memory_tracing("""transformers"""						)

											if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"""
													    """ with `args.memory=False`"""						)
											elif self.args.is_gpu:
													# gpu
													if not is_pyanvml_available():
															logger.warning(
															    """py3nvml not installed, we won't log GPU memory usage. """
															    """Install py3nvml (pip install py3nvml) to log information about GPU."""						)
															lowercase__	     =						"""N/A"""
													else:
															logger.info(
															    """Measuring total GPU usage on GPU device. Make sure to not have additional processes"""
															    """ running on the same GPU."""						)
															# init nvml
															nvml.nvmlInit()
															func()
															lowercase__	     =						nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx						)
															lowercase__	     =						nvml.nvmlDeviceGetMemoryInfo(_UpperCAmelCase						)
															lowercase__	     =						meminfo.used
															lowercase__	     =						Memory(_UpperCAmelCase						)
															# shutdown nvml
															nvml.nvmlShutdown()
											else:
													# cpu
													if self.args.trace_memory_line_by_line:
															logger.info(
															    """When enabling line by line tracing, the max peak memory for CPU is inaccurate in"""
															    """ TensorFlow."""						)
															lowercase__	     =						None
													else:
															lowercase__	     =						measure_peak_memory_cpu(_UpperCAmelCase						)
															lowercase__	     =						Memory(_UpperCAmelCase						) if isinstance(_UpperCAmelCase ,   _UpperCAmelCase						) else memory_bytes
											if self.args.trace_memory_line_by_line:
													lowercase__	     =						stop_memory_tracing(_UpperCAmelCase						)
													if memory is None:
															lowercase__	     =						summary.total
											else:
													lowercase__	     =						None

											return memory, summary
									except ResourceExhaustedError as e:
											self.print_fn(f'''Doesn\'t fit on GPU. {e}'''						)
											return "N/A", None





import os
import sys

A			:						Tuple        =  os.path.join(os.path.dirname(__file__), 'src')
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


A			:						Dict        =  [
    'torch',
    'numpy',
    'tokenizers',
    'filelock',
    'requests',
    'tqdm',
    'regex',
    'sentencepiece',
    'sacremoses',
    'importlib_metadata',
    'huggingface_hub',
]






@add_start_docstrings(AutoConfig.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				List[Any]	)  ->    str:
		"""simple docstring"""




		return AutoConfig.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoTokenizer.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Any       ,       **__magic_name__	:				Optional[Any]	)  ->    Optional[Any]:
		"""simple docstring"""




		return AutoTokenizer.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModel.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				Optional[Any]	)  ->    Tuple:
		"""simple docstring"""




		return AutoModel.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForCausalLM.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				Tuple	)  ->    List[str]:
		"""simple docstring"""




		return AutoModelForCausalLM.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForMaskedLM.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Optional[Any]       ,       **__magic_name__	:				Any	)  ->    Optional[int]:
		"""simple docstring"""




		return AutoModelForMaskedLM.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForSequenceClassification.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Any       ,       **__magic_name__	:				List[str]	)  ->    Union[str, Any]:
		"""simple docstring"""




		return AutoModelForSequenceClassification.from_pretrained(*__magic_name__       ,       **__magic_name__	)







@add_start_docstrings(AutoModelForQuestionAnswering.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Dict       ,       **__magic_name__	:				List[Any]	)  ->    int:
		"""simple docstring"""




		return AutoModelForQuestionAnswering.from_pretrained(*__magic_name__       ,       **__magic_name__	)

