


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
							import tensorflow as tf
							from tensorflow.python.framework.errors_impl import ResourceExhaustedError

							from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
							import pyanvml.pyanvml as nvml

snake_case_						:			List[str]													=							logging.get_logger(__name__)







def 						lowerCamelCase(   a__	,a__):
				def run_func(a__):
								@wraps(a__)
								def run_in_eager_mode(*a__	,**a__):
												return func(*a__	,**a__)

								@wraps(a__)
								@tf.function(experimental_compile=a__)
								def run_in_graph_mode(*a__	,**a__):
												return func(*a__	,**a__)

								if do_eager_mode is True:
												if use_xla is not False:
																raise ValueError(
																    '''Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.''')
												return run_in_eager_mode
								else:
												return run_in_graph_mode

				return run_func







def 						lowerCamelCase(   a__	,a__	,a__):
				_SCREAMING_SNAKE_CASE			  =random.Random()
				_SCREAMING_SNAKE_CASE			  =[rng.randint(0	,vocab_size - 1) for i in range(batch_size * sequence_length)]
				return tf.constant(a__	,shape=(batch_size, sequence_length)	,dtype=tf.intaa)


class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						42
						UpperCAmelCase             =						42
						UpperCAmelCase             =						"TensorFlow"




						@property
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Dict:


										"""simple docstring"""





										return tf.__version__




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     float:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.args.strategy
										if strategy is None:
														raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''  )
										_SCREAMING_SNAKE_CASE			  =self._prepare_inference_func(_a				,    _a				,    _a  )
										return self._measure_speed(_inference  )




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     float:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.args.strategy
										if strategy is None:
														raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''  )
										_SCREAMING_SNAKE_CASE			  =self._prepare_train_func(_a				,    _a				,    _a  )
										return self._measure_speed(_train  )




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     [Memory, Optional[MemorySummary]]:


										"""simple docstring"""





										if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.args.strategy
										if strategy is None:
														raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''  )
										_SCREAMING_SNAKE_CASE			  =self._prepare_inference_func(_a				,    _a				,    _a  )
										return self._measure_memory(_inference  )




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     [Memory, Optional[MemorySummary]]:


										"""simple docstring"""





										if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.args.strategy
										if strategy is None:
														raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''  )

										_SCREAMING_SNAKE_CASE			  =self._prepare_train_func(_a				,    _a				,    _a  )
										return self._measure_memory(_train  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     Callable[[], None]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.config_dict[model_name]

										if self.args.fpaa:
														raise NotImplementedError('''Mixed precision is currently not supported.'''  )

										_SCREAMING_SNAKE_CASE			  =(
										    hasattr(_a				,    '''architectures'''  )
										    and isinstance(config.architectures				,    _a  )
										    and len(config.architectures  ) > 0
										)
										if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																		_SCREAMING_SNAKE_CASE			  ='''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
																		_SCREAMING_SNAKE_CASE			  =__import__('''transformers'''				,    fromlist=[model_class]  )
																		_SCREAMING_SNAKE_CASE			  =getattr(_a				,    _a  )
																		_SCREAMING_SNAKE_CASE			  =model_cls(_a  )
														except ImportError:
																		raise ImportError(
																		    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
																		    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''  )
										else:
														_SCREAMING_SNAKE_CASE			  =TF_MODEL_MAPPING[config.__class__](_a  )

										# encoder-decoder has vocab size saved differently
										_SCREAMING_SNAKE_CASE			  =config.vocab_size if hasattr(_a				,    '''vocab_size'''  ) else config.encoder.vocab_size
										_SCREAMING_SNAKE_CASE			  =random_input_ids(_a				,    _a				,    _a  )

										@run_with_tf_optimizations(self.args.eager_mode				,    self.args.use_xla  )
										def encoder_decoder_forward():
														return model(_a				,    decoder_input_ids=_a				,    training=_a  )

										@run_with_tf_optimizations(self.args.eager_mode				,    self.args.use_xla  )
										def encoder_forward():
														return model(_a				,    training=_a  )

										_SCREAMING_SNAKE_CASE			  =encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

										return _inference




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      str				,    _a					:      int				,    _a					:      int  )							->     Callable[[], None]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.config_dict[model_name]

										if self.args.eager_mode is not False:
														raise ValueError('''Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'''  )

										if self.args.fpaa:
														raise NotImplementedError('''Mixed precision is currently not supported.'''  )

										_SCREAMING_SNAKE_CASE			  =(
										    hasattr(_a				,    '''architectures'''  )
										    and isinstance(config.architectures				,    _a  )
										    and len(config.architectures  ) > 0
										)
										if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																		_SCREAMING_SNAKE_CASE			  ='''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
																		_SCREAMING_SNAKE_CASE			  =__import__('''transformers'''				,    fromlist=[model_class]  )
																		_SCREAMING_SNAKE_CASE			  =getattr(_a				,    _a  )
																		_SCREAMING_SNAKE_CASE			  =model_cls(_a  )
														except ImportError:
																		raise ImportError(
																		    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
																		    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''  )
										else:
														_SCREAMING_SNAKE_CASE			  =TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_a  )

										# encoder-decoder has vocab size saved differently
										_SCREAMING_SNAKE_CASE			  =config.vocab_size if hasattr(_a				,    '''vocab_size'''  ) else config.encoder.vocab_size
										_SCREAMING_SNAKE_CASE			  =random_input_ids(_a				,    _a				,    _a  )

										@run_with_tf_optimizations(self.args.eager_mode				,    self.args.use_xla  )
										def encoder_decoder_train():
														_SCREAMING_SNAKE_CASE			  =model(_a				,    decoder_input_ids=_a				,    labels=_a				,    training=_a  )[0]
														_SCREAMING_SNAKE_CASE			  =tf.gradients(_a				,    model.trainable_variables  )
														return gradients

										@run_with_tf_optimizations(self.args.eager_mode				,    self.args.use_xla  )
										def encoder_train():
														_SCREAMING_SNAKE_CASE			  =model(_a				,    labels=_a				,    training=_a  )[0]
														_SCREAMING_SNAKE_CASE			  =tf.gradients(_a				,    model.trainable_variables  )
														return gradients

										_SCREAMING_SNAKE_CASE			  =encoder_decoder_train if config.is_encoder_decoder else encoder_train

										return _train




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      Union[str, Any]  )							->     float:


										"""simple docstring"""





										with self.args.strategy.scope():
														try:
																		if self.args.is_tpu or self.args.use_xla:
																						# run additional 10 times to stabilize compilation for tpu
																						logger.info('''Do inference on TPU. Running model 5 times to stabilize compilation'''  )
																						timeit.repeat(_a				,    repeat=1				,    number=5  )

																		# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																		_SCREAMING_SNAKE_CASE			  =timeit.repeat(
																		    _a				,    repeat=self.args.repeat				,    number=10				,    )

																		return min(_a  ) / 10.0
														except ResourceExhaustedError as e:
																		self.print_fn(f"Doesn't fit on GPU. {e}"  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Callable[[], None]  )							->     [Memory, MemorySummary]:


										"""simple docstring"""





										logger.info(
										    '''Note that TensorFlow allocates more memory than '''
										    '''it might need to speed up computation. '''
										    '''The memory reported here corresponds to the memory '''
										    '''reported by `nvidia-smi`, which can vary depending '''
										    '''on total available memory on the GPU that is used.'''  )
										with self.args.strategy.scope():
														try:
																		if self.args.trace_memory_line_by_line:
																						if not self.args.eager_mode:
																										raise ValueError(
																										    '''`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'''
																										    ''' consumption line by line.'''  )
																						_SCREAMING_SNAKE_CASE			  =start_memory_tracing('''transformers'''  )

																		if self.args.is_tpu:
																						# tpu
																						raise NotImplementedError(
																						    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'''
																						    ''' with `args.memory=False`'''  )
																		elif self.args.is_gpu:
																						# gpu
																						if not is_pyanvml_available():
																										logger.warning(
																										    '''py3nvml not installed, we won\'t log GPU memory usage. '''
																										    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''  )
																										_SCREAMING_SNAKE_CASE			  ='''N/A'''
																						else:
																										logger.info(
																										    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes'''
																										    ''' running on the same GPU.'''  )
																										# init nvml
																										nvml.nvmlInit()
																										func()
																										_SCREAMING_SNAKE_CASE			  =nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
																										_SCREAMING_SNAKE_CASE			  =nvml.nvmlDeviceGetMemoryInfo(_a  )
																										_SCREAMING_SNAKE_CASE			  =meminfo.used
																										_SCREAMING_SNAKE_CASE			  =Memory(_a  )
																										# shutdown nvml
																										nvml.nvmlShutdown()
																		else:
																						# cpu
																						if self.args.trace_memory_line_by_line:
																										logger.info(
																										    '''When enabling line by line tracing, the max peak memory for CPU is inaccurate in'''
																										    ''' TensorFlow.'''  )
																										_SCREAMING_SNAKE_CASE			  =None
																						else:
																										_SCREAMING_SNAKE_CASE			  =measure_peak_memory_cpu(_a  )
																										_SCREAMING_SNAKE_CASE			  =Memory(_a  ) if isinstance(_a				,    _a  ) else memory_bytes
																		if self.args.trace_memory_line_by_line:
																						_SCREAMING_SNAKE_CASE			  =stop_memory_tracing(_a  )
																						if memory is None:
																										_SCREAMING_SNAKE_CASE			  =summary.total
																		else:
																						_SCREAMING_SNAKE_CASE			  =None

																		return memory, summary
														except ResourceExhaustedError as e:
																		self.print_fn(f"Doesn't fit on GPU. {e}"  )
																		return "N/A", None



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


snake_case_						:			str													=							logging.get_logger(__name__)

snake_case_						:			str													=							{
    '''facebook/data2vec-text-base''': '''https://huggingface.co/data2vec/resolve/main/config.json''',
}

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						"data2vec-text"




						def __init__(       self					:      Optional[int]				,    _a					:      Union[str, Any]=3_0522				,    _a					:      Any=768				,    _a					:      List[Any]=12				,    _a					:      Any=12				,    _a					:      List[str]=3072				,    _a					:      Optional[Any]="gelu"				,    _a					:      List[str]=0.1				,    _a					:      Optional[Any]=0.1				,    _a					:      Optional[Any]=512				,    _a					:      Any=2				,    _a					:      Optional[Any]=0.02				,    _a					:      Dict=1E-12				,    _a					:      int=1				,    _a					:      Any=0				,    _a					:      List[Any]=2				,    _a					:      Dict="absolute"				,    _a					:      Optional[Any]=True				,    _a					:      Dict=None				,    **_a					:      Optional[Any]				,    )							->     Union[str, Any]:


										"""simple docstring"""





										super().__init__(pad_token_id=_a				,    bos_token_id=_a				,    eos_token_id=_a				,    **_a  )

										_SCREAMING_SNAKE_CASE			  =vocab_size
										_SCREAMING_SNAKE_CASE			  =hidden_size
										_SCREAMING_SNAKE_CASE			  =num_hidden_layers
										_SCREAMING_SNAKE_CASE			  =num_attention_heads
										_SCREAMING_SNAKE_CASE			  =hidden_act
										_SCREAMING_SNAKE_CASE			  =intermediate_size
										_SCREAMING_SNAKE_CASE			  =hidden_dropout_prob
										_SCREAMING_SNAKE_CASE			  =attention_probs_dropout_prob
										_SCREAMING_SNAKE_CASE			  =max_position_embeddings
										_SCREAMING_SNAKE_CASE			  =type_vocab_size
										_SCREAMING_SNAKE_CASE			  =initializer_range
										_SCREAMING_SNAKE_CASE			  =layer_norm_eps
										_SCREAMING_SNAKE_CASE			  =position_embedding_type
										_SCREAMING_SNAKE_CASE			  =use_cache
										_SCREAMING_SNAKE_CASE			  =classifier_dropout

class 						A__							(		UpperCamelCase__  ):




						@property
						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Mapping[str, Mapping[int, str]]:


										"""simple docstring"""





										if self.task == "multiple-choice":
														_SCREAMING_SNAKE_CASE			  ={0: '''batch''', 1: '''choice''', 2: '''sequence'''}
										else:
														_SCREAMING_SNAKE_CASE			  ={0: '''batch''', 1: '''sequence'''}
										return OrderedDict(
										    [
										        ('''input_ids''', dynamic_axis),
										        ('''attention_mask''', dynamic_axis),
										    ]  )